From: Derrick Stolee <stolee@gmail.com>
To: git@vger.kernel.org
Cc: dstolee@microsoft.com, stolee@gmail.com, git@jeffhostetler.com,
peff@peff.net, gitster@pobox.com, Johannes.Shindelin@gmx.de,
jrnieder@gmail.com
Subject: [RFC PATCH 14/18] midx: load midx files when loading packs
Date: Sun, 7 Jan 2018 13:14:55 -0500 [thread overview]
Message-ID: <20180107181459.222909-15-dstolee@microsoft.com> (raw)
In-Reply-To: <20180107181459.222909-1-dstolee@microsoft.com>
Replace prepare_packed_git() with prepare_packed_git_internal(use_midx) to
allow some consumers of prepare_packed_git() with a way to load MIDX files.
Consumers should only use the new method if they are prepared to use the
midxed_git struct alongside the packed_git struct.
If a packfile is found that is not referenced by the current MIDX, then add
it to the packed_git struct. This is important to keep the MIDX useful after
adding packs due to "fetch" commands and when third-party tools (such as
libgit2) add packs directly to the repo.
If prepare_packed_git_internal is called with use_midx = 0, then unload the
MIDX file and reload the packfiles in to the packed_git struct.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
---
midx.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
midx.h | 6 ++++--
packfile.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++---------
packfile.h | 1 +
4 files changed, 117 insertions(+), 11 deletions(-)
diff --git a/midx.c b/midx.c
index 3ce2b736ea..a66763b9e3 100644
--- a/midx.c
+++ b/midx.c
@@ -22,6 +22,9 @@
#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
+/* MIDX-git global storage */
+struct midxed_git *midxed_git = 0;
+
char* get_midx_filename_oid(const char *pack_dir,
struct object_id *oid)
{
@@ -197,6 +200,45 @@ struct midxed_git *get_midxed_git(const char *pack_dir, struct object_id *oid)
return m;
}
+static char* get_midx_filename_dir(const char *pack_dir)
+{
+ struct object_id oid;
+ if (!get_midx_head_oid(pack_dir, &oid))
+ return 0;
+
+ return get_midx_filename_oid(pack_dir, &oid);
+}
+
+static int prepare_midxed_git_head(char *pack_dir, int local)
+{
+ struct midxed_git *m = midxed_git;
+ char *midx_head_path = get_midx_filename_dir(pack_dir);
+
+ if (!core_midx)
+ return 1;
+
+ if (midx_head_path) {
+ midxed_git = load_midxed_git_one(midx_head_path, pack_dir);
+ midxed_git->next = m;
+ FREE_AND_NULL(midx_head_path);
+ return 1;
+ }
+
+ return 0;
+}
+
+int prepare_midxed_git_objdir(char *obj_dir, int local)
+{
+ int ret;
+ struct strbuf pack_dir = STRBUF_INIT;
+ strbuf_addstr(&pack_dir, obj_dir);
+ strbuf_add(&pack_dir, "/pack", 5);
+
+ ret = prepare_midxed_git_head(pack_dir.buf, local);
+ strbuf_release(&pack_dir);
+ return ret;
+}
+
struct pack_midx_details_internal {
uint32_t pack_int_id;
uint32_t internal_offset;
@@ -677,3 +719,18 @@ int close_midx(struct midxed_git *m)
return 1;
}
+
+void close_all_midx(void)
+{
+ struct midxed_git *m = midxed_git;
+ struct midxed_git *next;
+
+ while (m) {
+ next = m->next;
+ close_midx(m);
+ free(m);
+ m = next;
+ }
+
+ midxed_git = 0;
+}
diff --git a/midx.h b/midx.h
index 27d48163e9..d8ede8121c 100644
--- a/midx.h
+++ b/midx.h
@@ -27,7 +27,7 @@ struct pack_midx_header {
uint32_t num_packs;
};
-struct midxed_git {
+extern struct midxed_git {
struct midxed_git *next;
int midx_fd;
@@ -81,9 +81,10 @@ struct midxed_git {
/* something like ".git/objects/pack" */
char pack_dir[FLEX_ARRAY]; /* more */
-};
+} *midxed_git;
extern struct midxed_git *get_midxed_git(const char *pack_dir, struct object_id *oid);
+extern int prepare_midxed_git_objdir(char *obj_dir, int local);
struct pack_midx_details {
uint32_t pack_int_id;
@@ -118,5 +119,6 @@ extern const char *write_midx_file(const char *pack_dir,
uint32_t nr_objects);
extern int close_midx(struct midxed_git *m);
+extern void close_all_midx(void);
#endif
diff --git a/packfile.c b/packfile.c
index c36420b33f..1c0822878b 100644
--- a/packfile.c
+++ b/packfile.c
@@ -8,6 +8,7 @@
#include "list.h"
#include "streaming.h"
#include "sha1-lookup.h"
+#include "midx.h"
char *odb_pack_name(struct strbuf *buf,
const unsigned char *sha1,
@@ -309,10 +310,22 @@ void close_pack(struct packed_git *p)
void close_all_packs(void)
{
struct packed_git *p;
+ struct midxed_git *m;
+
+ for (m = midxed_git; m; m = m->next) {
+ int i;
+ for (i = 0; i < m->num_packs; i++) {
+ p = m->packs[i];
+ if (p && p->do_not_close)
+ BUG("want to close pack marked 'do-not-close'");
+ else if (p)
+ close_pack(p);
+ }
+ }
for (p = packed_git; p; p = p->next)
if (p->do_not_close)
- die("BUG: want to close pack marked 'do-not-close'");
+ BUG("want to close pack marked 'do-not-close'");
else
close_pack(p);
}
@@ -748,6 +761,7 @@ static void prepare_packed_git_one(char *objdir, int local)
dirnamelen = path.len;
while ((de = readdir(dir)) != NULL) {
struct packed_git *p;
+ struct midxed_git *m;
size_t base_len;
if (is_dot_or_dotdot(de->d_name))
@@ -758,15 +772,23 @@ static void prepare_packed_git_one(char *objdir, int local)
base_len = path.len;
if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
+ strbuf_setlen(&path, base_len + 1);
+ strbuf_add(&path, "pack", 4);
+
/* Don't reopen a pack we already have. */
+ for (m = midxed_git; m; m = m->next)
+ if (!memcmp(m->pack_dir, path.buf, dirnamelen - 1) &&
+ contains_pack(m, path.buf + dirnamelen))
+ break;
for (p = packed_git; p; p = p->next) {
- size_t len;
- if (strip_suffix(p->pack_name, ".pack", &len) &&
- len == base_len &&
- !memcmp(p->pack_name, path.buf, len))
+ if (!strcmp(p->pack_name, path.buf))
break;
}
- if (p == NULL &&
+
+ strbuf_setlen(&path, base_len + 1);
+ strbuf_add(&path, "idx", 3);
+
+ if (m == NULL && p == NULL &&
/*
* See if it really is a valid .idx file with
* corresponding .pack file that we can map.
@@ -872,21 +894,45 @@ static void prepare_packed_git_mru(void)
}
static int prepare_packed_git_run_once = 0;
-void prepare_packed_git(void)
+static int prepare_midxed_git_run_once = 0;
+void prepare_packed_git_internal(int use_midx)
{
struct alternate_object_database *alt;
+ char *obj_dir;
+
+ if (prepare_midxed_git_run_once) {
+ if (!use_midx) {
+ prepare_midxed_git_run_once = 0;
+ close_all_midx();
+ reprepare_packed_git();
+ }
+ return;
+ }
if (prepare_packed_git_run_once)
return;
- prepare_packed_git_one(get_object_directory(), 1);
+
+ obj_dir = get_object_directory();
+
+ if (use_midx)
+ prepare_midxed_git_objdir(obj_dir, 1);
+ prepare_packed_git_one(obj_dir, 1);
prepare_alt_odb();
- for (alt = alt_odb_list; alt; alt = alt->next)
+ for (alt = alt_odb_list; alt; alt = alt->next) {
+ if (use_midx)
+ prepare_midxed_git_objdir(alt->path, 0);
prepare_packed_git_one(alt->path, 0);
+ }
rearrange_packed_git();
prepare_packed_git_mru();
prepare_packed_git_run_once = 1;
+ prepare_midxed_git_run_once = use_midx;
}
+void prepare_packed_git(void)
+{
+ prepare_packed_git_internal(0);
+}
void reprepare_packed_git(void)
{
approximate_object_count_valid = 0;
diff --git a/packfile.h b/packfile.h
index 7cf4771029..25bac91efb 100644
--- a/packfile.h
+++ b/packfile.h
@@ -32,6 +32,7 @@ extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_
#define PACKDIR_FILE_GARBAGE 4
extern void (*report_garbage)(unsigned seen_bits, const char *path);
+extern void prepare_packed_git_internal(int use_midx);
extern void prepare_packed_git(void);
extern void reprepare_packed_git(void);
extern void install_packed_git(struct packed_git *pack);
--
2.15.0
next prev parent reply other threads:[~2018-01-07 18:15 UTC|newest]
Thread overview: 48+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-01-07 18:14 [RFC PATCH 00/18] Multi-pack index (MIDX) Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 01/18] docs: Multi-Pack Index (MIDX) Design Notes Derrick Stolee
2018-01-08 19:32 ` Jonathan Tan
2018-01-08 20:35 ` Derrick Stolee
2018-01-08 22:06 ` Jonathan Tan
2018-01-07 18:14 ` [RFC PATCH 02/18] midx: specify midx file format Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 03/18] midx: create core.midx config setting Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 04/18] midx: write multi-pack indexes for an object list Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 05/18] midx: create midx builtin with --write mode Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 06/18] midx: add t5318-midx.sh test script Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 07/18] midx: teach midx --write to update midx-head Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 08/18] midx: teach git-midx to read midx file details Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 09/18] midx: find details of nth object in midx Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 10/18] midx: use existing midx when writing Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 11/18] midx: teach git-midx to clear midx files Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 12/18] midx: teach git-midx to delete expired files Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 13/18] t5318-midx.h: confirm git actions are stable Derrick Stolee
2018-01-07 18:14 ` Derrick Stolee [this message]
2018-01-07 18:14 ` [RFC PATCH 15/18] midx: use midx for approximate object count Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 16/18] midx: nth_midxed_object_oid() and bsearch_midx() Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 17/18] sha1_name: use midx for abbreviations Derrick Stolee
2018-01-07 18:14 ` [RFC PATCH 18/18] packfile: use midx for object loads Derrick Stolee
2018-01-07 22:42 ` [RFC PATCH 00/18] Multi-pack index (MIDX) Ævar Arnfjörð Bjarmason
2018-01-08 0:08 ` Derrick Stolee
2018-01-08 10:20 ` Jeff King
2018-01-08 10:27 ` Jeff King
2018-01-08 12:28 ` Ævar Arnfjörð Bjarmason
2018-01-08 13:43 ` Johannes Schindelin
2018-01-09 6:50 ` Jeff King
2018-01-09 13:05 ` Johannes Schindelin
2018-01-09 19:51 ` Stefan Beller
2018-01-09 20:12 ` Junio C Hamano
2018-01-09 20:16 ` Stefan Beller
2018-01-09 21:31 ` Junio C Hamano
2018-01-10 17:05 ` Johannes Schindelin
2018-01-10 10:57 ` Jeff King
2018-01-08 13:43 ` Derrick Stolee
2018-01-09 7:12 ` Jeff King
2018-01-08 11:43 ` Ævar Arnfjörð Bjarmason
2018-06-06 8:13 ` Ævar Arnfjörð Bjarmason
2018-06-06 10:27 ` [RFC PATCH 0/2] unconditional O(1) SHA-1 abbreviation Ævar Arnfjörð Bjarmason
2018-06-06 10:27 ` [RFC PATCH 1/2] config.c: use braces on multiple conditional arms Ævar Arnfjörð Bjarmason
2018-06-06 10:27 ` [RFC PATCH 2/2] sha1-name: add core.validateAbbrev & relative core.abbrev Ævar Arnfjörð Bjarmason
2018-06-06 12:04 ` Christian Couder
2018-06-06 11:24 ` [RFC PATCH 00/18] Multi-pack index (MIDX) Derrick Stolee
2018-01-10 18:25 ` Martin Fick
2018-01-10 19:39 ` Derrick Stolee
2018-01-10 21:01 ` Martin Fick
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180107181459.222909-15-dstolee@microsoft.com \
--to=stolee@gmail.com \
--cc=Johannes.Shindelin@gmx.de \
--cc=dstolee@microsoft.com \
--cc=git@jeffhostetler.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jrnieder@gmail.com \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).