From: "René Scharfe" <l.s.r@web.de>
To: "Jeff King" <peff@peff.net>, "Geert Jansen" <gerardu@amazon.com>,
"Ævar Arnfjörð Bjarmason" <avarab@gmail.com>
Cc: Junio C Hamano <gitster@pobox.com>,
"git@vger.kernel.org" <git@vger.kernel.org>,
Takuto Ikuta <tikuta@chromium.org>
Subject: Re: [PATCH 8/9] sha1-file: use loose object cache for quick existence check
Date: Tue, 27 Nov 2018 21:48:57 +0100 [thread overview]
Message-ID: <ec25f85c-4b8c-0b83-addb-074957de1e1c@web.de> (raw)
In-Reply-To: <20181112145442.GH7400@sigill.intra.peff.net>
Am 12.11.2018 um 15:54 schrieb Jeff King:
> diff --git a/sha1-file.c b/sha1-file.c
> index 4aae716a37..e53da0b701 100644
> --- a/sha1-file.c
> +++ b/sha1-file.c
> @@ -921,6 +921,24 @@ static int open_sha1_file(struct repository *r,
> return -1;
> }
>
> +static int quick_has_loose(struct repository *r,
> + const unsigned char *sha1)
> +{
> + int subdir_nr = sha1[0];
> + struct object_id oid;
> + struct object_directory *odb;
> +
> + hashcpy(oid.hash, sha1);
> +
> + prepare_alt_odb(r);
> + for (odb = r->objects->odb; odb; odb = odb->next) {
> + odb_load_loose_cache(odb, subdir_nr);
Is this thread-safe? What happens if e.g. one index-pack thread resizes
the array while another one sorts it?
Loading the cache explicitly up-front would avoid that, and improves
performance a bit in my (very limited) tests on an SSD. Demo patch for
next at the bottom. How does it do against your test cases?
> + if (oid_array_lookup(&odb->loose_objects_cache, &oid) >= 0)
> + return 1;
> + }
> + return 0;
> +}
> +
> /*
> * Map the loose object at "path" if it is not NULL, or the path found by
> * searching for a loose object named "sha1".
> @@ -1171,6 +1189,8 @@ static int sha1_loose_object_info(struct repository *r,
> if (!oi->typep && !oi->type_name && !oi->sizep && !oi->contentp) {
> const char *path;
> struct stat st;
> + if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK))
> + return quick_has_loose(r, sha1) ? 0 : -1;
> if (stat_sha1_file(r, sha1, &st, &path) < 0)
> return -1;
> if (oi->disk_sizep)
>
builtin/fetch.c | 2 ++
builtin/index-pack.c | 2 ++
fetch-pack.c | 2 ++
object-store.h | 1 +
sha1-file.c | 30 +++++++++++++++++++++++++++---
5 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/builtin/fetch.c b/builtin/fetch.c
index e0140327aa..4b031f5da5 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -301,6 +301,8 @@ static void find_non_local_tags(const struct ref *refs,
refname_hash_init(&existing_refs);
refname_hash_init(&remote_refs);
+ repo_load_loose_cache(the_repository);
+
for_each_ref(add_one_refname, &existing_refs);
for (ref = refs; ref; ref = ref->next) {
if (!starts_with(ref->name, "refs/tags/"))
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index ac1f4ea9a7..7fc6321c77 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -1772,6 +1772,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
if (show_stat)
obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat));
ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry));
+ if (startup_info->have_repository)
+ repo_load_loose_cache(the_repository);
parse_pack_objects(pack_hash);
if (report_end_of_input)
write_in_full(2, "\0", 1);
diff --git a/fetch-pack.c b/fetch-pack.c
index dd6700bda9..96c4624d9e 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -656,6 +656,8 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
save_commit_buffer = 0;
+ repo_load_loose_cache(the_repository);
+
for (ref = *refs; ref; ref = ref->next) {
struct object *o;
diff --git a/object-store.h b/object-store.h
index 8dceed0f31..f98dd3c857 100644
--- a/object-store.h
+++ b/object-store.h
@@ -53,6 +53,7 @@ void add_to_alternates_memory(const char *dir);
* from 0 to 255 inclusive).
*/
void odb_load_loose_cache(struct object_directory *odb, int subdir_nr);
+void repo_load_loose_cache(struct repository *r);
struct packed_git {
struct packed_git *next;
diff --git a/sha1-file.c b/sha1-file.c
index 05f63dfd4e..ae12f0a198 100644
--- a/sha1-file.c
+++ b/sha1-file.c
@@ -921,10 +921,19 @@ static int open_sha1_file(struct repository *r,
return -1;
}
+static int quick_has_loose_odb(struct object_directory *odb,
+ const struct object_id *oid)
+{
+ int subdir_nr = oid->hash[0];
+
+ if (odb->loose_objects_subdir_seen[subdir_nr])
+ return oid_array_lookup(&odb->loose_objects_cache, oid) >= 0;
+ return check_and_freshen_odb(odb, oid, 0);
+}
+
static int quick_has_loose(struct repository *r,
const unsigned char *sha1)
{
- int subdir_nr = sha1[0];
struct object_id oid;
struct object_directory *odb;
@@ -932,8 +941,7 @@ static int quick_has_loose(struct repository *r,
prepare_alt_odb(r);
for (odb = r->objects->odb; odb; odb = odb->next) {
- odb_load_loose_cache(odb, subdir_nr);
- if (oid_array_lookup(&odb->loose_objects_cache, &oid) >= 0)
+ if (quick_has_loose_odb(odb, &oid))
return 1;
}
return 0;
@@ -2178,6 +2186,22 @@ void odb_load_loose_cache(struct object_directory *odb, int subdir_nr)
strbuf_release(&buf);
}
+void repo_load_loose_cache(struct repository *r)
+{
+ struct object_directory *odb;
+
+ prepare_alt_odb(r);
+ for (odb = r->objects->odb; odb; odb = odb->next) {
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(odb->loose_objects_subdir_seen); i++)
+ odb_load_loose_cache(odb, i);
+
+ /* Sort as a side-effect, only read the cache from here on. */
+ oid_array_lookup(&odb->loose_objects_cache, &null_oid);
+ }
+}
+
static int check_stream_sha1(git_zstream *stream,
const char *hdr,
unsigned long size,
next prev parent reply other threads:[~2018-11-27 20:49 UTC|newest]
Thread overview: 99+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-10-25 18:38 [RFC PATCH] index-pack: improve performance on NFS Jansen, Geert
2018-10-26 0:21 ` Junio C Hamano
2018-10-26 20:38 ` Ævar Arnfjörð Bjarmason
2018-10-27 7:26 ` Junio C Hamano
2018-10-27 9:33 ` Jeff King
2018-10-27 11:22 ` Ævar Arnfjörð Bjarmason
2018-10-28 22:50 ` [PATCH 0/4] index-pack: optionally turn off SHA-1 collision checking Ævar Arnfjörð Bjarmason
2018-10-30 2:49 ` Geert Jansen
2018-10-30 9:04 ` Junio C Hamano
2018-10-30 18:43 ` [PATCH v2 0/3] index-pack: test updates Ævar Arnfjörð Bjarmason
2018-11-13 20:19 ` [PATCH v3] index-pack: add ability to disable SHA-1 collision check Ævar Arnfjörð Bjarmason
2018-11-14 7:09 ` Junio C Hamano
2018-11-14 12:40 ` Ævar Arnfjörð Bjarmason
2018-10-30 18:43 ` [PATCH v2 1/3] pack-objects test: modernize style Ævar Arnfjörð Bjarmason
2018-10-30 18:43 ` [PATCH v2 2/3] pack-objects tests: don't leave test .git corrupt at end Ævar Arnfjörð Bjarmason
2018-10-30 18:43 ` [PATCH v2 3/3] index-pack tests: don't leave test repo dirty " Ævar Arnfjörð Bjarmason
2018-10-28 22:50 ` [PATCH 1/4] pack-objects test: modernize style Ævar Arnfjörð Bjarmason
2018-10-28 22:50 ` [PATCH 2/4] pack-objects tests: don't leave test .git corrupt at end Ævar Arnfjörð Bjarmason
2018-10-28 22:50 ` [PATCH 3/4] index-pack tests: don't leave test repo dirty " Ævar Arnfjörð Bjarmason
2018-10-28 22:50 ` [PATCH 4/4] index-pack: add ability to disable SHA-1 collision check Ævar Arnfjörð Bjarmason
2018-10-29 15:04 ` [RFC PATCH] index-pack: improve performance on NFS Jeff King
2018-10-29 15:09 ` Jeff King
2018-10-29 19:36 ` Ævar Arnfjörð Bjarmason
2018-10-29 23:27 ` Jeff King
2018-11-07 22:55 ` Geert Jansen
2018-11-08 12:02 ` Jeff King
2018-11-08 20:58 ` Geert Jansen
2018-11-08 21:18 ` Jeff King
2018-11-08 21:55 ` Geert Jansen
2018-11-08 22:20 ` Ævar Arnfjörð Bjarmason
2018-11-09 10:11 ` Ævar Arnfjörð Bjarmason
2018-11-12 14:31 ` Jeff King
2018-11-12 14:46 ` [PATCH 0/9] caching loose objects Jeff King
2018-11-12 14:46 ` [PATCH 1/9] fsck: do not reuse child_process structs Jeff King
2018-11-12 15:26 ` Derrick Stolee
2018-11-12 14:47 ` [PATCH 2/9] submodule--helper: prefer strip_suffix() to ends_with() Jeff King
2018-11-12 18:23 ` Stefan Beller
2018-11-12 14:48 ` [PATCH 3/9] rename "alternate_object_database" to "object_directory" Jeff King
2018-11-12 15:30 ` Derrick Stolee
2018-11-12 15:36 ` Jeff King
2018-11-12 19:41 ` Ramsay Jones
2018-11-12 14:48 ` [PATCH 4/9] sha1_file_name(): overwrite buffer instead of appending Jeff King
2018-11-12 15:32 ` Derrick Stolee
2018-11-12 14:49 ` [PATCH 5/9] handle alternates paths the same as the main object dir Jeff King
2018-11-12 15:38 ` Derrick Stolee
2018-11-12 15:46 ` Jeff King
2018-11-12 15:50 ` Derrick Stolee
2018-11-12 14:50 ` [PATCH 6/9] sha1-file: use an object_directory for " Jeff King
2018-11-12 15:48 ` Derrick Stolee
2018-11-12 16:09 ` Jeff King
2018-11-12 19:04 ` Stefan Beller
2018-11-22 17:42 ` Jeff King
2018-11-12 18:48 ` Stefan Beller
2018-11-12 14:50 ` [PATCH 7/9] object-store: provide helpers for loose_objects_cache Jeff King
2018-11-12 19:24 ` René Scharfe
2018-11-12 20:16 ` Jeff King
2018-11-12 14:54 ` [PATCH 8/9] sha1-file: use loose object cache for quick existence check Jeff King
2018-11-12 16:00 ` Derrick Stolee
2018-11-12 16:01 ` Ævar Arnfjörð Bjarmason
2018-11-12 16:21 ` Jeff King
2018-11-12 22:18 ` Ævar Arnfjörð Bjarmason
2018-11-12 22:30 ` Ævar Arnfjörð Bjarmason
2018-11-13 10:02 ` Ævar Arnfjörð Bjarmason
2018-11-14 18:21 ` René Scharfe
2018-12-02 10:52 ` René Scharfe
2018-12-03 22:04 ` Jeff King
2018-12-04 21:45 ` René Scharfe
2018-12-05 4:46 ` Jeff King
2018-12-05 6:02 ` René Scharfe
2018-12-05 6:51 ` Jeff King
2018-12-05 8:15 ` Jeff King
2018-12-05 18:41 ` René Scharfe
2018-12-05 20:17 ` Jeff King
2018-11-12 22:44 ` Geert Jansen
2018-11-27 20:48 ` René Scharfe [this message]
2018-12-01 19:49 ` Jeff King
2018-11-12 14:55 ` [PATCH 9/9] fetch-pack: drop custom loose object cache Jeff King
2018-11-12 19:25 ` René Scharfe
2018-11-12 19:32 ` Ævar Arnfjörð Bjarmason
2018-11-12 20:07 ` Jeff King
2018-11-12 20:13 ` René Scharfe
2018-11-12 16:02 ` [PATCH 0/9] caching loose objects Derrick Stolee
2018-11-12 19:10 ` Stefan Beller
2018-11-09 13:43 ` [RFC PATCH] index-pack: improve performance on NFS Ævar Arnfjörð Bjarmason
2018-11-09 16:08 ` Duy Nguyen
2018-11-10 14:04 ` Ævar Arnfjörð Bjarmason
2018-11-12 14:34 ` Jeff King
2018-11-12 22:58 ` Geert Jansen
2018-10-27 14:04 ` Duy Nguyen
2018-10-29 15:18 ` Jeff King
2018-10-29 0:48 ` Junio C Hamano
2018-10-29 15:20 ` Jeff King
2018-10-29 18:43 ` Ævar Arnfjörð Bjarmason
2018-10-29 21:34 ` Geert Jansen
2018-10-29 21:50 ` Jeff King
2018-10-29 22:21 ` Geert Jansen
2018-10-29 22:27 ` Jeff King
2018-10-29 22:35 ` Stefan Beller
2018-10-29 23:29 ` Jeff King
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=ec25f85c-4b8c-0b83-addb-074957de1e1c@web.de \
--to=l.s.r@web.de \
--cc=avarab@gmail.com \
--cc=gerardu@amazon.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=peff@peff.net \
--cc=tikuta@chromium.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).