* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 0:46 [PATCH] Enable index-pack threading in msysgit szager
@ 2014-03-19 7:30 ` Duy Nguyen
2014-03-19 7:50 ` Stefan Zager
2014-03-19 20:57 ` Junio C Hamano
` (2 subsequent siblings)
3 siblings, 1 reply; 19+ messages in thread
From: Duy Nguyen @ 2014-03-19 7:30 UTC (permalink / raw)
To: Stefan Zager; +Cc: Git Mailing List
On Wed, Mar 19, 2014 at 7:46 AM, <szager@chromium.org> wrote:
> This adds a Windows implementation of pread. Note that it is NOT
> safe to intersperse calls to read() and pread() on a file
> descriptor. According to the ReadFile spec, using the 'overlapped'
> argument should not affect the implicit position pointer of the
> descriptor. Experiments have shown that this is, in fact, a lie.
If I understand it correctly, new pread() is added because
compat/pread.c does not work because of some other read() in between?
Where are those read() (I can only see one in index-pack.c, but there
could be some hidden read()..)
>
> To accomodate that fact, this change also incorporates:
>
> http://article.gmane.org/gmane.comp.version-control.git/196042
>
> ... which gives each index-pack thread its own file descriptor.
> ---
> builtin/index-pack.c | 21 ++++++++++++++++-----
> compat/mingw.c | 31 ++++++++++++++++++++++++++++++-
> compat/mingw.h | 3 +++
> config.mak.uname | 1 -
> 4 files changed, 49 insertions(+), 7 deletions(-)
>
> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
> index 2f37a38..c02dd4c 100644
> --- a/builtin/index-pack.c
> +++ b/builtin/index-pack.c
> @@ -51,6 +51,7 @@ struct thread_local {
> #endif
> struct base_data *base_cache;
> size_t base_cache_used;
> + int pack_fd;
> };
>
> /*
> @@ -91,7 +92,8 @@ static off_t consumed_bytes;
> static unsigned deepest_delta;
> static git_SHA_CTX input_ctx;
> static uint32_t input_crc32;
> -static int input_fd, output_fd, pack_fd;
> +static const char *curr_pack;
> +static int input_fd, output_fd;
>
> #ifndef NO_PTHREADS
>
> @@ -134,6 +136,7 @@ static inline void unlock_mutex(pthread_mutex_t *mutex)
> */
> static void init_thread(void)
> {
> + int i;
> init_recursive_mutex(&read_mutex);
> pthread_mutex_init(&counter_mutex, NULL);
> pthread_mutex_init(&work_mutex, NULL);
> @@ -141,11 +144,17 @@ static void init_thread(void)
> pthread_mutex_init(&deepest_delta_mutex, NULL);
> pthread_key_create(&key, NULL);
> thread_data = xcalloc(nr_threads, sizeof(*thread_data));
> + for (i = 0; i < nr_threads; i++) {
> + thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
> + if (thread_data[i].pack_fd == -1)
> + die_errno("unable to open %s", curr_pack);
> + }
> threads_active = 1;
> }
>
> static void cleanup_thread(void)
> {
> + int i;
> if (!threads_active)
> return;
> threads_active = 0;
> @@ -155,6 +164,8 @@ static void cleanup_thread(void)
> if (show_stat)
> pthread_mutex_destroy(&deepest_delta_mutex);
> pthread_key_delete(key);
> + for (i = 0; i < nr_threads; i++)
> + close(thread_data[i].pack_fd);
> free(thread_data);
> }
>
> @@ -288,13 +299,13 @@ static const char *open_pack_file(const char *pack_name)
> output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
> if (output_fd < 0)
> die_errno(_("unable to create '%s'"), pack_name);
> - pack_fd = output_fd;
> + nothread_data.pack_fd = output_fd;
> } else {
> input_fd = open(pack_name, O_RDONLY);
> if (input_fd < 0)
> die_errno(_("cannot open packfile '%s'"), pack_name);
> output_fd = -1;
> - pack_fd = input_fd;
> + nothread_data.pack_fd = input_fd;
> }
> git_SHA1_Init(&input_ctx);
> return pack_name;
> @@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
>
> do {
> ssize_t n = (len < 64*1024) ? len : 64*1024;
> - n = pread(pack_fd, inbuf, n, from);
> + n = pread(get_thread_data()->pack_fd, inbuf, n, from);
> if (n < 0)
> die_errno(_("cannot pread pack file"));
> if (!n)
> @@ -1490,7 +1501,7 @@ static void show_pack_info(int stat_only)
> int cmd_index_pack(int argc, const char **argv, const char *prefix)
> {
> int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
> - const char *curr_pack, *curr_index;
> + const char *curr_index;
> const char *index_name = NULL, *pack_name = NULL;
> const char *keep_name = NULL, *keep_msg = NULL;
> char *index_name_buf = NULL, *keep_name_buf = NULL;
> diff --git a/compat/mingw.c b/compat/mingw.c
> index 383cafe..6cc85d6 100644
> --- a/compat/mingw.c
> +++ b/compat/mingw.c
> @@ -329,7 +329,36 @@ int mingw_mkdir(const char *path, int mode)
> return ret;
> }
>
> -int mingw_open (const char *filename, int oflags, ...)
> +
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset)
> +{
> + HANDLE hand = (HANDLE)_get_osfhandle(fd);
> + if (hand == INVALID_HANDLE_VALUE) {
> + errno = EBADF;
> + return -1;
> + }
> +
> + LARGE_INTEGER offset_value;
> + offset_value.QuadPart = offset;
> +
> + DWORD bytes_read = 0;
> + OVERLAPPED overlapped = {0};
> + overlapped.Offset = offset_value.LowPart;
> + overlapped.OffsetHigh = offset_value.HighPart;
> + BOOL result = ReadFile(hand, buf, count, &bytes_read, &overlapped);
> +
> + ssize_t ret = bytes_read;
> +
> + if (!result && GetLastError() != ERROR_HANDLE_EOF)
> + {
> + errno = err_win_to_posix(GetLastError());
> + ret = -1;
> + }
> +
> + return ret;
> +}
> +
> +int mingw_open(const char *filename, int oflags, ...)
> {
> va_list args;
> unsigned mode;
> diff --git a/compat/mingw.h b/compat/mingw.h
> index 08b83fe..377ba50 100644
> --- a/compat/mingw.h
> +++ b/compat/mingw.h
> @@ -174,6 +174,9 @@ int mingw_unlink(const char *pathname);
> int mingw_rmdir(const char *path);
> #define rmdir mingw_rmdir
>
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset);
> +#define pread mingw_pread
> +
> int mingw_open (const char *filename, int oflags, ...);
> #define open mingw_open
>
> diff --git a/config.mak.uname b/config.mak.uname
> index e8acc39..b405524 100644
> --- a/config.mak.uname
> +++ b/config.mak.uname
> @@ -474,7 +474,6 @@ ifeq ($(uname_S),NONSTOP_KERNEL)
> endif
> ifneq (,$(findstring MINGW,$(uname_S)))
> pathsep = ;
> - NO_PREAD = YesPlease
> NEEDS_CRYPTO_WITH_SSL = YesPlease
> NO_LIBGEN_H = YesPlease
> NO_POLL = YesPlease
> --
> 1.9.0.279.gdc9e3eb
>
> --
> To unsubscribe from this list: send the line "unsubscribe git" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
Duy
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 7:30 ` Duy Nguyen
@ 2014-03-19 7:50 ` Stefan Zager
2014-03-19 10:28 ` Duy Nguyen
0 siblings, 1 reply; 19+ messages in thread
From: Stefan Zager @ 2014-03-19 7:50 UTC (permalink / raw)
To: Duy Nguyen; +Cc: Stefan Zager, Git Mailing List
On Wed, Mar 19, 2014 at 12:30 AM, Duy Nguyen <pclouds@gmail.com> wrote:
> On Wed, Mar 19, 2014 at 7:46 AM, <szager@chromium.org> wrote:
>> This adds a Windows implementation of pread. Note that it is NOT
>> safe to intersperse calls to read() and pread() on a file
>> descriptor. According to the ReadFile spec, using the 'overlapped'
>> argument should not affect the implicit position pointer of the
>> descriptor. Experiments have shown that this is, in fact, a lie.
>
> If I understand it correctly, new pread() is added because
> compat/pread.c does not work because of some other read() in between?
> Where are those read() (I can only see one in index-pack.c, but there
> could be some hidden read()..)
I *think* it's the call to fixup_pack_header_footer(), but I'm not 100% sure.
I suppose it would be possible to fix the immediate problem just by
using one fd per thread, without a new pread implementation. But it
seems better overall to have a pread() implementation that is
thread-safe as long as read() and pread() aren't interspersed; and
then convert all existing read() calls to pread(). That would be a
good follow-up patch...
>
>>
>> To accomodate that fact, this change also incorporates:
>>
>> http://article.gmane.org/gmane.comp.version-control.git/196042
>>
>> ... which gives each index-pack thread its own file descriptor.
>> ---
>> builtin/index-pack.c | 21 ++++++++++++++++-----
>> compat/mingw.c | 31 ++++++++++++++++++++++++++++++-
>> compat/mingw.h | 3 +++
>> config.mak.uname | 1 -
>> 4 files changed, 49 insertions(+), 7 deletions(-)
>>
>> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
>> index 2f37a38..c02dd4c 100644
>> --- a/builtin/index-pack.c
>> +++ b/builtin/index-pack.c
>> @@ -51,6 +51,7 @@ struct thread_local {
>> #endif
>> struct base_data *base_cache;
>> size_t base_cache_used;
>> + int pack_fd;
>> };
>>
>> /*
>> @@ -91,7 +92,8 @@ static off_t consumed_bytes;
>> static unsigned deepest_delta;
>> static git_SHA_CTX input_ctx;
>> static uint32_t input_crc32;
>> -static int input_fd, output_fd, pack_fd;
>> +static const char *curr_pack;
>> +static int input_fd, output_fd;
>>
>> #ifndef NO_PTHREADS
>>
>> @@ -134,6 +136,7 @@ static inline void unlock_mutex(pthread_mutex_t *mutex)
>> */
>> static void init_thread(void)
>> {
>> + int i;
>> init_recursive_mutex(&read_mutex);
>> pthread_mutex_init(&counter_mutex, NULL);
>> pthread_mutex_init(&work_mutex, NULL);
>> @@ -141,11 +144,17 @@ static void init_thread(void)
>> pthread_mutex_init(&deepest_delta_mutex, NULL);
>> pthread_key_create(&key, NULL);
>> thread_data = xcalloc(nr_threads, sizeof(*thread_data));
>> + for (i = 0; i < nr_threads; i++) {
>> + thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
>> + if (thread_data[i].pack_fd == -1)
>> + die_errno("unable to open %s", curr_pack);
>> + }
>> threads_active = 1;
>> }
>>
>> static void cleanup_thread(void)
>> {
>> + int i;
>> if (!threads_active)
>> return;
>> threads_active = 0;
>> @@ -155,6 +164,8 @@ static void cleanup_thread(void)
>> if (show_stat)
>> pthread_mutex_destroy(&deepest_delta_mutex);
>> pthread_key_delete(key);
>> + for (i = 0; i < nr_threads; i++)
>> + close(thread_data[i].pack_fd);
>> free(thread_data);
>> }
>>
>> @@ -288,13 +299,13 @@ static const char *open_pack_file(const char *pack_name)
>> output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
>> if (output_fd < 0)
>> die_errno(_("unable to create '%s'"), pack_name);
>> - pack_fd = output_fd;
>> + nothread_data.pack_fd = output_fd;
>> } else {
>> input_fd = open(pack_name, O_RDONLY);
>> if (input_fd < 0)
>> die_errno(_("cannot open packfile '%s'"), pack_name);
>> output_fd = -1;
>> - pack_fd = input_fd;
>> + nothread_data.pack_fd = input_fd;
>> }
>> git_SHA1_Init(&input_ctx);
>> return pack_name;
>> @@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
>>
>> do {
>> ssize_t n = (len < 64*1024) ? len : 64*1024;
>> - n = pread(pack_fd, inbuf, n, from);
>> + n = pread(get_thread_data()->pack_fd, inbuf, n, from);
>> if (n < 0)
>> die_errno(_("cannot pread pack file"));
>> if (!n)
>> @@ -1490,7 +1501,7 @@ static void show_pack_info(int stat_only)
>> int cmd_index_pack(int argc, const char **argv, const char *prefix)
>> {
>> int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
>> - const char *curr_pack, *curr_index;
>> + const char *curr_index;
>> const char *index_name = NULL, *pack_name = NULL;
>> const char *keep_name = NULL, *keep_msg = NULL;
>> char *index_name_buf = NULL, *keep_name_buf = NULL;
>> diff --git a/compat/mingw.c b/compat/mingw.c
>> index 383cafe..6cc85d6 100644
>> --- a/compat/mingw.c
>> +++ b/compat/mingw.c
>> @@ -329,7 +329,36 @@ int mingw_mkdir(const char *path, int mode)
>> return ret;
>> }
>>
>> -int mingw_open (const char *filename, int oflags, ...)
>> +
>> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset)
>> +{
>> + HANDLE hand = (HANDLE)_get_osfhandle(fd);
>> + if (hand == INVALID_HANDLE_VALUE) {
>> + errno = EBADF;
>> + return -1;
>> + }
>> +
>> + LARGE_INTEGER offset_value;
>> + offset_value.QuadPart = offset;
>> +
>> + DWORD bytes_read = 0;
>> + OVERLAPPED overlapped = {0};
>> + overlapped.Offset = offset_value.LowPart;
>> + overlapped.OffsetHigh = offset_value.HighPart;
>> + BOOL result = ReadFile(hand, buf, count, &bytes_read, &overlapped);
>> +
>> + ssize_t ret = bytes_read;
>> +
>> + if (!result && GetLastError() != ERROR_HANDLE_EOF)
>> + {
>> + errno = err_win_to_posix(GetLastError());
>> + ret = -1;
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +int mingw_open(const char *filename, int oflags, ...)
>> {
>> va_list args;
>> unsigned mode;
>> diff --git a/compat/mingw.h b/compat/mingw.h
>> index 08b83fe..377ba50 100644
>> --- a/compat/mingw.h
>> +++ b/compat/mingw.h
>> @@ -174,6 +174,9 @@ int mingw_unlink(const char *pathname);
>> int mingw_rmdir(const char *path);
>> #define rmdir mingw_rmdir
>>
>> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset);
>> +#define pread mingw_pread
>> +
>> int mingw_open (const char *filename, int oflags, ...);
>> #define open mingw_open
>>
>> diff --git a/config.mak.uname b/config.mak.uname
>> index e8acc39..b405524 100644
>> --- a/config.mak.uname
>> +++ b/config.mak.uname
>> @@ -474,7 +474,6 @@ ifeq ($(uname_S),NONSTOP_KERNEL)
>> endif
>> ifneq (,$(findstring MINGW,$(uname_S)))
>> pathsep = ;
>> - NO_PREAD = YesPlease
>> NEEDS_CRYPTO_WITH_SSL = YesPlease
>> NO_LIBGEN_H = YesPlease
>> NO_POLL = YesPlease
>> --
>> 1.9.0.279.gdc9e3eb
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe git" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
>
>
> --
> Duy
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 7:50 ` Stefan Zager
@ 2014-03-19 10:28 ` Duy Nguyen
2014-03-19 16:57 ` Stefan Zager
0 siblings, 1 reply; 19+ messages in thread
From: Duy Nguyen @ 2014-03-19 10:28 UTC (permalink / raw)
To: Stefan Zager; +Cc: Git Mailing List
On Wed, Mar 19, 2014 at 2:50 PM, Stefan Zager <szager@chromium.org> wrote:
> On Wed, Mar 19, 2014 at 12:30 AM, Duy Nguyen <pclouds@gmail.com> wrote:
>> On Wed, Mar 19, 2014 at 7:46 AM, <szager@chromium.org> wrote:
>>> This adds a Windows implementation of pread. Note that it is NOT
>>> safe to intersperse calls to read() and pread() on a file
>>> descriptor. According to the ReadFile spec, using the 'overlapped'
>>> argument should not affect the implicit position pointer of the
>>> descriptor. Experiments have shown that this is, in fact, a lie.
>>
>> If I understand it correctly, new pread() is added because
>> compat/pread.c does not work because of some other read() in between?
>> Where are those read() (I can only see one in index-pack.c, but there
>> could be some hidden read()..)
>
> I *think* it's the call to fixup_pack_header_footer(), but I'm not 100% sure.
>
> I suppose it would be possible to fix the immediate problem just by
> using one fd per thread, without a new pread implementation. But it
> seems better overall to have a pread() implementation that is
> thread-safe as long as read() and pread() aren't interspersed; and
> then convert all existing read() calls to pread(). That would be a
> good follow-up patch...
I still don't understand how compat/pread.c does not work with pack_fd
per thread. I don't have Windows to test, but I forced compat/pread.c
on on Linux with similar pack_fd changes and it worked fine, helgrind
only complained about progress.c.
A pread() implementation that is thread-safe with condition sounds
like an invite for trouble later. And I don't think converting read()
to pread() is a good idea. Platforms that rely on pread() will hit
first because of more use of compat/pread.c. read() seeks while
pread() does not, so we have to audit more code..
--
Duy
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 10:28 ` Duy Nguyen
@ 2014-03-19 16:57 ` Stefan Zager
2014-03-19 19:15 ` Stefan Zager
0 siblings, 1 reply; 19+ messages in thread
From: Stefan Zager @ 2014-03-19 16:57 UTC (permalink / raw)
To: Duy Nguyen; +Cc: Stefan Zager, Git Mailing List
On Wed, Mar 19, 2014 at 3:28 AM, Duy Nguyen <pclouds@gmail.com> wrote:
> On Wed, Mar 19, 2014 at 2:50 PM, Stefan Zager <szager@chromium.org> wrote:
>>
>> I suppose it would be possible to fix the immediate problem just by
>> using one fd per thread, without a new pread implementation. But it
>> seems better overall to have a pread() implementation that is
>> thread-safe as long as read() and pread() aren't interspersed; and
>> then convert all existing read() calls to pread(). That would be a
>> good follow-up patch...
>
> I still don't understand how compat/pread.c does not work with pack_fd
> per thread. I don't have Windows to test, but I forced compat/pread.c
> on on Linux with similar pack_fd changes and it worked fine, helgrind
> only complained about progress.c.
>
> A pread() implementation that is thread-safe with condition sounds
> like an invite for trouble later. And I don't think converting read()
> to pread() is a good idea. Platforms that rely on pread() will hit
> first because of more use of compat/pread.c. read() seeks while
> pread() does not, so we have to audit more code..
Using one fd per thread is all well and good for something like
index-pack, which only accesses a single pack file. But using that
heuristic to add threading elsewhere is probably not going to work.
For example, I have a patch in progress to add threading to checkout,
and another one planned to add threading to status. In both cases, we
would need one fd per thread per pack file, which is pretty
ridiculous.
There really aren't very many calls to read() in the code. I don't
think it would be very difficult to eliminate the remaining ones. The
more interesting question, I think is: what platforms still don't have
a thread-safe pread implementation?
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 16:57 ` Stefan Zager
@ 2014-03-19 19:15 ` Stefan Zager
0 siblings, 0 replies; 19+ messages in thread
From: Stefan Zager @ 2014-03-19 19:15 UTC (permalink / raw)
To: Stefan Zager; +Cc: Duy Nguyen, Git Mailing List
On Wed, Mar 19, 2014 at 9:57 AM, Stefan Zager <szager@chromium.org> wrote:
>>
>> I still don't understand how compat/pread.c does not work with pack_fd
>> per thread. I don't have Windows to test, but I forced compat/pread.c
>> on on Linux with similar pack_fd changes and it worked fine, helgrind
>> only complained about progress.c.
>>
>> A pread() implementation that is thread-safe with condition sounds
>> like an invite for trouble later. And I don't think converting read()
>> to pread() is a good idea. Platforms that rely on pread() will hit
>> first because of more use of compat/pread.c. read() seeks while
>> pread() does not, so we have to audit more code..
>
> Using one fd per thread is all well and good for something like
> index-pack, which only accesses a single pack file. But using that
> heuristic to add threading elsewhere is probably not going to work.
> For example, I have a patch in progress to add threading to checkout,
> and another one planned to add threading to status. In both cases, we
> would need one fd per thread per pack file, which is pretty
> ridiculous.
>
> There really aren't very many calls to read() in the code. I don't
> think it would be very difficult to eliminate the remaining ones. The
> more interesting question, I think is: what platforms still don't have
> a thread-safe pread implementation?
I don't want to go too deep down the rabbit hole here. We don't have
to solve the read() vs. pread() issue once for all right now; that can
wait for another day. The pread() implementation in this patch is
certainly no worse than the one in compat/pread.c.
Stefan
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 0:46 [PATCH] Enable index-pack threading in msysgit szager
2014-03-19 7:30 ` Duy Nguyen
@ 2014-03-19 20:57 ` Junio C Hamano
2014-03-20 13:54 ` Karsten Blees
2014-03-26 8:35 ` [PATCH] Enable index-pack threading in msysgit Johannes Sixt
3 siblings, 0 replies; 19+ messages in thread
From: Junio C Hamano @ 2014-03-19 20:57 UTC (permalink / raw)
To: szager; +Cc: git
szager@chromium.org writes:
> This adds a Windows implementation of pread. Note that it is NOT
> safe to intersperse calls to read() and pread() on a file
> descriptor. According to the ReadFile spec, using the 'overlapped'
> argument should not affect the implicit position pointer of the
> descriptor. Experiments have shown that this is, in fact, a lie.
>
> To accomodate that fact, this change also incorporates:
>
> http://article.gmane.org/gmane.comp.version-control.git/196042
>
> ... which gives each index-pack thread its own file descriptor.
> ---
Sign-off?
The new "per-thread file descriptors to the same thing" in a generic
codepath is a bit of eyesore. For index-pack, keeping as many file
descritors open to the current pack as the worker threads are should
not be too bad, but could we have some comment next to the field
definition please (e.g. "Windows emulation of pread() needs separate
fd per thread; see $URL for details" or something)?
> builtin/index-pack.c | 21 ++++++++++++++++-----
> compat/mingw.c | 31 ++++++++++++++++++++++++++++++-
> compat/mingw.h | 3 +++
> config.mak.uname | 1 -
> 4 files changed, 49 insertions(+), 7 deletions(-)
>
> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
> index 2f37a38..c02dd4c 100644
> --- a/builtin/index-pack.c
> +++ b/builtin/index-pack.c
> @@ -51,6 +51,7 @@ struct thread_local {
> #endif
> struct base_data *base_cache;
> size_t base_cache_used;
> + int pack_fd;
> };
>
> /*
> @@ -91,7 +92,8 @@ static off_t consumed_bytes;
> static unsigned deepest_delta;
> static git_SHA_CTX input_ctx;
> static uint32_t input_crc32;
> -static int input_fd, output_fd, pack_fd;
> +static const char *curr_pack;
> +static int input_fd, output_fd;
>
> #ifndef NO_PTHREADS
>
> @@ -134,6 +136,7 @@ static inline void unlock_mutex(pthread_mutex_t *mutex)
> */
> static void init_thread(void)
> {
> + int i;
> init_recursive_mutex(&read_mutex);
> pthread_mutex_init(&counter_mutex, NULL);
> pthread_mutex_init(&work_mutex, NULL);
> @@ -141,11 +144,17 @@ static void init_thread(void)
> pthread_mutex_init(&deepest_delta_mutex, NULL);
> pthread_key_create(&key, NULL);
> thread_data = xcalloc(nr_threads, sizeof(*thread_data));
> + for (i = 0; i < nr_threads; i++) {
> + thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
> + if (thread_data[i].pack_fd == -1)
> + die_errno("unable to open %s", curr_pack);
> + }
> threads_active = 1;
> }
>
> static void cleanup_thread(void)
> {
> + int i;
> if (!threads_active)
> return;
> threads_active = 0;
> @@ -155,6 +164,8 @@ static void cleanup_thread(void)
> if (show_stat)
> pthread_mutex_destroy(&deepest_delta_mutex);
> pthread_key_delete(key);
> + for (i = 0; i < nr_threads; i++)
> + close(thread_data[i].pack_fd);
> free(thread_data);
> }
>
> @@ -288,13 +299,13 @@ static const char *open_pack_file(const char *pack_name)
> output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
> if (output_fd < 0)
> die_errno(_("unable to create '%s'"), pack_name);
> - pack_fd = output_fd;
> + nothread_data.pack_fd = output_fd;
> } else {
> input_fd = open(pack_name, O_RDONLY);
> if (input_fd < 0)
> die_errno(_("cannot open packfile '%s'"), pack_name);
> output_fd = -1;
> - pack_fd = input_fd;
> + nothread_data.pack_fd = input_fd;
> }
> git_SHA1_Init(&input_ctx);
> return pack_name;
> @@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
>
> do {
> ssize_t n = (len < 64*1024) ? len : 64*1024;
> - n = pread(pack_fd, inbuf, n, from);
> + n = pread(get_thread_data()->pack_fd, inbuf, n, from);
> if (n < 0)
> die_errno(_("cannot pread pack file"));
> if (!n)
> @@ -1490,7 +1501,7 @@ static void show_pack_info(int stat_only)
> int cmd_index_pack(int argc, const char **argv, const char *prefix)
> {
> int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
> - const char *curr_pack, *curr_index;
> + const char *curr_index;
> const char *index_name = NULL, *pack_name = NULL;
> const char *keep_name = NULL, *keep_msg = NULL;
> char *index_name_buf = NULL, *keep_name_buf = NULL;
> diff --git a/compat/mingw.c b/compat/mingw.c
> index 383cafe..6cc85d6 100644
> --- a/compat/mingw.c
> +++ b/compat/mingw.c
> @@ -329,7 +329,36 @@ int mingw_mkdir(const char *path, int mode)
> return ret;
> }
>
> -int mingw_open (const char *filename, int oflags, ...)
> +
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset)
> +{
> + HANDLE hand = (HANDLE)_get_osfhandle(fd);
> + if (hand == INVALID_HANDLE_VALUE) {
> + errno = EBADF;
> + return -1;
> + }
> +
> + LARGE_INTEGER offset_value;
> + offset_value.QuadPart = offset;
> +
> + DWORD bytes_read = 0;
> + OVERLAPPED overlapped = {0};
> + overlapped.Offset = offset_value.LowPart;
> + overlapped.OffsetHigh = offset_value.HighPart;
> + BOOL result = ReadFile(hand, buf, count, &bytes_read, &overlapped);
> +
> + ssize_t ret = bytes_read;
> +
> + if (!result && GetLastError() != ERROR_HANDLE_EOF)
> + {
> + errno = err_win_to_posix(GetLastError());
> + ret = -1;
> + }
> +
> + return ret;
> +}
> +
> +int mingw_open(const char *filename, int oflags, ...)
> {
> va_list args;
> unsigned mode;
> diff --git a/compat/mingw.h b/compat/mingw.h
> index 08b83fe..377ba50 100644
> --- a/compat/mingw.h
> +++ b/compat/mingw.h
> @@ -174,6 +174,9 @@ int mingw_unlink(const char *pathname);
> int mingw_rmdir(const char *path);
> #define rmdir mingw_rmdir
>
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset);
> +#define pread mingw_pread
> +
> int mingw_open (const char *filename, int oflags, ...);
> #define open mingw_open
>
> diff --git a/config.mak.uname b/config.mak.uname
> index e8acc39..b405524 100644
> --- a/config.mak.uname
> +++ b/config.mak.uname
> @@ -474,7 +474,6 @@ ifeq ($(uname_S),NONSTOP_KERNEL)
> endif
> ifneq (,$(findstring MINGW,$(uname_S)))
> pathsep = ;
> - NO_PREAD = YesPlease
> NEEDS_CRYPTO_WITH_SSL = YesPlease
> NO_LIBGEN_H = YesPlease
> NO_POLL = YesPlease
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 0:46 [PATCH] Enable index-pack threading in msysgit szager
2014-03-19 7:30 ` Duy Nguyen
2014-03-19 20:57 ` Junio C Hamano
@ 2014-03-20 13:54 ` Karsten Blees
2014-03-20 16:08 ` Stefan Zager
2014-03-26 8:35 ` [PATCH] Enable index-pack threading in msysgit Johannes Sixt
3 siblings, 1 reply; 19+ messages in thread
From: Karsten Blees @ 2014-03-20 13:54 UTC (permalink / raw)
To: szager, git
Am 19.03.2014 01:46, schrieb szager@chromium.org:
> This adds a Windows implementation of pread. Note that it is NOT
> safe to intersperse calls to read() and pread() on a file
> descriptor.
This is a bad idea. You're basically fixing the multi-threaded issue twice, while at the same time breaking single-threaded read/pread interop on the mingw and msvc platform. Users of pread already have to take care that its not thread-safe on some platforms, now you're adding another breakage that has to be considered in future development.
The mingw_pread implementation in [1] is both thread-safe and allows mixing read/pread in single-threaded scenarios, why not use this instead?
[1] http://article.gmane.org/gmane.comp.version-control.git/242120
>
> http://article.gmane.org/gmane.comp.version-control.git/196042
>
Duy's patch alone enables multi-threaded index-pack on all platforms (including cygwin), so IMO this should be a separate patch.
> + if (hand == INVALID_HANDLE_VALUE) {
> + errno = EBADF;
> + return -1;
> + }
This check is redundant, ReadFile already ckecks for invalid handles and err_win_to_posix converts to EBADF.
> +
> + LARGE_INTEGER offset_value;
> + offset_value.QuadPart = offset;
> +
> + DWORD bytes_read = 0;
> + OVERLAPPED overlapped = {0};
> + overlapped.Offset = offset_value.LowPart;
> + overlapped.OffsetHigh = offset_value.HighPart;
> + BOOL result = ReadFile(hand, buf, count, &bytes_read, &overlapped);
> +
> + ssize_t ret = bytes_read;
> +
> + if (!result && GetLastError() != ERROR_HANDLE_EOF)
According to MSDN docs, ReadFile never fails with ERROR_HANDLE_EOF, or is this another case where the documentation is wrong?
"When a synchronous read operation reaches the end of a file, ReadFile returns TRUE and sets *lpNumberOfBytesRead to zero."
Karsten
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 13:54 ` Karsten Blees
@ 2014-03-20 16:08 ` Stefan Zager
2014-03-20 21:35 ` Karsten Blees
2014-03-21 1:51 ` Duy Nguyen
0 siblings, 2 replies; 19+ messages in thread
From: Stefan Zager @ 2014-03-20 16:08 UTC (permalink / raw)
To: Karsten Blees, Nguyễn Thái Ngọc
Cc: Stefan Zager, Git Mailing List
On Thu, Mar 20, 2014 at 6:54 AM, Karsten Blees <karsten.blees@gmail.com> wrote:
> Am 19.03.2014 01:46, schrieb szager@chromium.org:
>> This adds a Windows implementation of pread. Note that it is NOT
>> safe to intersperse calls to read() and pread() on a file
>> descriptor.
>
> This is a bad idea. You're basically fixing the multi-threaded issue twice, while at the same time breaking single-threaded read/pread interop on the mingw and msvc platform. Users of pread already have to take care that its not thread-safe on some platforms, now you're adding another breakage that has to be considered in future development.
>
> The mingw_pread implementation in [1] is both thread-safe and allows mixing read/pread in single-threaded scenarios, why not use this instead?
>
> [1] http://article.gmane.org/gmane.comp.version-control.git/242120
That's not thread-safe. There is, presumably, a point between the
first and second calls to lseek64 when the implicit position pointer
is incorrect. If another thread executes its first call to lseek64 at
that time, then the file descriptor may end up with an incorrect
position pointer after all threads have finished.
> Duy's patch alone enables multi-threaded index-pack on all platforms (including cygwin), so IMO this should be a separate patch.
Fair enough, and it's a good first step. I would love to see it
landed soon. On the Chrome project, we're currently distributing a
patched version of msysgit; I would very much like for us to stop
doing that, but the performance penalty is too significant right now.
Duy, would you like to re-post your patch without the new pread implementation?
Going forward, there is still a lot of performance that gets left on
the table when you rule out threaded file access. There are not so
many calls to read, mmap, and pread in the code; it should be possible
to rationalize them and make them thread-safe -- at least, thread-safe
for posix-compliant systems and msysgit, which covers the great
majority of git users, I would hope.
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 16:08 ` Stefan Zager
@ 2014-03-20 21:35 ` Karsten Blees
2014-03-20 21:56 ` Stefan Zager
2014-03-21 1:51 ` Duy Nguyen
1 sibling, 1 reply; 19+ messages in thread
From: Karsten Blees @ 2014-03-20 21:35 UTC (permalink / raw)
To: Stefan Zager, Nguyễn Thái Ngọc; +Cc: Git Mailing List
Am 20.03.2014 17:08, schrieb Stefan Zager:
> On Thu, Mar 20, 2014 at 6:54 AM, Karsten Blees <karsten.blees@gmail.com> wrote:
>> Am 19.03.2014 01:46, schrieb szager@chromium.org:
>>> This adds a Windows implementation of pread. Note that it is NOT
>>> safe to intersperse calls to read() and pread() on a file
>>> descriptor.
>>
>> This is a bad idea. You're basically fixing the multi-threaded issue twice, while at the same time breaking single-threaded read/pread interop on the mingw and msvc platform. Users of pread already have to take care that its not thread-safe on some platforms, now you're adding another breakage that has to be considered in future development.
>>
>> The mingw_pread implementation in [1] is both thread-safe and allows mixing read/pread in single-threaded scenarios, why not use this instead?
>>
>> [1] http://article.gmane.org/gmane.comp.version-control.git/242120
>
>
> That's not thread-safe. There is, presumably, a point between the
> first and second calls to lseek64 when the implicit position pointer
> is incorrect. If another thread executes its first call to lseek64 at
> that time, then the file descriptor may end up with an incorrect
> position pointer after all threads have finished.
>
Correct, a multi-threaded code section using pread has the effect of randomizing the file position (btw., this is also true for your pread implementation). This can be easily fixed by resetting the file position after pthread_join, if necessary. Currently there's just six callers of pthread_join.
> Going forward, there is still a lot of performance that gets left on
> the table when you rule out threaded file access. There are not so
> many calls to read, mmap, and pread in the code; it should be possible
> to rationalize them and make them thread-safe -- at least, thread-safe
> for posix-compliant systems and msysgit, which covers the great
> majority of git users, I would hope.
>
IMO a "mostly" XSI compliant pread (or even the git_pread() emulation) is still better than forbidding the use of read() entirely. Switching from read to pread everywhere requires that all callers have to keep track of the file position, which means a _lot_ of code changes (read/xread/strbuf_read is used in ~70 places throughout git). And how do you plan to deal with platforms that don't have a thread-safe pread (HP, Cygwin)?
Considering all that, Duy's solution of opening separate file descriptors per thread seems to be the best pattern for future multi-threaded work.
Karsten
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 21:35 ` Karsten Blees
@ 2014-03-20 21:56 ` Stefan Zager
2014-03-21 1:33 ` Duy Nguyen
2014-03-21 20:01 ` Karsten Blees
0 siblings, 2 replies; 19+ messages in thread
From: Stefan Zager @ 2014-03-20 21:56 UTC (permalink / raw)
To: Karsten Blees
Cc: Stefan Zager, Nguyễn Thái Ngọc, Git Mailing List
On Thu, Mar 20, 2014 at 2:35 PM, Karsten Blees <karsten.blees@gmail.com> wrote:
> Am 20.03.2014 17:08, schrieb Stefan Zager:
>
>> Going forward, there is still a lot of performance that gets left on
>> the table when you rule out threaded file access. There are not so
>> many calls to read, mmap, and pread in the code; it should be possible
>> to rationalize them and make them thread-safe -- at least, thread-safe
>> for posix-compliant systems and msysgit, which covers the great
>> majority of git users, I would hope.
>>
>
> IMO a "mostly" XSI compliant pread (or even the git_pread() emulation) is still better than forbidding the use of read() entirely. Switching from read to pread everywhere requires that all callers have to keep track of the file position, which means a _lot_ of code changes (read/xread/strbuf_read is used in ~70 places throughout git). And how do you plan to deal with platforms that don't have a thread-safe pread (HP, Cygwin)?
>
> Considering all that, Duy's solution of opening separate file descriptors per thread seems to be the best pattern for future multi-threaded work.
Does that mean you would endorse the (N threads) * (M pack files)
approach to threading checkout and status? That seems kind of
crazy-town to me. Not to mention that pack windows are not shared, so
this approach to multi-threading can have the side-effect of blowing
out memory consumption. We have already had to dial back settings for
pack.threads and core.deltaBaseCacheLimit, because threaded index-pack
was causing OOM errors on 32-bit platforms.
Cygwin (and MSVC) should be able to share a "mostly" compliant pread
implementation. I don't have any insight into NonstopKernel; does is
really not have a thread-safe pread implementation? If so, then I
suppose we have to #ifdef NO_PREAD, just as we do now.
I realize that these are deep changes. However, the performance of
msysgit on the chromium repositories is pretty awful, enough so to
motivate this work.
Stefan
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 21:56 ` Stefan Zager
@ 2014-03-21 1:33 ` Duy Nguyen
2014-03-21 20:01 ` Karsten Blees
1 sibling, 0 replies; 19+ messages in thread
From: Duy Nguyen @ 2014-03-21 1:33 UTC (permalink / raw)
To: Stefan Zager; +Cc: Karsten Blees, Git Mailing List
On Fri, Mar 21, 2014 at 4:56 AM, Stefan Zager <szager@chromium.org> wrote:
>> Considering all that, Duy's solution of opening separate file descriptors per thread seems to be the best pattern for future multi-threaded work.
>
> Does that mean you would endorse the (N threads) * (M pack files)
> approach to threading checkout and status? That seems kind of
> crazy-town to me. Not to mention that pack windows are not shared, so
> this approach to multi-threading can have the side-effect of blowing
> out memory consumption.
Maybe we could protect and share the delta cache. Pack windows are
mmap'd so we should not need to worry about their memory consumption.
> We have already had to dial back settings for
> pack.threads and core.deltaBaseCacheLimit, because threaded index-pack
> was causing OOM errors on 32-bit platforms.
Hm.. I don't think index-pack uses sha1_file.c heavily. Local pack
access is only needed for verifying identical objects (and that should
never happen often). Something is fishy with these OOM errors.
> Cygwin (and MSVC) should be able to share a "mostly" compliant pread
> implementation. I don't have any insight into NonstopKernel; does is
> really not have a thread-safe pread implementation? If so, then I
> suppose we have to #ifdef NO_PREAD, just as we do now.
>
> I realize that these are deep changes. However, the performance of
> msysgit on the chromium repositories is pretty awful, enough so to
> motivate this work.
--
Duy
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 21:56 ` Stefan Zager
2014-03-21 1:33 ` Duy Nguyen
@ 2014-03-21 20:01 ` Karsten Blees
1 sibling, 0 replies; 19+ messages in thread
From: Karsten Blees @ 2014-03-21 20:01 UTC (permalink / raw)
To: Stefan Zager; +Cc: Nguyễn Thái Ngọc, Git Mailing List
Am 20.03.2014 22:56, schrieb Stefan Zager:
> On Thu, Mar 20, 2014 at 2:35 PM, Karsten Blees <karsten.blees@gmail.com> wrote:
>> Am 20.03.2014 17:08, schrieb Stefan Zager:
>>
>>> Going forward, there is still a lot of performance that gets left on
>>> the table when you rule out threaded file access. There are not so
>>> many calls to read, mmap, and pread in the code; it should be possible
>>> to rationalize them and make them thread-safe -- at least, thread-safe
>>> for posix-compliant systems and msysgit, which covers the great
>>> majority of git users, I would hope.
>>>
>>
>> IMO a "mostly" XSI compliant pread (or even the git_pread() emulation) is still better than forbidding the use of read() entirely. Switching from read to pread everywhere requires that all callers have to keep track of the file position, which means a _lot_ of code changes (read/xread/strbuf_read is used in ~70 places throughout git). And how do you plan to deal with platforms that don't have a thread-safe pread (HP, Cygwin)?
>>
>> Considering all that, Duy's solution of opening separate file descriptors per thread seems to be the best pattern for future multi-threaded work.
>
> Does that mean you would endorse the (N threads) * (M pack files)
> approach to threading checkout and status? That seems kind of
> crazy-town to me. Not to mention that pack windows are not shared, so
> this approach to multi-threading can have the side-effect of blowing
> out memory consumption. We have already had to dial back settings for
> pack.threads and core.deltaBaseCacheLimit, because threaded index-pack
> was causing OOM errors on 32-bit platforms.
>
Opening more file descriptors doesn't significantly increase the memory footprint, so it shouldn't matter whether the threads read data via shared or private descriptors.
git-status with core.preloadindex is already multithreaded (at least the first part), and AFAIK doesn't read pack files at all.
I'm still not convinced that multi-threaded git-checkout is a good idea. According to my tests this is actually slower than sequential checkout. You'd have to be very careful to only multi-thread the parts that don't do any IO, such as unpacking / undeltifying.
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-20 16:08 ` Stefan Zager
2014-03-20 21:35 ` Karsten Blees
@ 2014-03-21 1:51 ` Duy Nguyen
2014-03-21 5:21 ` Duy Nguyen
2014-03-25 13:41 ` [PATCH] index-pack: work around thread-unsafe pread() Nguyễn Thái Ngọc Duy
1 sibling, 2 replies; 19+ messages in thread
From: Duy Nguyen @ 2014-03-21 1:51 UTC (permalink / raw)
To: Stefan Zager; +Cc: Karsten Blees, Git Mailing List
On Thu, Mar 20, 2014 at 11:08 PM, Stefan Zager <szager@chromium.org> wrote:
> Duy, would you like to re-post your patch without the new pread implementation?
I will but let me try out the sliding window idea first. My quick
tests on git.git show me we may only need 21k mmap instead of 177k
pread. That hints some potential performance improvement.
--
Duy
^ permalink raw reply [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-21 1:51 ` Duy Nguyen
@ 2014-03-21 5:21 ` Duy Nguyen
2014-03-21 5:35 ` Stefan Zager
2014-03-25 13:41 ` [PATCH] index-pack: work around thread-unsafe pread() Nguyễn Thái Ngọc Duy
1 sibling, 1 reply; 19+ messages in thread
From: Duy Nguyen @ 2014-03-21 5:21 UTC (permalink / raw)
To: Stefan Zager; +Cc: Karsten Blees, Git Mailing List
On Fri, Mar 21, 2014 at 08:51:18AM +0700, Duy Nguyen wrote:
> On Thu, Mar 20, 2014 at 11:08 PM, Stefan Zager <szager@chromium.org> wrote:
> > Duy, would you like to re-post your patch without the new pread implementation?
>
> I will but let me try out the sliding window idea first. My quick
> tests on git.git show me we may only need 21k mmap instead of 177k
> pread. That hints some potential performance improvement.
The patch at the bottom reuses (un)use_pack() instead of pread(). The
results on linux-2.6 do not look any different. I guess we can drop
the idea.
It makes me wonder, though, what's wrong a simple patch like this to
make pread in index-pack thread-safe? It does not look any different
either from the performance point of view, perhaps because
unpack_data() reads small deltas most of the time
-- 8< --
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index a6b1c17..b91f4f8 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -40,11 +40,6 @@ struct base_data {
int ofs_first, ofs_last;
};
-#if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD)
-/* pread() emulation is not thread-safe. Disable threading. */
-#define NO_PTHREADS
-#endif
-
struct thread_local {
#ifndef NO_PTHREADS
pthread_t thread;
@@ -175,6 +170,22 @@ static void cleanup_thread(void)
#endif
+#if defined(NO_THREAD_SAFE_PREAD)
+static inline ssize_t pread_safe(int fd, void *buf, size_t count, off_t off)
+{
+ int ret;
+ read_lock();
+ ret = pread(fd, buf, count, off);
+ read_unlock();
+ return ret;
+}
+#else
+static inline ssize_t pread_safe(int fd, void *buf, size_t count, off_t off)
+{
+ return pread(fd, buf, count, off);
+}
+#endif
+
static int mark_link(struct object *obj, int type, void *data)
{
if (!obj)
@@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
do {
ssize_t n = (len < 64*1024) ? len : 64*1024;
- n = pread(pack_fd, inbuf, n, from);
+ n = pread_safe(pack_fd, inbuf, n, from);
if (n < 0)
die_errno(_("cannot pread pack file"));
if (!n)
-- 8< --
And the sliding window patch for the list archive
-- 8< --
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index a6b1c17..6f5c6d9 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -91,7 +91,8 @@ static off_t consumed_bytes;
static unsigned deepest_delta;
static git_SHA_CTX input_ctx;
static uint32_t input_crc32;
-static int input_fd, output_fd, pack_fd;
+static int input_fd, output_fd;
+static struct packed_git *pack;
#ifndef NO_PTHREADS
@@ -224,8 +225,10 @@ static unsigned check_objects(void)
static void flush(void)
{
if (input_offset) {
- if (output_fd >= 0)
+ if (output_fd >= 0) {
write_or_die(output_fd, input_buffer, input_offset);
+ pack->pack_size += input_offset;
+ }
git_SHA1_Update(&input_ctx, input_buffer, input_offset);
memmove(input_buffer, input_buffer + input_offset, input_len);
input_offset = 0;
@@ -277,6 +280,10 @@ static void use(int bytes)
static const char *open_pack_file(const char *pack_name)
{
+ pack = xmalloc(sizeof(*pack) + 1);
+ memset(pack, 0, sizeof(*pack));
+ pack->pack_name[0] = '\0';
+
if (from_stdin) {
input_fd = 0;
if (!pack_name) {
@@ -288,13 +295,17 @@ static const char *open_pack_file(const char *pack_name)
output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
if (output_fd < 0)
die_errno(_("unable to create '%s'"), pack_name);
- pack_fd = output_fd;
+ pack->pack_fd = output_fd;
} else {
+ struct stat st;
input_fd = open(pack_name, O_RDONLY);
if (input_fd < 0)
die_errno(_("cannot open packfile '%s'"), pack_name);
+ if (lstat(pack_name, &st))
+ die_errno(_("cannot stat packfile '%s'"), pack_name);
output_fd = -1;
- pack_fd = input_fd;
+ pack->pack_fd = input_fd;
+ pack->pack_size = st.st_size;
}
git_SHA1_Init(&input_ctx);
return pack_name;
@@ -531,9 +542,15 @@ static void *unpack_data(struct object_entry *obj,
unsigned char *data, *inbuf;
git_zstream stream;
int status;
+ struct pack_window *w_cursor = NULL;
+
+ if (from + len > pack->pack_size)
+ die(Q_("premature end of pack file, %lu byte missing",
+ "premature end of pack file, %lu bytes missing",
+ from + len - pack->pack_size),
+ (unsigned long)(from + len - pack->pack_size));
data = xmalloc(consume ? 64*1024 : obj->size);
- inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
memset(&stream, 0, sizeof(stream));
git_inflate_init(&stream);
@@ -541,15 +558,12 @@ static void *unpack_data(struct object_entry *obj,
stream.avail_out = consume ? 64*1024 : obj->size;
do {
- ssize_t n = (len < 64*1024) ? len : 64*1024;
- n = pread(pack_fd, inbuf, n, from);
- if (n < 0)
- die_errno(_("cannot pread pack file"));
- if (!n)
- die(Q_("premature end of pack file, %lu byte missing",
- "premature end of pack file, %lu bytes missing",
- len),
- len);
+ ssize_t n;
+ unsigned long left;
+ read_lock();
+ inbuf = use_pack(pack, &w_cursor, from, &left);
+ read_unlock();
+ n = left > len ? len : left;
from += n;
len -= n;
stream.next_in = inbuf;
@@ -568,6 +582,9 @@ static void *unpack_data(struct object_entry *obj,
stream.avail_out = 64*1024;
} while (status == Z_OK && stream.avail_in);
}
+ read_lock();
+ unuse_pack(&w_cursor);
+ read_unlock();
} while (len && status == Z_OK && !stream.avail_in);
/* This has been inflated OK when first encountered, so... */
@@ -575,7 +592,6 @@ static void *unpack_data(struct object_entry *obj,
die(_("serious inflate inconsistency"));
git_inflate_end(&stream);
- free(inbuf);
if (consume) {
free(data);
data = NULL;
@@ -1657,6 +1673,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
free(objects);
free(index_name_buf);
free(keep_name_buf);
+ close_pack_windows(pack);
+ free(pack);
if (pack_name == NULL)
free((void *) curr_pack);
if (index_name == NULL)
diff --git a/sha1_file.c b/sha1_file.c
index 187f5a6..aa0b16d 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -977,7 +977,13 @@ unsigned char *use_pack(struct packed_git *p,
*/
if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
die("packfile %s cannot be accessed", p->pack_name);
- if (offset > (p->pack_size - 20))
+ /*
+ * index-pack uses this function even if the pack is not
+ * complete yet (i.e. trailing SHA-1 missing). Loosen the
+ * check a bit in this case (pack_empty name uses as the
+ * indicator).
+ */
+ if (offset > (p->pack_size - (*p->pack_name ? 20 : 0)))
die("offset beyond end of packfile (truncated pack?)");
if (!win || !in_window(win, offset)) {
-- 8< --
--
Duy
^ permalink raw reply related [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-21 5:21 ` Duy Nguyen
@ 2014-03-21 5:35 ` Stefan Zager
2014-03-21 18:55 ` Karsten Blees
0 siblings, 1 reply; 19+ messages in thread
From: Stefan Zager @ 2014-03-21 5:35 UTC (permalink / raw)
To: Duy Nguyen; +Cc: Stefan Zager, Karsten Blees, Git Mailing List
On Thu, Mar 20, 2014 at 10:21 PM, Duy Nguyen <pclouds@gmail.com> wrote:
> On Fri, Mar 21, 2014 at 08:51:18AM +0700, Duy Nguyen wrote:
>> On Thu, Mar 20, 2014 at 11:08 PM, Stefan Zager <szager@chromium.org> wrote:
>> > Duy, would you like to re-post your patch without the new pread implementation?
>>
>> I will but let me try out the sliding window idea first. My quick
>> tests on git.git show me we may only need 21k mmap instead of 177k
>> pread. That hints some potential performance improvement.
>
> The patch at the bottom reuses (un)use_pack() instead of pread(). The
> results on linux-2.6 do not look any different. I guess we can drop
> the idea.
>
> It makes me wonder, though, what's wrong a simple patch like this to
> make pread in index-pack thread-safe? It does not look any different
> either from the performance point of view, perhaps because
> unpack_data() reads small deltas most of the time
When you serialize disk access in this way, the effect on performance
is really dependent on the behavior of the OS, as well as the locality
of the read offsets. Assuming -- fairly, I think -- that the reads
will be pretty randomly distributed (i.e., no locality to speak of),
then your best bet is get as many read operations in flight as
possible, and let the disk scheduler optimize the seek time.
If I have a chance, I will try out this patch in msysgit on the
chromium repositories.
>
> -- 8< --
> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
> index a6b1c17..b91f4f8 100644
> --- a/builtin/index-pack.c
> +++ b/builtin/index-pack.c
> @@ -40,11 +40,6 @@ struct base_data {
> int ofs_first, ofs_last;
> };
>
> -#if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD)
> -/* pread() emulation is not thread-safe. Disable threading. */
> -#define NO_PTHREADS
> -#endif
> -
> struct thread_local {
> #ifndef NO_PTHREADS
> pthread_t thread;
> @@ -175,6 +170,22 @@ static void cleanup_thread(void)
> #endif
>
>
> +#if defined(NO_THREAD_SAFE_PREAD)
> +static inline ssize_t pread_safe(int fd, void *buf, size_t count, off_t off)
> +{
> + int ret;
> + read_lock();
> + ret = pread(fd, buf, count, off);
> + read_unlock();
> + return ret;
> +}
> +#else
> +static inline ssize_t pread_safe(int fd, void *buf, size_t count, off_t off)
> +{
> + return pread(fd, buf, count, off);
> +}
> +#endif
> +
> static int mark_link(struct object *obj, int type, void *data)
> {
> if (!obj)
> @@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
>
> do {
> ssize_t n = (len < 64*1024) ? len : 64*1024;
> - n = pread(pack_fd, inbuf, n, from);
> + n = pread_safe(pack_fd, inbuf, n, from);
> if (n < 0)
> die_errno(_("cannot pread pack file"));
> if (!n)
> -- 8< --
>
> And the sliding window patch for the list archive
>
> -- 8< --
> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
> index a6b1c17..6f5c6d9 100644
> --- a/builtin/index-pack.c
> +++ b/builtin/index-pack.c
> @@ -91,7 +91,8 @@ static off_t consumed_bytes;
> static unsigned deepest_delta;
> static git_SHA_CTX input_ctx;
> static uint32_t input_crc32;
> -static int input_fd, output_fd, pack_fd;
> +static int input_fd, output_fd;
> +static struct packed_git *pack;
>
> #ifndef NO_PTHREADS
>
> @@ -224,8 +225,10 @@ static unsigned check_objects(void)
> static void flush(void)
> {
> if (input_offset) {
> - if (output_fd >= 0)
> + if (output_fd >= 0) {
> write_or_die(output_fd, input_buffer, input_offset);
> + pack->pack_size += input_offset;
> + }
> git_SHA1_Update(&input_ctx, input_buffer, input_offset);
> memmove(input_buffer, input_buffer + input_offset, input_len);
> input_offset = 0;
> @@ -277,6 +280,10 @@ static void use(int bytes)
>
> static const char *open_pack_file(const char *pack_name)
> {
> + pack = xmalloc(sizeof(*pack) + 1);
> + memset(pack, 0, sizeof(*pack));
> + pack->pack_name[0] = '\0';
> +
> if (from_stdin) {
> input_fd = 0;
> if (!pack_name) {
> @@ -288,13 +295,17 @@ static const char *open_pack_file(const char *pack_name)
> output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
> if (output_fd < 0)
> die_errno(_("unable to create '%s'"), pack_name);
> - pack_fd = output_fd;
> + pack->pack_fd = output_fd;
> } else {
> + struct stat st;
> input_fd = open(pack_name, O_RDONLY);
> if (input_fd < 0)
> die_errno(_("cannot open packfile '%s'"), pack_name);
> + if (lstat(pack_name, &st))
> + die_errno(_("cannot stat packfile '%s'"), pack_name);
> output_fd = -1;
> - pack_fd = input_fd;
> + pack->pack_fd = input_fd;
> + pack->pack_size = st.st_size;
> }
> git_SHA1_Init(&input_ctx);
> return pack_name;
> @@ -531,9 +542,15 @@ static void *unpack_data(struct object_entry *obj,
> unsigned char *data, *inbuf;
> git_zstream stream;
> int status;
> + struct pack_window *w_cursor = NULL;
> +
> + if (from + len > pack->pack_size)
> + die(Q_("premature end of pack file, %lu byte missing",
> + "premature end of pack file, %lu bytes missing",
> + from + len - pack->pack_size),
> + (unsigned long)(from + len - pack->pack_size));
>
> data = xmalloc(consume ? 64*1024 : obj->size);
> - inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
>
> memset(&stream, 0, sizeof(stream));
> git_inflate_init(&stream);
> @@ -541,15 +558,12 @@ static void *unpack_data(struct object_entry *obj,
> stream.avail_out = consume ? 64*1024 : obj->size;
>
> do {
> - ssize_t n = (len < 64*1024) ? len : 64*1024;
> - n = pread(pack_fd, inbuf, n, from);
> - if (n < 0)
> - die_errno(_("cannot pread pack file"));
> - if (!n)
> - die(Q_("premature end of pack file, %lu byte missing",
> - "premature end of pack file, %lu bytes missing",
> - len),
> - len);
> + ssize_t n;
> + unsigned long left;
> + read_lock();
> + inbuf = use_pack(pack, &w_cursor, from, &left);
> + read_unlock();
> + n = left > len ? len : left;
> from += n;
> len -= n;
> stream.next_in = inbuf;
> @@ -568,6 +582,9 @@ static void *unpack_data(struct object_entry *obj,
> stream.avail_out = 64*1024;
> } while (status == Z_OK && stream.avail_in);
> }
> + read_lock();
> + unuse_pack(&w_cursor);
> + read_unlock();
> } while (len && status == Z_OK && !stream.avail_in);
>
> /* This has been inflated OK when first encountered, so... */
> @@ -575,7 +592,6 @@ static void *unpack_data(struct object_entry *obj,
> die(_("serious inflate inconsistency"));
>
> git_inflate_end(&stream);
> - free(inbuf);
> if (consume) {
> free(data);
> data = NULL;
> @@ -1657,6 +1673,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
> free(objects);
> free(index_name_buf);
> free(keep_name_buf);
> + close_pack_windows(pack);
> + free(pack);
> if (pack_name == NULL)
> free((void *) curr_pack);
> if (index_name == NULL)
> diff --git a/sha1_file.c b/sha1_file.c
> index 187f5a6..aa0b16d 100644
> --- a/sha1_file.c
> +++ b/sha1_file.c
> @@ -977,7 +977,13 @@ unsigned char *use_pack(struct packed_git *p,
> */
> if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
> die("packfile %s cannot be accessed", p->pack_name);
> - if (offset > (p->pack_size - 20))
> + /*
> + * index-pack uses this function even if the pack is not
> + * complete yet (i.e. trailing SHA-1 missing). Loosen the
> + * check a bit in this case (pack_empty name uses as the
> + * indicator).
> + */
> + if (offset > (p->pack_size - (*p->pack_name ? 20 : 0)))
> die("offset beyond end of packfile (truncated pack?)");
>
> if (!win || !in_window(win, offset)) {
> -- 8< --
>
> --
> Duy
^ permalink raw reply [flat|nested] 19+ messages in thread* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-21 5:35 ` Stefan Zager
@ 2014-03-21 18:55 ` Karsten Blees
0 siblings, 0 replies; 19+ messages in thread
From: Karsten Blees @ 2014-03-21 18:55 UTC (permalink / raw)
To: Stefan Zager, Duy Nguyen; +Cc: Git Mailing List
Am 21.03.2014 06:35, schrieb Stefan Zager:
> On Thu, Mar 20, 2014 at 10:21 PM, Duy Nguyen <pclouds@gmail.com> wrote:
>> On Fri, Mar 21, 2014 at 08:51:18AM +0700, Duy Nguyen wrote:
>>> On Thu, Mar 20, 2014 at 11:08 PM, Stefan Zager <szager@chromium.org> wrote:
>>>> Duy, would you like to re-post your patch without the new pread implementation?
>>>
>>> I will but let me try out the sliding window idea first. My quick
>>> tests on git.git show me we may only need 21k mmap instead of 177k
>>> pread. That hints some potential performance improvement.
>>
>> The patch at the bottom reuses (un)use_pack() instead of pread(). The
>> results on linux-2.6 do not look any different. I guess we can drop
>> the idea.
>>
>> It makes me wonder, though, what's wrong a simple patch like this to
>> make pread in index-pack thread-safe? It does not look any different
>> either from the performance point of view, perhaps because
>> unpack_data() reads small deltas most of the time
>
> When you serialize disk access in this way, the effect on performance
> is really dependent on the behavior of the OS, as well as the locality
> of the read offsets. Assuming -- fairly, I think -- that the reads
> will be pretty randomly distributed (i.e., no locality to speak of),
> then your best bet is get as many read operations in flight as
> possible, and let the disk scheduler optimize the seek time.
>
The read() implementation in MSVCRT.DLL is synchronized anyway, and I strongly suspect that this is also true for ReadFile() (at least for synchronous file handles, i.e. opened without FILE_FLAG_OVERLAPPED). So I guess separate file descriptors would help with parallel IO as well.
^ permalink raw reply [flat|nested] 19+ messages in thread
* [PATCH] index-pack: work around thread-unsafe pread()
2014-03-21 1:51 ` Duy Nguyen
2014-03-21 5:21 ` Duy Nguyen
@ 2014-03-25 13:41 ` Nguyễn Thái Ngọc Duy
1 sibling, 0 replies; 19+ messages in thread
From: Nguyễn Thái Ngọc Duy @ 2014-03-25 13:41 UTC (permalink / raw)
To: git; +Cc: Stefan Zager, Karsten Blees, Nguyễn Thái Ngọc Duy
pread() implementation for Cygwin and MSYS is not thread safe, which
led to multithreading being disabled in c0f8654 (index-pack: Disable
threading on cygwin - 2012-06-26). Work around it by opening one file
handle per thread, so parallel pread() (on different file handle)
can't step on each other. Also remove NO_THREAD_SAFE_PREAD that was
introduced in c0f8654 because it's no longer used anywhere.
This workaround is unconditional, even for platforms with thread-safe
pread() because the overhead is small (a couple file handles more) and
not worth fragmenting the code.
Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com>
---
On Fri, Mar 21, 2014 at 8:51 AM, Duy Nguyen <pclouds@gmail.com> wrote:
> On Thu, Mar 20, 2014 at 11:08 PM, Stefan Zager <szager@chromium.org> wrote:
>> Duy, would you like to re-post your patch without the new pread implementation?
>
> I will but let me try out the sliding window idea first. My quick
> tests on git.git show me we may only need 21k mmap instead of 177k
> pread. That hints some potential performance improvement.
Here it is. But I still think it's worth measuring the simpler patch
that protects pread() from the caller. I suspect we won't see any
difference.
Makefile | 7 -------
builtin/index-pack.c | 27 +++++++++++++++++----------
config.mak.uname | 1 -
3 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/Makefile b/Makefile
index 3646391..0089fad 100644
--- a/Makefile
+++ b/Makefile
@@ -183,9 +183,6 @@ all::
# Define NO_STRUCT_ITIMERVAL if you don't have struct itimerval
# This also implies NO_SETITIMER
#
-# Define NO_THREAD_SAFE_PREAD if your pread() implementation is not
-# thread-safe. (e.g. compat/pread.c or cygwin)
-#
# Define NO_FAST_WORKING_DIRECTORY if accessing objects in pack files is
# generally faster on your platform than accessing the working directory.
#
@@ -1336,10 +1333,6 @@ endif
ifdef NO_PREAD
COMPAT_CFLAGS += -DNO_PREAD
COMPAT_OBJS += compat/pread.o
- NO_THREAD_SAFE_PREAD = YesPlease
-endif
-ifdef NO_THREAD_SAFE_PREAD
- BASIC_CFLAGS += -DNO_THREAD_SAFE_PREAD
endif
ifdef NO_FAST_WORKING_DIRECTORY
BASIC_CFLAGS += -DNO_FAST_WORKING_DIRECTORY
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index a6b1c17..676d39d 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -40,17 +40,13 @@ struct base_data {
int ofs_first, ofs_last;
};
-#if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD)
-/* pread() emulation is not thread-safe. Disable threading. */
-#define NO_PTHREADS
-#endif
-
struct thread_local {
#ifndef NO_PTHREADS
pthread_t thread;
#endif
struct base_data *base_cache;
size_t base_cache_used;
+ int pack_fd;
};
/*
@@ -91,7 +87,8 @@ static off_t consumed_bytes;
static unsigned deepest_delta;
static git_SHA_CTX input_ctx;
static uint32_t input_crc32;
-static int input_fd, output_fd, pack_fd;
+static int input_fd, output_fd;
+static const char *curr_pack;
#ifndef NO_PTHREADS
@@ -134,6 +131,7 @@ static inline void unlock_mutex(pthread_mutex_t *mutex)
*/
static void init_thread(void)
{
+ int i;
init_recursive_mutex(&read_mutex);
pthread_mutex_init(&counter_mutex, NULL);
pthread_mutex_init(&work_mutex, NULL);
@@ -141,11 +139,18 @@ static void init_thread(void)
pthread_mutex_init(&deepest_delta_mutex, NULL);
pthread_key_create(&key, NULL);
thread_data = xcalloc(nr_threads, sizeof(*thread_data));
+ for (i = 0; i < nr_threads; i++) {
+ thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
+ if (thread_data[i].pack_fd == -1)
+ die_errno(_("unable to open %s"), curr_pack);
+ }
+
threads_active = 1;
}
static void cleanup_thread(void)
{
+ int i;
if (!threads_active)
return;
threads_active = 0;
@@ -154,6 +159,8 @@ static void cleanup_thread(void)
pthread_mutex_destroy(&work_mutex);
if (show_stat)
pthread_mutex_destroy(&deepest_delta_mutex);
+ for (i = 0; i < nr_threads; i++)
+ close(thread_data[i].pack_fd);
pthread_key_delete(key);
free(thread_data);
}
@@ -288,13 +295,13 @@ static const char *open_pack_file(const char *pack_name)
output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
if (output_fd < 0)
die_errno(_("unable to create '%s'"), pack_name);
- pack_fd = output_fd;
+ nothread_data.pack_fd = output_fd;
} else {
input_fd = open(pack_name, O_RDONLY);
if (input_fd < 0)
die_errno(_("cannot open packfile '%s'"), pack_name);
output_fd = -1;
- pack_fd = input_fd;
+ nothread_data.pack_fd = input_fd;
}
git_SHA1_Init(&input_ctx);
return pack_name;
@@ -542,7 +549,7 @@ static void *unpack_data(struct object_entry *obj,
do {
ssize_t n = (len < 64*1024) ? len : 64*1024;
- n = pread(pack_fd, inbuf, n, from);
+ n = pread(get_thread_data()->pack_fd, inbuf, n, from);
if (n < 0)
die_errno(_("cannot pread pack file"));
if (!n)
@@ -1490,7 +1497,7 @@ static void show_pack_info(int stat_only)
int cmd_index_pack(int argc, const char **argv, const char *prefix)
{
int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
- const char *curr_pack, *curr_index;
+ const char *curr_index;
const char *index_name = NULL, *pack_name = NULL;
const char *keep_name = NULL, *keep_msg = NULL;
char *index_name_buf = NULL, *keep_name_buf = NULL;
diff --git a/config.mak.uname b/config.mak.uname
index 6069a44..c9febe1 100644
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -157,7 +157,6 @@ ifeq ($(uname_O),Cygwin)
NO_SYMLINK_HEAD = YesPlease
NO_IPV6 = YesPlease
OLD_ICONV = UnfortunatelyYes
- NO_THREAD_SAFE_PREAD = YesPlease
# There are conflicting reports about this.
# On some boxes NO_MMAP is needed, and not so elsewhere.
# Try commenting this out if you suspect MMAP is more efficient
--
1.9.1.345.ga1a145c
^ permalink raw reply related [flat|nested] 19+ messages in thread
* Re: [PATCH] Enable index-pack threading in msysgit.
2014-03-19 0:46 [PATCH] Enable index-pack threading in msysgit szager
` (2 preceding siblings ...)
2014-03-20 13:54 ` Karsten Blees
@ 2014-03-26 8:35 ` Johannes Sixt
3 siblings, 0 replies; 19+ messages in thread
From: Johannes Sixt @ 2014-03-26 8:35 UTC (permalink / raw)
To: szager; +Cc: git, Nguyễn Thái Ngọc Duy
Am 3/19/2014 1:46, schrieb szager@chromium.org:
> This adds a Windows implementation of pread. Note that it is NOT
> safe to intersperse calls to read() and pread() on a file
> descriptor. According to the ReadFile spec, using the 'overlapped'
> argument should not affect the implicit position pointer of the
> descriptor. Experiments have shown that this is, in fact, a lie.
>
> To accomodate that fact, this change also incorporates:
>
> http://article.gmane.org/gmane.comp.version-control.git/196042
>
> .... which gives each index-pack thread its own file descriptor.
> ---
> builtin/index-pack.c | 21 ++++++++++++++++-----
> compat/mingw.c | 31 ++++++++++++++++++++++++++++++-
> compat/mingw.h | 3 +++
> config.mak.uname | 1 -
> 4 files changed, 49 insertions(+), 7 deletions(-)
t5302 does not pass with this patch (sz/mingw-index-pack-threaded).
It fails like this:
+ eval 'git index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
git prune-packed &&
git count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
cmp "test-1-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
++ git index-pack --index-version=1 --stdin
pack 1c54d893dd9bf6645ecee2886ea72f2c2030bea1
++ git prune-packed
error: packfile .git/objects/pack/pack-1c54d893dd9bf6645ecee2886ea72f2c2030bea1.pack does not match index
warning: packfile .git/objects/pack/pack-1c54d893dd9bf6645ecee2886ea72f2c2030bea1.pack cannot be accessed
[... these 2 messages repeat ~250 times ...]
++ git count-objects
++ read nr rest
++ test 303 -eq 1
I haven't tested Duy's latest patch (index-pack: work around
thread-unsafe pread() yesterday), yet.
-- Hannes
>
> diff --git a/builtin/index-pack.c b/builtin/index-pack.c
> index 2f37a38..c02dd4c 100644
> --- a/builtin/index-pack.c
> +++ b/builtin/index-pack.c
> @@ -51,6 +51,7 @@ struct thread_local {
> #endif
> struct base_data *base_cache;
> size_t base_cache_used;
> + int pack_fd;
> };
>
> /*
> @@ -91,7 +92,8 @@ static off_t consumed_bytes;
> static unsigned deepest_delta;
> static git_SHA_CTX input_ctx;
> static uint32_t input_crc32;
> -static int input_fd, output_fd, pack_fd;
> +static const char *curr_pack;
> +static int input_fd, output_fd;
>
> #ifndef NO_PTHREADS
>
> @@ -134,6 +136,7 @@ static inline void unlock_mutex(pthread_mutex_t *mutex)
> */
> static void init_thread(void)
> {
> + int i;
> init_recursive_mutex(&read_mutex);
> pthread_mutex_init(&counter_mutex, NULL);
> pthread_mutex_init(&work_mutex, NULL);
> @@ -141,11 +144,17 @@ static void init_thread(void)
> pthread_mutex_init(&deepest_delta_mutex, NULL);
> pthread_key_create(&key, NULL);
> thread_data = xcalloc(nr_threads, sizeof(*thread_data));
> + for (i = 0; i < nr_threads; i++) {
> + thread_data[i].pack_fd = open(curr_pack, O_RDONLY);
> + if (thread_data[i].pack_fd == -1)
> + die_errno("unable to open %s", curr_pack);
> + }
> threads_active = 1;
> }
>
> static void cleanup_thread(void)
> {
> + int i;
> if (!threads_active)
> return;
> threads_active = 0;
> @@ -155,6 +164,8 @@ static void cleanup_thread(void)
> if (show_stat)
> pthread_mutex_destroy(&deepest_delta_mutex);
> pthread_key_delete(key);
> + for (i = 0; i < nr_threads; i++)
> + close(thread_data[i].pack_fd);
> free(thread_data);
> }
>
> @@ -288,13 +299,13 @@ static const char *open_pack_file(const char *pack_name)
> output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
> if (output_fd < 0)
> die_errno(_("unable to create '%s'"), pack_name);
> - pack_fd = output_fd;
> + nothread_data.pack_fd = output_fd;
> } else {
> input_fd = open(pack_name, O_RDONLY);
> if (input_fd < 0)
> die_errno(_("cannot open packfile '%s'"), pack_name);
> output_fd = -1;
> - pack_fd = input_fd;
> + nothread_data.pack_fd = input_fd;
> }
> git_SHA1_Init(&input_ctx);
> return pack_name;
> @@ -542,7 +553,7 @@ static void *unpack_data(struct object_entry *obj,
>
> do {
> ssize_t n = (len < 64*1024) ? len : 64*1024;
> - n = pread(pack_fd, inbuf, n, from);
> + n = pread(get_thread_data()->pack_fd, inbuf, n, from);
> if (n < 0)
> die_errno(_("cannot pread pack file"));
> if (!n)
> @@ -1490,7 +1501,7 @@ static void show_pack_info(int stat_only)
> int cmd_index_pack(int argc, const char **argv, const char *prefix)
> {
> int i, fix_thin_pack = 0, verify = 0, stat_only = 0;
> - const char *curr_pack, *curr_index;
> + const char *curr_index;
> const char *index_name = NULL, *pack_name = NULL;
> const char *keep_name = NULL, *keep_msg = NULL;
> char *index_name_buf = NULL, *keep_name_buf = NULL;
> diff --git a/compat/mingw.c b/compat/mingw.c
> index 383cafe..6cc85d6 100644
> --- a/compat/mingw.c
> +++ b/compat/mingw.c
> @@ -329,7 +329,36 @@ int mingw_mkdir(const char *path, int mode)
> return ret;
> }
>
> -int mingw_open (const char *filename, int oflags, ...)
> +
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset)
> +{
> + HANDLE hand = (HANDLE)_get_osfhandle(fd);
> + if (hand == INVALID_HANDLE_VALUE) {
> + errno = EBADF;
> + return -1;
> + }
> +
> + LARGE_INTEGER offset_value;
> + offset_value.QuadPart = offset;
> +
> + DWORD bytes_read = 0;
> + OVERLAPPED overlapped = {0};
> + overlapped.Offset = offset_value.LowPart;
> + overlapped.OffsetHigh = offset_value.HighPart;
> + BOOL result = ReadFile(hand, buf, count, &bytes_read, &overlapped);
> +
> + ssize_t ret = bytes_read;
> +
> + if (!result && GetLastError() != ERROR_HANDLE_EOF)
> + {
> + errno = err_win_to_posix(GetLastError());
> + ret = -1;
> + }
> +
> + return ret;
> +}
> +
> +int mingw_open(const char *filename, int oflags, ...)
> {
> va_list args;
> unsigned mode;
> diff --git a/compat/mingw.h b/compat/mingw.h
> index 08b83fe..377ba50 100644
> --- a/compat/mingw.h
> +++ b/compat/mingw.h
> @@ -174,6 +174,9 @@ int mingw_unlink(const char *pathname);
> int mingw_rmdir(const char *path);
> #define rmdir mingw_rmdir
>
> +ssize_t mingw_pread(int fd, void *buf, size_t count, off64_t offset);
> +#define pread mingw_pread
> +
> int mingw_open (const char *filename, int oflags, ...);
> #define open mingw_open
>
> diff --git a/config.mak.uname b/config.mak.uname
> index e8acc39..b405524 100644
> --- a/config.mak.uname
> +++ b/config.mak.uname
> @@ -474,7 +474,6 @@ ifeq ($(uname_S),NONSTOP_KERNEL)
> endif
> ifneq (,$(findstring MINGW,$(uname_S)))
> pathsep = ;
> - NO_PREAD = YesPlease
> NEEDS_CRYPTO_WITH_SSL = YesPlease
> NO_LIBGEN_H = YesPlease
> NO_POLL = YesPlease
>
^ permalink raw reply [flat|nested] 19+ messages in thread