[PATCH][RFC] git on Mac OS and precomposed unicode

[PATCH][RFC] git on Mac OS and precomposed unicode

[Torsten Bögershausen]

Allow git on Mac OS to store file names in the index in precomposed unicode,
while the file system used decomposed unicode.

When a file called "LATIN CAPITAL LETTER A WITH DIAERESIS"
(in utf-8 encoded as 0xc3 0x84) is created,
the filesystem converts "precomposed unicode" into "decomposed unicode",
which means that readdir() will return 0x41 0xcc 0x88.
When true, git reverts the unicode decomposition of filenames.
This is useful when pulling/pushing from repositories containing utf-8
encoded filenames using precomposed utf-8 (like Linux).

This feature is automatically switched on when "git init" is run,
and the file system is doing UTF-8 decompostion.
(Which has been observed on HFS+, SMBFS and VFAT, but not on NFS)
It can be switched off by setting core.macosforcenfc=false

It is implemented by re-defining the readdir() functions.
File names are converted into precomposed UTF-8.

Signed-off-by: Torsten Bögershausen <tboegi@web.de>
---
 Documentation/config.txt     |    9 ++
 Makefile                     |    3 +
 builtin/init-db.c            |   22 +++++
 compat/darwin.c              |  200 ++++++++++++++++++++++++++++++++++++++++++
 compat/darwin.h              |   31 +++++++
 git-compat-util.h            |    8 ++
 git.c                        |    1 +
 t/t0050-filesystem.sh        |    1 +
 t/t3910-mac-os-precompose.sh |  104 ++++++++++++++++++++++
 9 files changed, 379 insertions(+), 0 deletions(-)
 create mode 100644 compat/darwin.c
 create mode 100644 compat/darwin.h
 create mode 100755 t/t3910-mac-os-precompose.sh

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 2959390..01b9465 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -175,6 +175,15 @@ The default is false, except linkgit:git-clone[1] or linkgit:git-init[1]
 will probe and set core.ignorecase true if appropriate when the repository
 is created.
 
+core.precomposedunicode::
+	This option is only used by Mac OS implementation of git.
+	When core.precomposedunicode=true,
+	git reverts the unicode decomposition of filenames done by Mac OS.
+	This is useful when pulling/pushing from repositories containing utf-8
+	encoded filenames using precomposed unicode (like Linux).
+	When false, file names are handled fully transparent by git.
+	If in doubt, set core.precomposedunicode=false.
+
 core.trustctime::
 	If false, the ctime differences between the index and the
 	working tree are ignored; useful when the inode change time
diff --git a/Makefile b/Makefile
index b21d2f1..596900e 100644
--- a/Makefile
+++ b/Makefile
@@ -519,6 +519,7 @@ LIB_H += compat/bswap.h
 LIB_H += compat/cygwin.h
 LIB_H += compat/mingw.h
 LIB_H += compat/obstack.h
+LIB_H += compat/darwin.h
 LIB_H += compat/win32/pthread.h
 LIB_H += compat/win32/syslog.h
 LIB_H += compat/win32/poll.h
@@ -884,6 +885,8 @@ ifeq ($(uname_S),Darwin)
 	endif
 	NO_MEMMEM = YesPlease
 	USE_ST_TIMESPEC = YesPlease
+	COMPAT_OBJS += compat/darwin.o
+	BASIC_CFLAGS += -DPRECOMPOSED_UNICODE
 endif
 ifeq ($(uname_S),SunOS)
 	NEEDS_SOCKET = YesPlease
diff --git a/builtin/init-db.c b/builtin/init-db.c
index 0dacb8b..88c9de1 100644
--- a/builtin/init-db.c
+++ b/builtin/init-db.c
@@ -290,6 +290,28 @@ static int create_default_files(const char *template_path)
 		strcpy(path + len, "CoNfIg");
 		if (!access(path, F_OK))
 			git_config_set("core.ignorecase", "true");
+#if defined (PRECOMPOSED_UNICODE)
+		{
+			const static char *auml_nfc = "\xc3\xa4";
+			const static char *auml_nfd = "\x61\xcc\x88";
+			int output_fd;
+			path[len] = 0;
+			strcpy(path + len, auml_nfc);
+			output_fd = open(path, O_CREAT|O_EXCL|O_RDWR, 0600);
+			if (output_fd >=0) {
+				close(output_fd);
+				path[len] = 0;
+				strcpy(path + len, auml_nfd);
+				if (0 == access(path, R_OK))
+					git_config_set("core.precomposedunicode", "true");
+				else
+					git_config_set("core.precomposedunicode", "false");
+				path[len] = 0;
+				strcpy(path + len, auml_nfc);
+				unlink(path);
+			}
+		}
+#endif
 	}
 
 	return reinit;
diff --git a/compat/darwin.c b/compat/darwin.c
new file mode 100644
index 0000000..15de7c2
--- /dev/null
+++ b/compat/darwin.c
@@ -0,0 +1,200 @@
+#define __DARWIN_C__
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdint.h>
+
+#include "../cache.h"
+#include "../utf8.h"
+
+#include "darwin.h"
+
+static int mac_os_precomposed_unicode;
+const static char *repo_encoding = "UTF-8";
+const static char *path_encoding = "UTF-8-MAC";
+
+
+/* Code borrowed from utf8.c */
+#if defined(OLD_ICONV) || (defined(__sun__) && !defined(_XPG6))
+	typedef const char * iconv_ibp;
+#else
+	typedef char * iconv_ibp;
+#endif
+static char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv)
+{
+	size_t outsz, outalloc;
+	char *out, *outpos;
+	iconv_ibp cp;
+
+	outsz = insz;
+	outalloc = outsz + 1; /* for terminating NUL */
+	out = xmalloc(outalloc);
+	outpos = out;
+	cp = (iconv_ibp)in;
+
+	while (1) {
+		size_t cnt = iconv(conv, &cp, &insz, &outpos, &outsz);
+
+		if (cnt == -1) {
+			size_t sofar;
+			if (errno != E2BIG) {
+				free(out);
+				iconv_close(conv);
+				return NULL;
+			}
+			/* insz has remaining number of bytes.
+			 * since we started outsz the same as insz,
+			 * it is likely that insz is not enough for
+			 * converting the rest.
+			 */
+			sofar = outpos - out;
+			outalloc = sofar + insz * 2 + 32;
+			out = xrealloc(out, outalloc);
+			outpos = out + sofar;
+			outsz = outalloc - sofar - 1;
+		}
+		else {
+			*outpos = '\0';
+			break;
+		}
+	}
+	return out;
+}
+
+static size_t
+has_utf8(const char *s, size_t maxlen, size_t *strlen_c)
+{
+	const uint8_t *utf8p = (const uint8_t*) s;
+	size_t strlen_chars = 0;
+	size_t ret = 0;
+
+	if ((!utf8p) || (!*utf8p))
+		return 0;
+
+	while((*utf8p) && maxlen) {
+		if (*utf8p & 0x80)
+			ret++;
+		strlen_chars++;
+		utf8p++;
+		maxlen--;
+	}
+	if (strlen_c)
+		*strlen_c = strlen_chars;
+
+	return ret;
+}
+
+static int
+precomposed_unicode_config(const char *var, const char *value, void *cb)
+{
+	if (!strcasecmp(var, "core.precomposedunicode")) {
+		mac_os_precomposed_unicode = git_config_bool(var, value);
+		return 0;
+	}
+	return 1;
+}
+
+void
+argv_precompose(int argc, const char **argv)
+{
+	int i = 0;
+	const char *oldarg;
+	char *newarg;
+	iconv_t ic_precompose;
+
+	if (!strcmp("commit", argv[0]))
+		return;
+
+	git_config(precomposed_unicode_config, NULL);
+	if (!mac_os_precomposed_unicode)
+		return;
+
+	ic_precompose = iconv_open(repo_encoding, path_encoding);
+	if (ic_precompose == (iconv_t) -1)
+		return;
+
+	while (i < argc) {
+		size_t namelen;
+		oldarg = argv[i];
+
+		if (has_utf8(oldarg, (size_t)-1, &namelen)) {
+			newarg = reencode_string_iconv(oldarg, namelen, ic_precompose);
+			if (newarg)
+				argv[i] = newarg;
+		}
+		i++;
+	}
+	iconv_close(ic_precompose);
+}
+
+
+DARWIN_DIR *
+darwin_opendir(const char *dirname)
+{
+	DARWIN_DIR *darwin_dir;
+	darwin_dir = malloc(sizeof(DARWIN_DIR));
+	if (!darwin_dir)
+		return NULL;
+
+	darwin_dir->dirp = opendir(dirname);
+	if (!darwin_dir->dirp) {
+		free(darwin_dir);
+		return NULL;
+	}
+	darwin_dir->ic_precompose = iconv_open(repo_encoding, path_encoding);
+	if (darwin_dir->ic_precompose == (iconv_t) -1) {
+		closedir(darwin_dir->dirp);
+		free(darwin_dir);
+		return NULL;
+	}
+
+	return darwin_dir;
+}
+
+struct dirent *
+darwin_readdir(DARWIN_DIR *darwin_dirp)
+{
+	struct dirent *res;
+	size_t namelen = 0;
+
+	res = readdir(darwin_dirp->dirp);
+	if (!res || !mac_os_precomposed_unicode || !has_utf8(res->d_name, (size_t)-1, &namelen))
+		return res;
+	else {
+		int olderrno = errno;
+		size_t outsz = sizeof(darwin_dirp->dirent_nfc.d_name) - 1; /* one for \0 */
+		char *outpos = darwin_dirp->dirent_nfc.d_name;
+		iconv_ibp cp;
+		size_t cnt;
+		size_t insz = namelen;
+		cp = (iconv_ibp)res->d_name;
+
+		/* Copy all data except the name */
+		memcpy(&darwin_dirp->dirent_nfc,
+					 res,
+					 sizeof(darwin_dirp->dirent_nfc)-sizeof(darwin_dirp->dirent_nfc.d_name));
+		errno = 0;
+
+		cnt = iconv(darwin_dirp->ic_precompose, &cp, &insz, &outpos, &outsz);
+		if (cnt < sizeof(darwin_dirp->dirent_nfc.d_name) -1) {
+			*outpos = 0;
+			errno = olderrno;
+			return &darwin_dirp->dirent_nfc;
+		}
+		errno = olderrno;
+		return res;
+	}
+}
+
+
+int
+darwin_closedir(DARWIN_DIR *darwin_dirp)
+{
+	int ret_value;
+	ret_value = closedir(darwin_dirp->dirp);
+	if (darwin_dirp->ic_precompose != (iconv_t)-1)
+		iconv_close(darwin_dirp->ic_precompose);
+	free(darwin_dirp);
+	return ret_value;
+}
diff --git a/compat/darwin.h b/compat/darwin.h
new file mode 100644
index 0000000..094f930
--- /dev/null
+++ b/compat/darwin.h
@@ -0,0 +1,31 @@
+#ifndef __DARWIN_H__
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <iconv.h>
+
+
+typedef struct {
+	iconv_t ic_precompose;
+	DIR *dirp;
+	struct dirent dirent_nfc;
+} DARWIN_DIR;
+
+char *str_precompose(const char *in, iconv_t ic_precompose);
+
+void argv_precompose(int argc, const char **argv);
+
+DARWIN_DIR *darwin_opendir(const char *dirname);
+struct dirent *darwin_readdir(DARWIN_DIR *dirp);
+int darwin_closedir(DARWIN_DIR *dirp);
+
+#ifndef __DARWIN_C__
+#define opendir(n) darwin_opendir(n)
+#define readdir(d) darwin_readdir(d)
+#define closedir(d) darwin_closedir(d)
+#define DIR DARWIN_DIR
+
+#endif  /* __DARWIN_C__ */
+
+#define  __DARWIN_H__
+#endif /* __DARWIN_H__ */
diff --git a/git-compat-util.h b/git-compat-util.h
index 230e198..859dfcf 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -90,6 +90,14 @@
 #include <windows.h>
 #endif
 
+#if defined (PRECOMPOSED_UNICODE)
+#include "compat/darwin.h"
+#else
+#define str_precompose(in,i_nfd2nfc) (NULL)
+#define argv_precompose(c,v)
+
+#endif
+
 #include <unistd.h>
 #include <stdio.h>
 #include <sys/stat.h>
diff --git a/git.c b/git.c
index 8e34903..6b2ffb7 100644
--- a/git.c
+++ b/git.c
@@ -298,6 +298,7 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
 		    startup_info->have_repository) /* get_git_dir() may set up repo, avoid that */
 			trace_repo_setup(prefix);
 	}
+	argv_precompose(argc, argv);
 	commit_pager_choice();
 
 	if (!help && p->option & NEED_WORK_TREE)
diff --git a/t/t0050-filesystem.sh b/t/t0050-filesystem.sh
index 1542cf6..befe39e 100755
--- a/t/t0050-filesystem.sh
+++ b/t/t0050-filesystem.sh
@@ -126,6 +126,7 @@ test_expect_success "setup unicode normalization tests" '
 
   test_create_repo unicode &&
   cd unicode &&
+  git config core.precomposedunicode false &&
   touch "$aumlcdiar" &&
   git add "$aumlcdiar" &&
   git commit -m initial &&
diff --git a/t/t3910-mac-os-precompose.sh b/t/t3910-mac-os-precompose.sh
new file mode 100755
index 0000000..d4763c5
--- /dev/null
+++ b/t/t3910-mac-os-precompose.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+#
+# Copyright (c) 2012 Torsten Bögershausen
+#
+
+test_description='utf-8 decomposed (nfd) converted to precomposed (nfc)'
+
+. ./test-lib.sh
+
+Adiarnfc=`printf '\303\204'`
+Odiarnfc=`printf '\303\226'`
+Adiarnfd=`printf 'A\314\210'`
+Odiarnfd=`printf 'O\314\210'`
+
+mkdir junk &&
+>junk/"$Adiarnfc" &&
+case "$(cd junk && echo *)" in
+	"$Adiarnfd")
+	test_nfd=1
+	;;
+	*)	;;
+esac
+rm -rf junk
+
+if test "$test_nfd"
+then
+	test_expect_success "detect if nfd needed" '
+		precomposedunicode=`git config --bool core.precomposedunicode` &&
+		test "$precomposedunicode" = true
+	'
+	test_expect_success "setup" '
+		>x &&
+		git add x &&
+		git commit -m "1st commit" &&
+		git rm x &&
+		git commit -m "rm x"
+	'
+	test_expect_success "setup case mac" '
+	git checkout -b mac_os
+	'
+	# This will test nfd2nfc in readdir()
+	test_expect_success "add file Adiarnfc" '
+		echo f.Adiarnfc >f.$Adiarnfc &&
+		git add f.$Adiarnfc &&
+		git commit -m "add f.$Adiarnfc"
+	'
+	# This will test nfd2nfc in git add()
+	test_expect_success "stage file d.Adiarnfd/f.Adiarnfd" '
+		mkdir d.$Adiarnfd &&
+		echo d.$Adiarnfd/f.$Adiarnfd >d.$Adiarnfd/f.$Adiarnfd &&
+		git stage d.$Adiarnfd/f.$Adiarnfd &&
+		git commit -m "add d.$Adiarnfd/f.$Adiarnfd"
+	'
+	test_expect_success "add link Adiarnfc" '
+		ln -s d.$Adiarnfd/f.$Adiarnfd l.$Adiarnfc &&
+		git add l.$Adiarnfc &&
+		git commit -m "add l.Adiarnfc"
+	'
+	# This will test git log
+	test_expect_success "git log f.Adiar" '
+		git log f.$Adiarnfc > f.Adiarnfc.log &&
+		git log f.$Adiarnfd > f.Adiarnfd.log &&
+		test -s f.Adiarnfc.log &&
+		test -s f.Adiarnfd.log &&
+		test_cmp f.Adiarnfc.log f.Adiarnfd.log &&
+		rm f.Adiarnfc.log f.Adiarnfd.log
+	'
+	# This will test git ls-files
+	test_expect_success "git lsfiles f.Adiar" '
+		git ls-files f.$Adiarnfc > f.Adiarnfc.log &&
+		git ls-files f.$Adiarnfd > f.Adiarnfd.log &&
+		test -s f.Adiarnfc.log &&
+		test -s f.Adiarnfd.log &&
+		test_cmp f.Adiarnfc.log f.Adiarnfd.log &&
+		rm f.Adiarnfc.log f.Adiarnfd.log
+	'
+	# This will test git mv
+	test_expect_success "git mv" '
+		git mv f.$Adiarnfd f.$Odiarnfc &&
+		git mv d.$Adiarnfd d.$Odiarnfc &&
+		git mv l.$Adiarnfd l.$Odiarnfc &&
+		git commit -m "mv Adiarnfd Odiarnfc"
+	'
+	# Files can be checked out as nfc
+	# And the link has been corrected from nfd to nfc
+	test_expect_success "git checkout nfc" '
+		rm f.$Odiarnfc &&
+		git checkout f.$Odiarnfc
+	'
+	# Make it possible to checkout files with their NFD names
+	test_expect_success "git checkout file nfd" '
+		rm -f f.* &&
+		git checkout f.$Odiarnfd
+	'
+	# Make it possible to checkout links with their NFD names
+	test_expect_success "git checkout link nfd" '
+		rm l.* &&
+		git checkout l.$Odiarnfd
+	'
+else
+	 say "Skipping nfc/nfd tests"
+fi
+
+test_done
-- 
1.7.8.rc0.43.gb49a8

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Miles Bader]

BTW, about the names, e.g. "darwin.c" etc -- is this code actually
Darwin-specific, or simply Systems-that-happen-to-force-decomposed-
unicode specific?

If the latter, maybe more generic names might be better.

Thanks,

-Miles

-- 
`To alcohol!  The cause of, and solution to,
 all of life's problems' --Homer J. Simpson

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Torsten Bögershausen]

On 08.01.12 07:01, Miles Bader wrote:
> BTW, about the names, e.g. "darwin.c" etc -- is this code actually
> Darwin-specific, or simply Systems-that-happen-to-force-decomposed-
> unicode specific?
> 
> If the latter, maybe more generic names might be better.
> 
> Thanks,
> 
> -Miles
> 
As far as I know, Mac OS (darwin) is the only existing OS which likes
decomposed unicode so much, that forces decomposed unicode that way.
/Torsten


 

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Torsten Bögershausen]

On 01/09/2012 08:29 PM, Junio C Hamano wrote:
> Torsten Bögershausen<tboegi@web.de>  writes:
>
>> On 08.01.12 03:46, Junio C Hamano wrote:
>> ...
>>> That also sounds sensible, but...
>>>
>>>> This is done in git.c by calling argv_precompose() for all commands
>>>> except "git commit".
>>>
>>> ... I think it generally is a bad idea to say "all except foo". There may
>>> be a reason why "foo" happens to be special in today's code, but who says
>>> there won't be another command "bar" that shares the same reason with
>>> "foo" to be treated specially? Or depending on the options, perhaps some
>>> codepath of "foo" may not want the special casing and want to go through
>>> the argv_precompose(), no?
>>>
>>> After all, "git commit -- pathspec" will have to get the pathspec from the
>>> command line,...
>>
>> Thanks Junio for catching this.
>> I added a new test case as well as fixed the code.
>
> I think you are sidestepping the real issue I raised, which is:
>
>      What is the reason why you do not want to feed the precompose helper
>      with some arguments to 'git commit', while it is OK to pass all
>      arguments to other commands through precomposition?
>
> I admit it was my fault that I did not spell it out clearly in my
> response.
>
> I understand that arguments other than pathspec and revs could be left in
> decomposed form, but is there any harm in canonicalizing any and all
> command line parameters given in decomposed form consistently into
> precomposed form? What problem are you trying to solve by special casing
> "git commit"? That is the real question to be answered, as there may be
> other commands some of whose arguments may not want to be canonicalized
> due to the same reason, but you simply overlooked them. When other people
> need to fix that oversight, they need a clearly written criterion what
> kind of arguments should not be fixed and why.
>
> And the reason cannot be a desire to pass the value to "--message"
> argument intact [*1*]; it is not like osx cannot handle text in
> precomposed form, right?

The short answer for treating "git commit" special:
   The test suite didn't pass any more. (t4201-shortlog.sh)
   This seems more and more to be a bad excuse...
The long answer:
   I have to look into that more deeply.

Thanks for your replies.
/Torsten

     (And yes, Mac OS can handle precomposed unicode (at least the
      western european code points))

[snip]

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Junio C Hamano]

Torsten Bögershausen <tboegi@web.de> writes:

> On 08.01.12 03:46, Junio C Hamano wrote:
> ...
>> That also sounds sensible, but...
>> 
>>> This is done in git.c by calling argv_precompose() for all commands
>>> except "git commit".
>> 
>> ... I think it generally is a bad idea to say "all except foo". There may
>> be a reason why "foo" happens to be special in today's code, but who says
>> there won't be another command "bar" that shares the same reason with
>> "foo" to be treated specially? Or depending on the options, perhaps some
>> codepath of "foo" may not want the special casing and want to go through
>> the argv_precompose(), no?
>> 
>> After all, "git commit -- pathspec" will have to get the pathspec from the
>> command line,...
>
> Thanks Junio for catching this.
> I added a new test case as well as fixed the code.

I think you are sidestepping the real issue I raised, which is:

    What is the reason why you do not want to feed the precompose helper
    with some arguments to 'git commit', while it is OK to pass all
    arguments to other commands through precomposition?

I admit it was my fault that I did not spell it out clearly in my
response.

I understand that arguments other than pathspec and revs could be left in
decomposed form, but is there any harm in canonicalizing any and all
command line parameters given in decomposed form consistently into
precomposed form? What problem are you trying to solve by special casing
"git commit"? That is the real question to be answered, as there may be
other commands some of whose arguments may not want to be canonicalized
due to the same reason, but you simply overlooked them. When other people
need to fix that oversight, they need a clearly written criterion what
kind of arguments should not be fixed and why.

And the reason cannot be a desire to pass the value to "--message"
argument intact [*1*]; it is not like osx cannot handle text in
precomposed form, right?

In general, I do not want to see ugly code that says "this one potentially
names a path so we add a call to fix it from decomposed form, but that
other one is not a path and we take it intact" sprinkled all over in the
codebase, without a good reason.

It may seem that one alternative to munging argv[] is to have the
precomposition [*2*] applied inside get_pathspec() and have it take effect
only on the pathspecs, which after all ought to be the only place where
this matters, but I doubt it would result in maintainable code. The names
of branches and tags taken from the command line that are used as revision
names will also be compared with results from readdir in $GIT_DIR/refs/
and need to be canonicalized, for example. So I tend to agree with your
"brute force" approach to canonicalize argv[] before any real part of git
sees them; that is where my suggestion to wrap main() to do so came from.

Also some commands (e.g. "rev-list --stdin") take pathspecs and revs from
their standard input stream, so you would need to be careful about them.


[Footnotes]

*1* Also as other commands like "git merge" also take textual message, and
you do pass the helper to canonicalize it.  No, I am not suggesting you to
special case "git merge".

*2* By the way, this may need a better name if the patch touches anywhere
outside compat/osx --- it is about "canonicalize pathname and pathspec
given from the command line into the form used internally by git", and
from an osx person's point of view, the only difference might be
decomposed vs precomposed, but on other odd systems it might be that
pathnames on the filesystem may be using a different encoding from what is
used for pathnames in the index).

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Torsten Bögershausen]

On 08.01.12 03:46, Junio C Hamano wrote:
> Torsten Bögershausen <tboegi@web.de> writes:
> 
>> Implementation:
>> Two files are added to the "compat" directory, darwin.h and darwin.c.
>> They implement basically 3 new functions:
>> darwin_opendir(), darwin_readdir() and darwin_closedir().
> 
> I haven't looked at the patch yet but that sounds exactly the right way to
> go about this. Nice.
> 
>> No decomposed file names in a git repository:
>> In order to prevent that ever a file name in decomposed unicode is entering
>> the index, a "brute force" attempt is taken:
>> all arguments into git (technically argv[1]..argv[n]) are converted into
>> precomposed unicode.
> 
> That also sounds sensible, but...
> 
>> This is done in git.c by calling argv_precompose() for all commands
>> except "git commit".
> 
> ... I think it generally is a bad idea to say "all except foo". There may
> be a reason why "foo" happens to be special in today's code, but who says
> there won't be another command "bar" that shares the same reason with
> "foo" to be treated specially? Or depending on the options, perhaps some
> codepath of "foo" may not want the special casing and want to go through
> the argv_precompose(), no?
> 
> After all, "git commit -- pathspec" will have to get the pathspec from the
> command line, and match them against the paths in the index, the latter of
> which you are keeping in the canonical form, so you would want the argv[]
> also be in the same form, and applying your argv_precompose() would be a
> sensible way to do so, no?

Thanks Junio for catching this.
I added a new test case as well as fixed the code.

> I would also suspect that the cleanest way to implement it is to replace
> the main() entry point (see how compat/mingw.h does this).

We only need to that argv conversion in git.c, (and not in daemon.c), so I sticked
to the old model for V1.
I send a new patch soon
/Torsten

Re: [PATCH][RFC] git on Mac OS and precomposed unicode

[Junio C Hamano]

Torsten Bögershausen <tboegi@web.de> writes:

> Implementation:
> Two files are added to the "compat" directory, darwin.h and darwin.c.
> They implement basically 3 new functions:
> darwin_opendir(), darwin_readdir() and darwin_closedir().

I haven't looked at the patch yet but that sounds exactly the right way to
go about this. Nice.

> No decomposed file names in a git repository:
> In order to prevent that ever a file name in decomposed unicode is entering
> the index, a "brute force" attempt is taken:
> all arguments into git (technically argv[1]..argv[n]) are converted into
> precomposed unicode.

That also sounds sensible, but...

> This is done in git.c by calling argv_precompose() for all commands
> except "git commit".

... I think it generally is a bad idea to say "all except foo". There may
be a reason why "foo" happens to be special in today's code, but who says
there won't be another command "bar" that shares the same reason with
"foo" to be treated specially? Or depending on the options, perhaps some
codepath of "foo" may not want the special casing and want to go through
the argv_precompose(), no?

After all, "git commit -- pathspec" will have to get the pathspec from the
command line, and match them against the paths in the index, the latter of
which you are keeping in the canonical form, so you would want the argv[]
also be in the same form, and applying your argv_precompose() would be a
sensible way to do so, no?

I would also suspect that the cleanest way to implement it is to replace
the main() entry point (see how compat/mingw.h does this).

[PATCH][RFC] git on Mac OS and precomposed unicode

[Torsten Bögershausen]

Purpose:
This patch is a suggestion to work around the unpleasenties
when Mac OS is decomposing unicode filenames.

The suggested change:
a) is only used under Mac OS
b) can be switched off by a configuration variable
c) is optimized to handle ASCII only filename
d) will improve the interwork between Mac OS, Linux and Windows*
   via git push/pull, using USB sticks (technically speaking VFAT)
   or mounted network shares using samba.

* (Not all Windows versions support UTF-8 yet:
   Msysgit needs the unicode branch, cygwin supports UTF-8 since 1.7)


Runtime configuration:
A new confguration variable is added: "core.precomposedunicode"
This variable is only used on Mac OS.
If set to false, git behaves exactly as older versions of git.
When a new git version is installed and there is a repository
where the configuration "core.precomposedunicode" is not present,
the new git is backward compatible.

When core.precomposedunicode=true, all filenames are stored in precomposed
unicode in the index (technically speaking precomposed UTF-8).
Even when readdir() under Mac OS returns filenames as decomposed.

Implementation:
Two files are added to the "compat" directory, darwin.h and darwin.c.
They implement basically 3 new functions:
darwin_opendir(), darwin_readdir() and darwin_closedir().


Compile time configuration:
A new compiler option PRECOMPOSED_UNICODE is introduced in the Makefile,
so that the patch can be switched off completely at compile time.

No decomposed file names in a git repository:
In order to prevent that ever a file name in decomposed unicode is entering
the index, a "brute force" attempt is taken:
all arguments into git (technically argv[1]..argv[n]) are converted into
precomposed unicode.
This is done in git.c by calling argv_precompose() for all commands
except "git commit".
This function is actually a #define, and it is only defined under Mac OS.
Nothing is converted on any other OS.

Implementation details:
The main work is done in darwin_readdir() and argv_precompose().
The conversion into precomposed unicode is done by using iconv,
where decomposed is denoted by "UTF-8-MAC" and precomposed is "UTF-8".
When already precomposed unicode is precomposed, the string is returned
unchanged.

Thread save:
Since there is no need for argv_precompose()to be thread-save, one iconv
instance is created at the beginning and kept for all conversions.
Even readdir() is not thread-save, so that darwin_opendir() will call
iconv_open() once and keep the instance for all calls of darwin_readdir()
until darwin_close() is called.

Auto sensing:
When creating a new git repository with "git init" or "git clone", the
"core.precomposedunicode" will be set automatically to "true" or "false".

Typically core.precomposedunicode is "true" on HFS and VFAT.
It is even true for file systems mounted via SAMBA onto a Linux box,
and "false" for drives mounted via NFS onto a Linux box.


New test case:
The new t3910-mac-os-precompose.sh is added to check if a filename
can be reached either in precomposed or decomposed unicode (NFC or NFD).


 Documentation/config.txt     |    9 ++
 Makefile                     |    3 +
 builtin/init-db.c            |   22 +++++
 compat/darwin.c              |  200 ++++++++++++++++++++++++++++++++++++++++++
 compat/darwin.h              |   31 +++++++
 git-compat-util.h            |    8 ++
 git.c                        |    1 +
 t/t0050-filesystem.sh        |    1 +
 t/t3910-mac-os-precompose.sh |  104 ++++++++++++++++++++++
 9 files changed, 379 insertions(+), 0 deletions(-)
 create mode 100644 compat/darwin.c
 create mode 100644 compat/darwin.h
 create mode 100755 t/t3910-mac-os-precompose.sh

-- 
1.7.8.rc0.43.gb49a8