[PATCH 1/4] Add routines for filenames encoding <local encoding> <-> UTF-8

[PATCH 1/4] Add routines for filenames encoding <local encoding> <-> UTF-8

[Timur Sufiev]

The point is to make Git aware of filenames local encoding and make it
keep all filenames in UTF-8 internally. If
`i18n.filenameslocalencoding' option was set via git-config to a
correct <codepage> encoding, 2 things should be done:

1. Translate all filenames read by READDIR from <codepage> into UTF-8.

2. Translate all filenames passed to IO-routines from UTF-8 into
<codepage>.

Signed-off-by: Timur Sufiev <timur@iris-comp.ru>
---
 cache.h       |    1 +
 config.c      |    3 ++
 environment.c |    1 +
 io-i18n.c     |   82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 87 insertions(+), 0 deletions(-)
 create mode 100644 io-i18n.c

diff --git a/cache.h b/cache.h
index 96840c7..7f19f7a 100644
--- a/cache.h
+++ b/cache.h
@@ -919,6 +919,7 @@ extern int user_ident_explicitly_given;
 
 extern const char *git_commit_encoding;
 extern const char *git_log_output_encoding;
+extern const char *git_filenames_local_encoding;
 extern const char *git_mailmap_file;
 
 /* IO helper functions */
diff --git a/config.c b/config.c
index c644061..2be6531 100644
--- a/config.c
+++ b/config.c
@@ -539,6 +539,9 @@ static int git_default_i18n_config(const char *var, const char *value)
 	if (!strcmp(var, "i18n.logoutputencoding"))
 		return git_config_string(&git_log_output_encoding, var, value);
 
+	if (!strcmp(var, "i18n.filenameslocalencoding"))
+	     return git_config_string(&git_filenames_local_encoding, var, value);
+
 	/* Add other config variables here and to Documentation/config.txt. */
 	return 0;
 }
diff --git a/environment.c b/environment.c
index 5de6837..b101f7b 100644
--- a/environment.c
+++ b/environment.c
@@ -24,6 +24,7 @@ int warn_ambiguous_refs = 1;
 int repository_format_version;
 const char *git_commit_encoding;
 const char *git_log_output_encoding;
+const char *git_filenames_local_encoding;
 int shared_repository = PERM_UMASK;
 const char *apply_default_whitespace;
 const char *apply_default_ignorewhitespace;
diff --git a/io-i18n.c b/io-i18n.c
new file mode 100644
index 0000000..4dcc2db
--- /dev/null
+++ b/io-i18n.c
@@ -0,0 +1,82 @@
+#include "utf8.h"
+#include "cache.h"
+
+inline static int is_string_ascii(const char *str)
+{
+	int is_ascii = 1;
+
+	for (; *str && is_ascii; str++)
+		is_ascii &= isascii(*str);
+
+	return is_ascii;
+}
+
+static char *filename_to_utf8(const char *filename)
+{
+	char *out;
+
+	if (is_string_ascii(filename))
+		return NULL;
+
+#ifndef NO_ICONV
+	if (git_filenames_local_encoding && !is_utf8(filename)) {
+		out = reencode_string(filename,
+				      "utf-8", git_filenames_local_encoding);
+#ifdef DEBUG_I18N
+		fprintf(stderr, "Local -> UTF8 encoding: <%s> -> <%s>\n",
+			filename, out);
+#endif
+		return out;
+	} else if (git_filenames_local_encoding && is_utf8(filename)) {
+#ifdef DEBUG_I18N
+		fprintf(stderr,
+			"Filename <%s> is already utf8-encoded, doing nothing...\n",
+			filename);
+#endif
+		return NULL;
+	} else {
+#ifdef DEBUG_I18N
+		fprintf(stderr, "No local encoding set, doing nothing...\n");
+#endif
+		return NULL;
+	}
+#else /* #ifdef NO_ICONV */
+	warning("No iconv support, doing nothing...\n");
+	return NULL;
+#endif
+}
+
+char *filename_to_local(const char *filename)
+{
+	char *out;
+
+	if (is_string_ascii(filename))
+		return NULL;
+
+#ifndef NO_ICONV
+	if (git_filenames_local_encoding && is_utf8(filename)) {
+		out = reencode_string(filename,
+				      git_filenames_local_encoding, "utf-8");
+#ifdef DEBUG_I18N
+		fprintf(stderr, "UTF8 -> local encoding: <%s> -> <%s>\n",
+			filename, out);
+#endif
+		return out;
+	} else if (git_filenames_local_encoding && !is_utf8(filename)) {
+#ifdef DEBUG_I18N
+		fprintf(stderr,
+			"Filename <%s> is already local-encoded, doing nothing...\n",
+			filename);
+#endif
+		return NULL;
+	} else {
+#ifdef DEBUG_I18N
+		fprintf(stderr, "No local encoding set, doing nothing...\n");
+#endif
+		return NULL;
+	}
+#else /* #ifdef NO_ICONV */
+	warning("No iconv support, doing nothing...\n");
+	return NULL;
+#endif
+}
-- 
1.6.5.1

[PATCH 2/4] Add I18N-wrappers for low-level IO-routines

[Timur Sufiev]

Signed-off-by: Timur Sufiev <timur@iris-comp.ru>
---
 Makefile  |    2 +
 io-i18n.c |  129 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 io-i18n.h |   23 +++++++++++
 3 files changed, 154 insertions(+), 0 deletions(-)
 create mode 100644 io-i18n.h

diff --git a/Makefile b/Makefile
index 42b7d60..ac8e807 100644
--- a/Makefile
+++ b/Makefile
@@ -459,6 +459,7 @@ LIB_H += tree-walk.h
 LIB_H += unpack-trees.h
 LIB_H += userdiff.h
 LIB_H += utf8.h
+LIB_H += io-i18n.h
 LIB_H += wt-status.h
 
 LIB_OBJS += abspath.o
@@ -562,6 +563,7 @@ LIB_OBJS += unpack-trees.o
 LIB_OBJS += usage.o
 LIB_OBJS += userdiff.o
 LIB_OBJS += utf8.o
+LIB_OBJS += io-i18n.o
 LIB_OBJS += walker.o
 LIB_OBJS += wrapper.o
 LIB_OBJS += write_or_die.o
diff --git a/io-i18n.c b/io-i18n.c
index 4dcc2db..9d89ac3 100644
--- a/io-i18n.c
+++ b/io-i18n.c
@@ -1,3 +1,4 @@
+#include "io-i18n.h"
 #include "utf8.h"
 #include "cache.h"
 
@@ -80,3 +81,131 @@ char *filename_to_local(const char *filename)
 	return NULL;
 #endif
 }
+
+int stat_i18n(const char *filename, struct stat *buf)
+{
+	int ret;
+	char *out = filename_to_local(filename);
+
+	if (out != NULL) {
+		ret = stat(out, buf);
+		free(out);
+	} else
+		ret = stat(filename, buf);
+	return ret;
+}
+
+int lstat_i18n(const char *filename, struct stat *buf)
+{
+
+	int ret;
+	char *out = filename_to_local(filename);
+
+	if (out != NULL) {
+		ret = lstat(out, buf);
+		free(out);
+	} else
+		ret = lstat(filename, buf);
+	return ret;
+}
+
+DIR *opendir_i18n(const char *dirname)
+{
+	DIR *dir;
+	char *out = filename_to_local(dirname);
+
+	if (out != NULL) {
+		dir = opendir(out);
+		free(out);
+	} else
+		dir = opendir(dirname);
+	return dir;
+}
+
+struct dirent *readdir_i18n(DIR * dirstream)
+{
+	struct dirent *de = readdir(dirstream);
+
+	if (de) {
+		char *out = filename_to_utf8(de->d_name);
+
+		if (out) {
+			int len = strlen(out);
+			if (len >= NAME_MAX) {
+				warning("readdir_i18n: converted dir entry name length exceeds NAME_MAX and will be truncated\n");
+				len = NAME_MAX - 1;
+			}
+			memcpy(de->d_name, out, len);
+			de->d_name[len] = '\0';
+			free(out);
+		}
+		return de;
+	} else
+		return NULL;
+}
+
+int open_i18n(const char *filename, int flags, mode_t mode)
+{
+	int ret;
+	char *out = filename_to_local(filename);
+
+	if (out != NULL) {
+		ret = open(out, flags, mode);
+		free(out);
+	} else
+		ret = open(filename, flags, mode);
+	return ret;
+}
+
+FILE *fopen_i18n(const char *filename, const char *opentype)
+{
+	FILE *file;
+	char *out = filename_to_local(filename);
+
+	if (out != NULL) {
+		file = fopen(out, opentype);
+		free(out);
+	} else
+		file = fopen(filename, opentype);
+	return file;
+}
+
+int chmod_i18n(const char *filename, mode_t mode)
+{
+	int ret;
+	char *out = filename_to_local(filename);
+
+	if (out != NULL) {
+		ret = chmod(out, mode);
+		free(out);
+	} else
+		ret = chmod(filename, mode);
+	return ret;
+}
+
+int link_i18n(const char *oldname, const char *newname)
+{
+	char *old_out = filename_to_local(oldname);
+	char *new_out = filename_to_local(newname);
+	int ret = link(old_out ? old_out : oldname,
+		       new_out ? new_out : newname);
+
+	if (old_out)
+		free(old_out);
+	if (new_out)
+		free(new_out);
+	return ret;
+}
+
+int unlink_i18n(const char *filename)
+{
+	char *out = filename_to_local(filename);
+	int ret;
+
+	if (out) {
+		ret = unlink(out);
+		free(out);
+	} else
+		ret = unlink(filename);
+	return ret;
+}
diff --git a/io-i18n.h b/io-i18n.h
new file mode 100644
index 0000000..c386e20
--- /dev/null
+++ b/io-i18n.h
@@ -0,0 +1,23 @@
+#ifndef GIT_IO_I18N_H
+#define GIT_IO_I18N_H
+
+#define _FILE_OFFSET_BITS 64
+
+#include <sys/stat.h>
+#include <dirent.h>
+#include <stdio.h>
+
+#define DEFAULT_OPEN_MODE 0
+
+char *filename_to_local (const char* filename);
+int stat_i18n(const char *filename, struct stat *buf);
+int lstat_i18n(const char *filename, struct stat *buf);
+DIR* opendir_i18n(const char *dirname);
+struct dirent *readdir_i18n(DIR *dirstream);
+int open_i18n(const char *filename, int flags, mode_t mode);
+FILE *fopen_i18n(const char *filename, const char *opentype);
+int chmod_i18n(const char *filename, mode_t mode);
+int link_i18n(const char *oldname, const char *newname);
+int unlink_i18n(const char *filename);
+
+#endif /* GIT_IO_I18N_H */
-- 
1.6.5.1

[PATCH 4/4] Make mingw-compatibility layer to be aware of I18N-wrappers

[Timur Sufiev]

Signed-off-by: Timur Sufiev <timur@iris-comp.ru>
---
 compat/fopen.c       |    5 +++--
 compat/mingw.c       |   24 ++++++++++++++++++------
 compat/mingw.h       |   12 ++++++++++++
 compat/mkstemps.c    |    3 ++-
 compat/regex/regex.c |    9 ++++++++-
 compat/regex/regex.h |    3 +++
 compat/win32.h       |   13 +++++++++++--
 io-i18n.h            |    8 ++++++++
 8 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/compat/fopen.c b/compat/fopen.c
index b5ca142..9136a14 100644
--- a/compat/fopen.c
+++ b/compat/fopen.c
@@ -10,6 +10,7 @@
  */
 #undef FREAD_READS_DIRECTORIES
 #include "../git-compat-util.h"
+#include "io-i18n.h"
 
 FILE *git_fopen(const char *path, const char *mode)
 {
@@ -17,9 +18,9 @@ FILE *git_fopen(const char *path, const char *mode)
 	struct stat st;
 
 	if (mode[0] == 'w' || mode[0] == 'a')
-		return fopen(path, mode);
+		return fopen_i18n(path, mode);
 
-	if (!(fp = fopen(path, mode)))
+	if (!(fp = fopen_i18n(path, mode)))
 		return NULL;
 
 	if (fstat(fileno(fp), &st)) {
diff --git a/compat/mingw.c b/compat/mingw.c
index 6b5b5b2..9cb135a 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -253,7 +253,7 @@ int mingw_utime (const char *file_name, const struct utimbuf *times)
 	int fh, rc;
 
 	/* must have write permission */
-	if ((fh = open(file_name, O_RDWR | O_BINARY)) < 0)
+	if ((fh = open_i18n(file_name, O_RDWR | O_BINARY, DEFAULT_OPEN_MODE)) < 0)
 		return -1;
 
 	time_t_to_filetime(times->modtime, &mft);
@@ -278,7 +278,7 @@ int mkstemp(char *template)
 	char *filename = mktemp(template);
 	if (filename == NULL)
 		return -1;
-	return open(filename, O_RDWR | O_CREAT, 0600);
+	return open_i18n(filename, O_RDWR | O_CREAT, 0600);
 }
 
 int gettimeofday(struct timeval *tv, void *tz)
@@ -519,7 +519,7 @@ static const char *parse_interpreter(const char *cmd)
 	if (n >= 4 && !strcasecmp(cmd+n-4, ".exe"))
 		return NULL;
 
-	fd = open(cmd, O_RDONLY);
+	fd = open_i18n(cmd, O_RDONLY, DEFAULT_OPEN_MODE);
 	if (fd < 0)
 		return NULL;
 	n = read(fd, buf, sizeof(buf)-1);
@@ -1135,10 +1135,14 @@ sig_handler_t mingw_signal(int sig, sig_handler_t handler)
 static const char *make_backslash_path(const char *path)
 {
 	static char buf[PATH_MAX + 1];
+	char *out = filename_to_local(path);
 	char *c;
 
-	if (strlcpy(buf, path, PATH_MAX) >= PATH_MAX)
-		die("Too long path: %.*s", 60, path);
+	if ( strlcpy(buf, out ? out : path, PATH_MAX) >= PATH_MAX) {
+	     die("Too long path: %.*s", 60, out ? out : path);
+	     free(out);
+	}
+	free(out);
 
 	for (c = buf; *c; c++) {
 		if (*c == '/')
@@ -1158,6 +1162,9 @@ int link(const char *oldpath, const char *newpath)
 {
 	typedef BOOL (WINAPI *T)(const char*, const char*, LPSECURITY_ATTRIBUTES);
 	static T create_hard_link = NULL;
+	char *old_out = filename_to_local(oldpath);
+	char *new_out = filename_to_local(newpath);
+
 	if (!create_hard_link) {
 		create_hard_link = (T) GetProcAddress(
 			GetModuleHandle("kernel32.dll"), "CreateHardLinkA");
@@ -1168,10 +1175,15 @@ int link(const char *oldpath, const char *newpath)
 		errno = ENOSYS;
 		return -1;
 	}
-	if (!create_hard_link(newpath, oldpath, NULL)) {
+	if (!create_hard_link(new_out ? new_out : newpath,
+			      old_out ? old_out : oldpath, NULL)) {
+		free(new_out);
+		free(old_out);
 		errno = err_win_to_posix(GetLastError());
 		return -1;
 	}
+	free(new_out);
+	free(old_out);
 	return 0;
 }
 
diff --git a/compat/mingw.h b/compat/mingw.h
index 5b5258b..0d1dc83 100644
--- a/compat/mingw.h
+++ b/compat/mingw.h
@@ -1,5 +1,16 @@
+#ifndef GIT_MINGW_H
+#define GIT_MINGW_H
 #include <winsock2.h>
 
+/* 3 following lines are taken from io-i18n.h because we cannot
+   include it here: if we do so, readdir_i18n prototype will
+   be inconsistent with with other code which properly uses all
+   mingw defines
+*/
+char *filename_to_local (const char* filename);
+int open_i18n(const char *filename, int flags, mode_t mode);
+#define DEFAULT_OPEN_MODE 0
+
 /*
  * things that are not available in header files
  */
@@ -273,3 +284,4 @@ struct mingw_dirent
 #define readdir(x) mingw_readdir(x)
 struct dirent *mingw_readdir(DIR *dir);
 #endif // !NO_MINGW_REPLACE_READDIR
+#endif // GIT_MINGW_H
diff --git a/compat/mkstemps.c b/compat/mkstemps.c
index 14179c8..0cdd42d 100644
--- a/compat/mkstemps.c
+++ b/compat/mkstemps.c
@@ -1,4 +1,5 @@
 #include "../git-compat-util.h"
+#include "../io-i18n.h"
 
 /* Adapted from libiberty's mkstemp.c. */
 
@@ -47,7 +48,7 @@ int gitmkstemps(char *pattern, int suffix_len)
 		template[4] = letters[v % num_letters]; v /= num_letters;
 		template[5] = letters[v % num_letters]; v /= num_letters;
 
-		fd = open(pattern, O_CREAT | O_EXCL | O_RDWR, 0600);
+		fd = open_i18n(pattern, O_CREAT | O_EXCL | O_RDWR, 0600);
 		if (fd > 0)
 			return fd;
 		/*
diff --git a/compat/regex/regex.c b/compat/regex/regex.c
index 67d5c37..2deec55 100644
--- a/compat/regex/regex.c
+++ b/compat/regex/regex.c
@@ -450,8 +450,15 @@ static int debug = 0;
 #define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)			\
   if (debug) print_double_string (w, s1, sz1, s2, sz2)
 
-
+#ifdef __USE_MINGW_ACCESS
+void
+ printchar(char c)
+{
+  putc(c, stderr);
+}
+#else
 extern void printchar ();
+#endif // __USE_MINGW_ACCESS
 
 /* Print the fastmap in human-readable form.  */
 
diff --git a/compat/regex/regex.h b/compat/regex/regex.h
index 6eb64f1..a5f2809 100644
--- a/compat/regex/regex.h
+++ b/compat/regex/regex.h
@@ -29,6 +29,9 @@
 #include <stddef.h>
 #endif
 
+#ifdef __USE_MINGW_ACCESS
+#include <conio.h>
+#endif
 
 /* The following bits are used to determine the regexp syntax we
    recognize.  The set/not-set meanings are chosen so that Emacs syntax
diff --git a/compat/win32.h b/compat/win32.h
index 8ce9104..f71e36d 100644
--- a/compat/win32.h
+++ b/compat/win32.h
@@ -20,8 +20,17 @@ static inline int file_attr_to_st_mode (DWORD attr)
 
 static inline int get_file_attr(const char *fname, WIN32_FILE_ATTRIBUTE_DATA *fdata)
 {
-	if (GetFileAttributesExA(fname, GetFileExInfoStandard, fdata))
-		return 0;
+	char *out = filename_to_local(fname);
+	int ret;
+
+	if ( out != NULL ) {
+	     ret = GetFileAttributesExA(out, GetFileExInfoStandard, fdata);
+	     free(out);
+	} else
+	     ret = GetFileAttributesExA(fname, GetFileExInfoStandard, fdata);
+
+	if ( ret )
+	     return 0;
 
 	switch (GetLastError()) {
 	case ERROR_ACCESS_DENIED:
diff --git a/io-i18n.h b/io-i18n.h
index c386e20..7329105 100644
--- a/io-i18n.h
+++ b/io-i18n.h
@@ -6,6 +6,14 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include <stdio.h>
+#include <limits.h>
+
+#ifdef __USE_MINGW_ACCESS
+#include <windows.h>
+#define NAME_MAX MAX_PATH
+#define stat _stati64
+#define dirent mingw_dirent
+#endif
 
 #define DEFAULT_OPEN_MODE 0
 
-- 
1.6.5.1

Re: [PATCH 1/4] Add routines for filenames encoding <local encoding> <-> UTF-8

[Peter Krefting]

Timur Sufiev:

> The point is to make Git aware of filenames local encoding and make it 
> keep all filenames in UTF-8 internally.

Good.

> If `i18n.filenameslocalencoding' option was set via git-config to a 
> correct <codepage> encoding, 2 things should be done:

Windows supports UTF-16 file names, but need to use wchar_t APIs for fopen() 
and friends. Have you looked at any of that?

-- 
\\// Peter - http://www.softwolves.pp.se/

Re: [PATCH 2/4] Add I18N-wrappers for low-level IO-routines

[Jeff King]

On Tue, Oct 27, 2009 at 04:54:01PM +0300, Timur Sufiev wrote:

> Signed-off-by: Timur Sufiev <timur@iris-comp.ru>

Hmm. Two questions about this series:

  1. Patch 3/4 didn't seem to make it to the list. Presumably that is
     where you actually use these routines in git? Or are they just for
     mingw?

  2. I seem to recall that Linus added a filename translation layer for
     doing much more, like handling unicode normalizations (but I
     confess I haven't looked closely at that code). Should this be part
     of that system?

-Peff

Re: [PATCH 1/4] Add routines for filenames encoding <local encoding> <-> UTF-8

[Timur Sufiev]

>> If `i18n.filenameslocalencoding' option was set via git-config to a 
>> correct <codepage> encoding, 2 things should be done:
> 
> Windows supports UTF-16 file names, but need to use wchar_t APIs for fopen() 
> and friends. Have you looked at any of that?

No, I didn't look towards Windows handling of Unicode, because Git under
Windows works well enough without that: I've built Git in MinGW/MSYS
environmment (using the patches we're discussing) and it handles
cyrillic filenames in Windows ANSI CP1251 codepage just as was planned:
writes filenames to tree objects in UTF-8, checks out them into working
dir in CP1251, other stuff as git-clone, git-diff, git-status, etc also
works ok. One remaining issue (offtopic here, as far as I understand) to
make TortoiseGit work with UTF-8 Git's output.
 
> -- 
> \\// Peter - http://www.softwolves.pp.se/

-- 
Timur Sufiev

Re: [PATCH 2/4] Add I18N-wrappers for low-level IO-routines

[Timur Sufiev]

> Hmm. Two questions about this series:
> 
>   1. Patch 3/4 didn't seem to make it to the list. Presumably that is
>      where you actually use these routines in git? Or are they just for
>      mingw?

Yes, it actually haven't made it to the list. Perhaps this was due to
patch size: it was approx. 3300 lines long (BTW, what's the message size
limit?) So I've rewritten the patch to make it more compact, using mingw
approach with macros. Subj prefix for a patch series is 'PATCH I18N
filenames v2'. 

>   2. I seem to recall that Linus added a filename translation layer for
>      doing much more, like handling unicode normalizations (but I
>      confess I haven't looked closely at that code). Should this be part
>      of that system?

I've heard nothing about that :(. Could you point me directly at Linus'
changes?

> -Peff

-- 
Timur Sufiev

Re: [PATCH 2/4] Add I18N-wrappers for low-level IO-routines

[Jeff King]

On Wed, Oct 28, 2009 at 09:01:21PM +0300, Timur Sufiev wrote:

> Yes, it actually haven't made it to the list. Perhaps this was due to
> patch size: it was approx. 3300 lines long (BTW, what's the message size
> limit?) So I've rewritten the patch to make it more compact, using mingw
> approach with macros. Subj prefix for a patch series is 'PATCH I18N
> filenames v2'. 

Thanks. The rules for vger are here:

  http://vger.kernel.org/majordomo-info.html

The max size is 100K, but you may also be triggering something from the
taboo list accidentally.

> >   2. I seem to recall that Linus added a filename translation layer for
> >      doing much more, like handling unicode normalizations (but I
> >      confess I haven't looked closely at that code). Should this be part
> >      of that system?
> 
> I've heard nothing about that :(. Could you point me directly at Linus'
> changes?

Try looking at this series:

  http://thread.gmane.org/gmane.comp.version-control.git/119222

-Peff