linux-erofs.lists.ozlabs.org archive mirror
 help / color / mirror / Atom feed
* [PATCH v2] erofs-utils: introduce exclude dirs and files
@ 2020-02-03 15:38 Li Guifu
  2020-02-04 17:02 ` Gao Xiang via Linux-erofs
  0 siblings, 1 reply; 4+ messages in thread
From: Li Guifu @ 2020-02-03 15:38 UTC (permalink / raw)
  To: linux-erofs


From: Li Guifu <blucer.lee@foxmail.com>

Add excluded file feature "--exclude-path=", which can be used
to build EROFS image without some user specific files or dirs.
Such multiple files can be seperated by ','.

Signed-off-by: Li Guifu <blucer.lee@foxmail.com>
---
 include/erofs/exclude.h |  28 ++++++++
 lib/Makefile.am         |   2 +-
 lib/exclude.c           | 149 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  10 +++
 mkfs/main.c             |  11 +++
 5 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c

diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 0000000..3581046
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * exclude.h
+ * Created by Li Guifu <blucer.lee@foxmail.com>
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct exclude_rule {
+	struct list_head list;
+	struct list_head hlist;
+	char *pattern;
+};
+
+void erofs_init_exclude_rules(void);
+void erofs_free_exclude_rules(void);
+int erofs_parse_exclude_path(const char *args);
+struct exclude_rule *erofs_pattern_matched(const char *s);
+struct exclude_rule *erofs_entry_matched(struct exclude_rule *e,
+					 const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct exclude_rule *e)
+{
+	return list_empty(&e->hlist);
+}
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 0000000..1c65fb9
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * exclude.c
+ * Created by Li Guifu <blucer.lee@foxmail.com>
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+#include "erofs/err.h"
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct exclude_rule exclude_rules;
+
+static struct exclude_rule *new_rule(const char *s, unsigned int len)
+{
+	struct exclude_rule *e = malloc(sizeof(struct exclude_rule));
+
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+
+	e->pattern = strndup(s, len);
+	if (!e->pattern)
+		goto err_strdup;
+
+	init_list_head(&e->hlist);
+	init_list_head(&e->list);
+
+	return e;
+
+err_strdup:
+	free(e);
+	return ERR_PTR(-ENOMEM);
+}
+
+struct exclude_rule *erofs_entry_matched(struct exclude_rule *e,
+					 const char *s, unsigned int len)
+{
+	struct exclude_rule *pos;
+
+	while (*s == '/') {
+		s++;
+		len--;
+	}
+	list_for_each_entry(pos, &e->hlist, list) {
+		unsigned int l = strlen(pos->pattern);
+
+		if (l == len && !strcmp(pos->pattern, s))
+			return pos;
+	}
+
+	return ERR_PTR(-ENOMEM);
+}
+
+static int add_exclude_rules(struct exclude_rule *e, const char *s)
+{
+	const char *ptr;
+	struct exclude_rule *erule;
+
+	while (*s == '/')
+		s++;
+	ptr = s;
+
+forward:
+	while(*ptr != '/' && *ptr != '\0')
+		ptr++;
+
+	erule = erofs_entry_matched(e, s, ptr - s);
+	if (IS_ERR(erule)) {
+		struct exclude_rule *r = new_rule(s, ptr - s);
+
+		if (IS_ERR(r))
+			return PTR_ERR(r);
+		list_add_tail(&r->list, &e->hlist);
+		erule = r;
+	}
+	e = erule;
+
+	if (*ptr++ != '\0') {
+		s = ptr;
+		goto forward;
+	}
+
+	return 0;
+}
+
+static void free_exclude_rules(struct list_head *h)
+{
+	struct exclude_rule *e, *n;
+
+	list_for_each_entry_safe(e, n, h, list) {
+		list_del(&e->list);
+		free_exclude_rules(&e->hlist);
+		free(e->pattern);
+		free(e);
+	}
+}
+void erofs_init_exclude_rules(void)
+{
+	init_list_head(&exclude_rules.list);
+	init_list_head(&exclude_rules.hlist);
+	exclude_rules.pattern = "/";
+}
+
+void erofs_free_exclude_rules(void)
+{
+	free_exclude_rules(&exclude_rules.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+	const char *str, *ptr = args;
+	const char sep = ',';
+
+forward:
+	while(*ptr != sep && *ptr != '\0')
+		ptr++;
+
+	str = strndup(args, ptr - args);
+	if (!str)
+		goto err_free_paths;
+
+	if (add_exclude_rules(&exclude_rules, str))
+		goto err_free_paths;
+
+	if (*ptr++ != '\0') {
+		args = ptr;
+		goto forward;
+	}
+
+	return 0;
+
+err_free_paths:
+	erofs_free_exclude_rules();
+	return -ENOMEM;
+}
+
+struct exclude_rule *erofs_pattern_matched(const char *s)
+{
+	unsigned int len = strlen(s);
+
+	if (!len)
+		return &exclude_rules;
+
+	return erofs_entry_matched(&exclude_rules, s, len);
+}
diff --git a/lib/inode.c b/lib/inode.c
index bd0652b..567f1b8 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -825,6 +826,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 	DIR *_dir;
 	struct dirent *dp;
 	struct erofs_dentry *d;
+	struct exclude_rule *e;
 
 	ret = erofs_prepare_xattr_ibody(dir->i_srcpath, &dir->i_xattrs);
 	if (ret < 0)
@@ -863,6 +865,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		return ERR_PTR(-errno);
 	}
 
+	e = erofs_pattern_matched(dir->i_srcpath + strlen(cfg.c_src_path));
 	while (1) {
 		/*
 		 * set errno to 0 before calling readdir() in order to
@@ -876,7 +879,14 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		if (is_dot_dotdot(dp->d_name) ||
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
+		if (!IS_ERR(e)) {
+			struct exclude_rule *le;
 
+			le = erofs_entry_matched(e, dp->d_name,
+						 strlen(dp->d_name));
+			if (!IS_ERR(le) && erofs_is_pattern_end(le))
+				continue;
+		}
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
diff --git a/mkfs/main.c b/mkfs/main.c
index 817a6c1..6f32c60 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 2},
 	{0, 0, 0, 0},
 };
 
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 2:
+			if (erofs_parse_exclude_path(optarg)) {
+				usage();
+				exit(0);
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -334,6 +343,7 @@ int main(int argc, char **argv)
 	struct timeval t;
 
 	erofs_init_configure();
+	erofs_init_exclude_rules();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
 
 	cfg.c_legacy_compress = false;
@@ -429,6 +439,7 @@ exit:
 	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
+	erofs_free_exclude_rules();
 
 	if (err) {
 		erofs_err("\tCould not format the device : %s\n",
-- 
2.17.1



^ permalink raw reply related	[flat|nested] 4+ messages in thread

* Re: [PATCH v2] erofs-utils: introduce exclude dirs and files
  2020-02-03 15:38 [PATCH v2] erofs-utils: introduce exclude dirs and files Li Guifu
@ 2020-02-04 17:02 ` Gao Xiang via Linux-erofs
       [not found]   ` <108bd2aa-182f-c5d3-53eb-54ad0e313b76@foxmail.com>
  2020-02-16  5:27   ` Li GuiFu via Linux-erofs
  0 siblings, 2 replies; 4+ messages in thread
From: Gao Xiang via Linux-erofs @ 2020-02-04 17:02 UTC (permalink / raw)
  To: Li Guifu; +Cc: linux-erofs

Hi Guifu,

I cleanup the whole patch and get rid of all dedupe check in order
to make it as simple as possible since I think not too many exclude
files in general.

Please take some time helping testing the following patch and
hope to get your feedback.

Thanks,
Gao Xiang



From 600fc166cdaaa0e458181729245ce11affa83ac5 Mon Sep 17 00:00:00 2001
From: Li Guifu <blucer.lee@foxmail.com>
Date: Mon, 3 Feb 2020 23:38:11 +0800
Subject: [PATCH v3] erofs-utils: introduce exclude dirs and files

Add excluded file feature "--exclude-path=", which can be used
to build EROFS image without some user specific files or dirs.

Note that you may give multiple `--exclude-path' options.

Signed-off-by: Li Guifu <blucer.lee@foxmail.com>
Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
---
changes since v2:
 - cleanup the whole implementation;
 - complete usage message;
 - complete manual page.

 include/erofs/exclude.h | 23 +++++++++++
 lib/Makefile.am         |  2 +-
 lib/exclude.c           | 89 +++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  5 +++
 man/mkfs.erofs.1        |  4 ++
 mkfs/main.c             | 26 +++++++++---
 6 files changed, 142 insertions(+), 7 deletions(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c

diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 0000000..580fefe
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/include/erofs/exclude.h
+ *
+ * Created by Li Guifu <blucer.lee@foxmail.com>
+ */
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct erofs_exclude_rule {
+	struct list_head list;
+
+	char *pattern;
+};
+
+void erofs_exclude_set_root(const char *rootdir);
+void erofs_cleanup_exclude_rules(void);
+
+int erofs_parse_exclude_path(const char *args);
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+						 const char *name);
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 0000000..9b48325
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/exclude.c
+ *
+ * Created by Li Guifu <blucer.lee@foxmail.com>
+ */
+#include <string.h>
+#include <stdlib.h>
+#include "erofs/err.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static LIST_HEAD(exclude_head);
+static unsigned int rpathlen;		/* root directory prefix length */
+
+void erofs_exclude_set_root(const char *rootdir)
+{
+	rpathlen = strlen(rootdir);
+}
+
+static struct erofs_exclude_rule *erofs_insert_exclude(const char *s)
+{
+	struct erofs_exclude_rule *e;
+
+	e = malloc(sizeof(*e));
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+
+	/* exact match */
+	e->pattern = strdup(s);
+	if (!e->pattern) {
+		free(e);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	list_add_tail(&e->list, &exclude_head);
+	erofs_info("exclude path %s inserted", e->pattern);
+	return e;
+}
+
+void erofs_cleanup_exclude_rules(void)
+{
+	struct erofs_exclude_rule *e, *n;
+
+	list_for_each_entry_safe(e, n, &exclude_head, list) {
+		list_del(&e->list);
+		free(e->pattern);
+		free(e);
+	}
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+	struct erofs_exclude_rule *e = erofs_insert_exclude(args);
+
+	if (IS_ERR(e)) {
+		erofs_cleanup_exclude_rules();
+		return PTR_ERR(e);
+	}
+	return 0;
+}
+
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+						 const char *name)
+{
+	char buf[PATH_MAX];
+	const char *s;
+	struct erofs_exclude_rule *e;
+
+	if (!dir) {
+		/* no prefix */
+		s = name;
+	} else {
+		sprintf(buf, "%s/%s", dir, name);
+		s = buf;
+	}
+
+	s += rpathlen;
+	while (*s == '/')
+		s++;
+
+	list_for_each_entry(e, &exclude_head, list) {
+		if (!strcmp(e->pattern, s))
+			return e;
+	}
+	return NULL;
+}
+
diff --git a/lib/inode.c b/lib/inode.c
index bd0652b..7114023 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -877,6 +878,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
 
+		/* skip if it's a exclude file */
+		if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
+			continue;
+
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index d6bf828..aa927a9 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
 Set all files to the given UNIX timestamp. Reproducible builds requires setting
 all to a specific one.
 .TP
+.BI "\-\-exclude-path=" path
+Ignore file that matches the exact literal path.
+You may give multiple `--exclude-path' options.
+.TP
 .B \-\-help
 Display this help and exit.
 .SH AUTHOR
diff --git a/mkfs/main.c b/mkfs/main.c
index 817a6c1..d913b5d 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 2},
 	{0, 0, 0, 0},
 };
 
@@ -50,12 +52,13 @@ static void usage(void)
 {
 	fputs("usage: [options] FILE DIRECTORY\n\n"
 	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
-	      " -zX[,Y]   X=compressor (Y=compression level, optional)\n"
-	      " -d#       set output message level to # (maximum 9)\n"
-	      " -x#       set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
-	      " -EX[,...] X=extended options\n"
-	      " -T#       set a fixed UNIX timestamp # to all files\n"
-	      " --help    display this help and exit\n"
+	      " -zX[,Y]          X=compressor (Y=compression level, optional)\n"
+	      " -d#              set output message level to # (maximum 9)\n"
+	      " -x#              set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
+	      " -EX[,...]        X=extended options\n"
+	      " -T#              set a fixed UNIX timestamp # to all files\n"
+	      " --exclude-path=X avoid including file X (X = exact literal path)\n"
+	      " --help           display this help and exit\n"
 	      "\nAvailable compressors are: ", stderr);
 	print_available_compressors(stderr, ", ");
 }
@@ -178,6 +181,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 2:
+			opt = erofs_parse_exclude_path(optarg);
+			if (opt) {
+				erofs_err("failed to parse exclude path: %s",
+					  erofs_strerror(opt));
+				return opt;
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -372,6 +384,7 @@ int main(int argc, char **argv)
 	}
 
 	erofs_show_config();
+	erofs_exclude_set_root(cfg.c_src_path);
 
 	sb_bh = erofs_buffer_init();
 	if (IS_ERR(sb_bh)) {
@@ -428,6 +441,7 @@ int main(int argc, char **argv)
 exit:
 	z_erofs_compress_exit();
 	dev_close();
+	erofs_cleanup_exclude_rules();
 	erofs_exit_configure();
 
 	if (err) {
-- 
2.20.1



^ permalink raw reply related	[flat|nested] 4+ messages in thread

* Re: [PATCH v2] erofs-utils: introduce exclude dirs and files
       [not found]   ` <108bd2aa-182f-c5d3-53eb-54ad0e313b76@foxmail.com>
@ 2020-02-10 16:34     ` Gao Xiang via Linux-erofs
  0 siblings, 0 replies; 4+ messages in thread
From: Gao Xiang via Linux-erofs @ 2020-02-10 16:34 UTC (permalink / raw)
  To: Li Guifu; +Cc: linux-erofs

On Mon, Feb 10, 2020 at 09:41:04PM +0800, Li Guifu wrote:
> Gao Xiang,
>  It would make a long string match on each path sprintfed from a dir.

It seems that your @foxmail.com email is still broken, because I haven't
seen this email on mailing list till now. I'd suggest you could fix
your email first.

> The parent's path don'y need to compare every times,just loop the sub dirs.

It's a very simple approach to implement this feature (exact match) as
the first step. Because for most paths without valid prefix, it will
just bail out on the first char and it will have no impact if we don't
use this feature.

Laterly, I'd suggest some sorted string array to solve what you
mentioned. (You could improve it in the following way.)

Take an example, we could keep such sorted string array in memory.

  0 A/B
  1 A/B/C
  2 A/B/D
  3 A/B/E/F
  4 A/G
  5 H

For each access, we can binary search and narrow down the range dynamicly.
e.g. if we open A dir, the search range will be from 0 to 4. and if we
open A/B dir, the search range will then be narrowed down to 0~3.

>  It also would better to use relate path on image path not the absolute
> path in the exclude path.

No, I think it works on relative path. You could test it out first.

Thanks,
Gao Xiang

> 
>   What your's opinion ?
> 
> On 2020/2/5 1:02, Gao Xiang via Linux-erofs wrote:
> > Hi Guifu,
> > 
> > I cleanup the whole patch and get rid of all dedupe check in order
> > to make it as simple as possible since I think not too many exclude
> > files in general.
> > 
> > Please take some time helping testing the following patch and
> > hope to get your feedback.
> > 
> > Thanks,
> > Gao Xiang
> > 
> > 
> > 
> >>From 600fc166cdaaa0e458181729245ce11affa83ac5 Mon Sep 17 00:00:00 2001
> > From: Li Guifu <blucer.lee@foxmail.com>
> > Date: Mon, 3 Feb 2020 23:38:11 +0800
> > Subject: [PATCH v3] erofs-utils: introduce exclude dirs and files
> > 
> > Add excluded file feature "--exclude-path=", which can be used
> > to build EROFS image without some user specific files or dirs.
> > 
> > Note that you may give multiple `--exclude-path' options.
> > 
> > Signed-off-by: Li Guifu <blucer.lee@foxmail.com>
> > Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
> > ---
> > changes since v2:
> >  - cleanup the whole implementation;
> >  - complete usage message;
> >  - complete manual page.
> > 
> >  include/erofs/exclude.h | 23 +++++++++++
> >  lib/Makefile.am         |  2 +-
> >  lib/exclude.c           | 89 +++++++++++++++++++++++++++++++++++++++++
> >  lib/inode.c             |  5 +++
> >  man/mkfs.erofs.1        |  4 ++
> >  mkfs/main.c             | 26 +++++++++---
> >  6 files changed, 142 insertions(+), 7 deletions(-)
> >  create mode 100644 include/erofs/exclude.h
> >  create mode 100644 lib/exclude.c
> > 
> > diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
> > new file mode 100644
> > index 0000000..580fefe
> > --- /dev/null
> > +++ b/include/erofs/exclude.h
> > @@ -0,0 +1,23 @@
> > +/* SPDX-License-Identifier: GPL-2.0+ */
> > +/*
> > + * erofs-utils/include/erofs/exclude.h
> > + *
> > + * Created by Li Guifu <blucer.lee@foxmail.com>
> > + */
> > +#ifndef __EROFS_EXCLUDE_H
> > +#define __EROFS_EXCLUDE_H
> > +
> > +struct erofs_exclude_rule {
> > +	struct list_head list;
> > +
> > +	char *pattern;
> > +};
> > +
> > +void erofs_exclude_set_root(const char *rootdir);
> > +void erofs_cleanup_exclude_rules(void);
> > +
> > +int erofs_parse_exclude_path(const char *args);
> > +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> > +						 const char *name);
> > +#endif
> > +
> > diff --git a/lib/Makefile.am b/lib/Makefile.am
> > index 1ff81f9..e4b51e6 100644
> > --- a/lib/Makefile.am
> > +++ b/lib/Makefile.am
> > @@ -3,7 +3,7 @@
> >  
> >  noinst_LTLIBRARIES = liberofs.la
> >  liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
> > -		      compress.c compressor.c
> > +		      compress.c compressor.c exclude.c
> >  liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
> >  if ENABLE_LZ4
> >  liberofs_la_CFLAGS += ${LZ4_CFLAGS}
> > diff --git a/lib/exclude.c b/lib/exclude.c
> > new file mode 100644
> > index 0000000..9b48325
> > --- /dev/null
> > +++ b/lib/exclude.c
> > @@ -0,0 +1,89 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/lib/exclude.c
> > + *
> > + * Created by Li Guifu <blucer.lee@foxmail.com>
> > + */
> > +#include <string.h>
> > +#include <stdlib.h>
> > +#include "erofs/err.h"
> > +#include "erofs/list.h"
> > +#include "erofs/print.h"
> > +#include "erofs/exclude.h"
> > +
> > +static LIST_HEAD(exclude_head);
> > +static unsigned int rpathlen;		/* root directory prefix length */
> > +
> > +void erofs_exclude_set_root(const char *rootdir)
> > +{
> > +	rpathlen = strlen(rootdir);
> > +}
> > +
> > +static struct erofs_exclude_rule *erofs_insert_exclude(const char *s)
> > +{
> > +	struct erofs_exclude_rule *e;
> > +
> > +	e = malloc(sizeof(*e));
> > +	if (!e)
> > +		return ERR_PTR(-ENOMEM);
> > +
> > +	/* exact match */
> > +	e->pattern = strdup(s);
> > +	if (!e->pattern) {
> > +		free(e);
> > +		return ERR_PTR(-ENOMEM);
> > +	}
> > +
> > +	list_add_tail(&e->list, &exclude_head);
> > +	erofs_info("exclude path %s inserted", e->pattern);
> > +	return e;
> > +}
> > +
> > +void erofs_cleanup_exclude_rules(void)
> > +{
> > +	struct erofs_exclude_rule *e, *n;
> > +
> > +	list_for_each_entry_safe(e, n, &exclude_head, list) {
> > +		list_del(&e->list);
> > +		free(e->pattern);
> > +		free(e);
> > +	}
> > +}
> > +
> > +int erofs_parse_exclude_path(const char *args)
> > +{
> > +	struct erofs_exclude_rule *e = erofs_insert_exclude(args);
> > +
> > +	if (IS_ERR(e)) {
> > +		erofs_cleanup_exclude_rules();
> > +		return PTR_ERR(e);
> > +	}
> > +	return 0;
> > +}
> > +
> > +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> > +						 const char *name)
> > +{
> > +	char buf[PATH_MAX];
> > +	const char *s;
> > +	struct erofs_exclude_rule *e;
> > +
> > +	if (!dir) {
> > +		/* no prefix */
> > +		s = name;
> > +	} else {
> > +		sprintf(buf, "%s/%s", dir, name);
> > +		s = buf;
> > +	}
> > +
> > +	s += rpathlen;
> > +	while (*s == '/')
> > +		s++;
> > +
> > +	list_for_each_entry(e, &exclude_head, list) {
> > +		if (!strcmp(e->pattern, s))
> > +			return e;
> > +	}
> > +	return NULL;
> > +}
> > +
> > diff --git a/lib/inode.c b/lib/inode.c
> > index bd0652b..7114023 100644
> > --- a/lib/inode.c
> > +++ b/lib/inode.c
> > @@ -20,6 +20,7 @@
> >  #include "erofs/io.h"
> >  #include "erofs/compress.h"
> >  #include "erofs/xattr.h"
> > +#include "erofs/exclude.h"
> >  
> >  struct erofs_sb_info sbi;
> >  
> > @@ -877,6 +878,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
> >  		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
> >  			continue;
> >  
> > +		/* skip if it's a exclude file */
> > +		if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
> > +			continue;
> > +
> >  		d = erofs_d_alloc(dir, dp->d_name);
> >  		if (IS_ERR(d)) {
> >  			ret = PTR_ERR(d);
> > diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> > index d6bf828..aa927a9 100644
> > --- a/man/mkfs.erofs.1
> > +++ b/man/mkfs.erofs.1
> > @@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
> >  Set all files to the given UNIX timestamp. Reproducible builds requires setting
> >  all to a specific one.
> >  .TP
> > +.BI "\-\-exclude-path=" path
> > +Ignore file that matches the exact literal path.
> > +You may give multiple `--exclude-path' options.
> > +.TP
> >  .B \-\-help
> >  Display this help and exit.
> >  .SH AUTHOR
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index 817a6c1..d913b5d 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -21,6 +21,7 @@
> >  #include "erofs/io.h"
> >  #include "erofs/compress.h"
> >  #include "erofs/xattr.h"
> > +#include "erofs/exclude.h"
> >  
> >  #ifdef HAVE_LIBUUID
> >  #include <uuid/uuid.h>
> > @@ -30,6 +31,7 @@
> >  
> >  static struct option long_options[] = {
> >  	{"help", no_argument, 0, 1},
> > +	{"exclude-path", required_argument, NULL, 2},
> >  	{0, 0, 0, 0},
> >  };
> >  
> > @@ -50,12 +52,13 @@ static void usage(void)
> >  {
> >  	fputs("usage: [options] FILE DIRECTORY\n\n"
> >  	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
> > -	      " -zX[,Y]   X=compressor (Y=compression level, optional)\n"
> > -	      " -d#       set output message level to # (maximum 9)\n"
> > -	      " -x#       set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> > -	      " -EX[,...] X=extended options\n"
> > -	      " -T#       set a fixed UNIX timestamp # to all files\n"
> > -	      " --help    display this help and exit\n"
> > +	      " -zX[,Y]          X=compressor (Y=compression level, optional)\n"
> > +	      " -d#              set output message level to # (maximum 9)\n"
> > +	      " -x#              set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> > +	      " -EX[,...]        X=extended options\n"
> > +	      " -T#              set a fixed UNIX timestamp # to all files\n"
> > +	      " --exclude-path=X avoid including file X (X = exact literal path)\n"
> > +	      " --help           display this help and exit\n"
> >  	      "\nAvailable compressors are: ", stderr);
> >  	print_available_compressors(stderr, ", ");
> >  }
> > @@ -178,6 +181,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> >  			}
> >  			break;
> >  
> > +		case 2:
> > +			opt = erofs_parse_exclude_path(optarg);
> > +			if (opt) {
> > +				erofs_err("failed to parse exclude path: %s",
> > +					  erofs_strerror(opt));
> > +				return opt;
> > +			}
> > +			break;
> > +
> >  		case 1:
> >  			usage();
> >  			exit(0);
> > @@ -372,6 +384,7 @@ int main(int argc, char **argv)
> >  	}
> >  
> >  	erofs_show_config();
> > +	erofs_exclude_set_root(cfg.c_src_path);
> >  
> >  	sb_bh = erofs_buffer_init();
> >  	if (IS_ERR(sb_bh)) {
> > @@ -428,6 +441,7 @@ int main(int argc, char **argv)
> >  exit:
> >  	z_erofs_compress_exit();
> >  	dev_close();
> > +	erofs_cleanup_exclude_rules();
> >  	erofs_exit_configure();
> >  
> >  	if (err) {
> > 

^ permalink raw reply	[flat|nested] 4+ messages in thread

* Re: Re: [PATCH v2] erofs-utils: introduce exclude dirs and files
  2020-02-04 17:02 ` Gao Xiang via Linux-erofs
       [not found]   ` <108bd2aa-182f-c5d3-53eb-54ad0e313b76@foxmail.com>
@ 2020-02-16  5:27   ` Li GuiFu via Linux-erofs
  1 sibling, 0 replies; 4+ messages in thread
From: Li GuiFu via Linux-erofs @ 2020-02-16  5:27 UTC (permalink / raw)
  To: Gao Xiang; +Cc: linux-erofs

Hi
My e-mail of foxmail was broken in the mainline, use aliyun instead.

On 2020/2/5 1:02, Gao Xiang via Linux-erofs wrote:
> Hi Guifu,
> 
> I cleanup the whole patch and get rid of all dedupe check in order
> to make it as simple as possible since I think not too many exclude
> files in general.
> 
> Please take some time helping testing the following patch and
> hope to get your feedback.
> 
> Thanks,
> Gao Xiang
> 
> 
> 
>>From 600fc166cdaaa0e458181729245ce11affa83ac5 Mon Sep 17 00:00:00 2001
> From: Li Guifu <blucer.lee@foxmail.com>
> Date: Mon, 3 Feb 2020 23:38:11 +0800
> Subject: [PATCH v3] erofs-utils: introduce exclude dirs and files
> 
> Add excluded file feature "--exclude-path=", which can be used
> to build EROFS image without some user specific files or dirs.
> 
> Note that you may give multiple `--exclude-path' options.
> 
> Signed-off-by: Li Guifu <blucer.lee@foxmail.com>
> Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
> ---
> changes since v2:
>  - cleanup the whole implementation;
>  - complete usage message;
>  - complete manual page.
> 
>  include/erofs/exclude.h | 23 +++++++++++
>  lib/Makefile.am         |  2 +-
>  lib/exclude.c           | 89 +++++++++++++++++++++++++++++++++++++++++
>  lib/inode.c             |  5 +++
>  man/mkfs.erofs.1        |  4 ++
>  mkfs/main.c             | 26 +++++++++---
>  6 files changed, 142 insertions(+), 7 deletions(-)
>  create mode 100644 include/erofs/exclude.h
>  create mode 100644 lib/exclude.c
> 
> diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
> new file mode 100644
> index 0000000..580fefe
> --- /dev/null
> +++ b/include/erofs/exclude.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0+ */
> +/*
> + * erofs-utils/include/erofs/exclude.h
> + *
> + * Created by Li Guifu <blucer.lee@foxmail.com>
> + */
> +#ifndef __EROFS_EXCLUDE_H
> +#define __EROFS_EXCLUDE_H
> +
> +struct erofs_exclude_rule {
> +	struct list_head list;
> +
> +	char *pattern;
> +};
> +
> +void erofs_exclude_set_root(const char *rootdir);
> +void erofs_cleanup_exclude_rules(void);
> +
> +int erofs_parse_exclude_path(const char *args);
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> +						 const char *name);
> +#endif
> +
> diff --git a/lib/Makefile.am b/lib/Makefile.am
> index 1ff81f9..e4b51e6 100644
> --- a/lib/Makefile.am
> +++ b/lib/Makefile.am
> @@ -3,7 +3,7 @@
>  
>  noinst_LTLIBRARIES = liberofs.la
>  liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
> -		      compress.c compressor.c
> +		      compress.c compressor.c exclude.c
>  liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
>  if ENABLE_LZ4
>  liberofs_la_CFLAGS += ${LZ4_CFLAGS}
> diff --git a/lib/exclude.c b/lib/exclude.c
> new file mode 100644
> index 0000000..9b48325
> --- /dev/null
> +++ b/lib/exclude.c
> @@ -0,0 +1,89 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/exclude.c
> + *
> + * Created by Li Guifu <blucer.lee@foxmail.com>
> + */
> +#include <string.h>
> +#include <stdlib.h>
> +#include "erofs/err.h"
> +#include "erofs/list.h"
> +#include "erofs/print.h"
> +#include "erofs/exclude.h"
> +
> +static LIST_HEAD(exclude_head);
> +static unsigned int rpathlen;		/* root directory prefix length */
> +
> +void erofs_exclude_set_root(const char *rootdir)
> +{
> +	rpathlen = strlen(rootdir);
> +}
> +
> +static struct erofs_exclude_rule *erofs_insert_exclude(const char *s)
> +{
> +	struct erofs_exclude_rule *e;
> +
> +	e = malloc(sizeof(*e));
> +	if (!e)
> +		return ERR_PTR(-ENOMEM);
> +
> +	/* exact match */
> +	e->pattern = strdup(s);
> +	if (!e->pattern) {
> +		free(e);
> +		return ERR_PTR(-ENOMEM);
> +	}
> +
> +	list_add_tail(&e->list, &exclude_head);
> +	erofs_info("exclude path %s inserted", e->pattern);
> +	return e;
> +}
> +
> +void erofs_cleanup_exclude_rules(void)
> +{
> +	struct erofs_exclude_rule *e, *n;
> +
> +	list_for_each_entry_safe(e, n, &exclude_head, list) {
> +		list_del(&e->list);
> +		free(e->pattern);
> +		free(e);
> +	}
> +}
> +
> +int erofs_parse_exclude_path(const char *args)
> +{
> +	struct erofs_exclude_rule *e = erofs_insert_exclude(args);
> +
> +	if (IS_ERR(e)) {
> +		erofs_cleanup_exclude_rules();
> +		return PTR_ERR(e);
> +	}
> +	return 0;
> +}
> +
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> +						 const char *name)
> +{
> +	char buf[PATH_MAX];
> +	const char *s;
> +	struct erofs_exclude_rule *e;
> +
> +	if (!dir) {
> +		/* no prefix */
> +		s = name;
> +	} else {
> +		sprintf(buf, "%s/%s", dir, name);
> +		s = buf;
> +	}
> +
> +	s += rpathlen;
> +	while (*s == '/')
> +		s++;
> +
> +	list_for_each_entry(e, &exclude_head, list) {
> +		if (!strcmp(e->pattern, s))
> +			return e;
> +	}
> +	return NULL;
> +}
> +
> diff --git a/lib/inode.c b/lib/inode.c
> index bd0652b..7114023 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -20,6 +20,7 @@
>  #include "erofs/io.h"
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
> +#include "erofs/exclude.h"
>  
>  struct erofs_sb_info sbi;
>  
> @@ -877,6 +878,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
>  		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
>  			continue;
>  
> +		/* skip if it's a exclude file */
> +		if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
> +			continue;
> +
>  		d = erofs_d_alloc(dir, dp->d_name);
>  		if (IS_ERR(d)) {
>  			ret = PTR_ERR(d);
> diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> index d6bf828..aa927a9 100644
> --- a/man/mkfs.erofs.1
> +++ b/man/mkfs.erofs.1
> @@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
>  Set all files to the given UNIX timestamp. Reproducible builds requires setting
>  all to a specific one.
>  .TP
> +.BI "\-\-exclude-path=" path
> +Ignore file that matches the exact literal path.
> +You may give multiple `--exclude-path' options.
> +.TP
>  .B \-\-help
>  Display this help and exit.
>  .SH AUTHOR
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 817a6c1..d913b5d 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -21,6 +21,7 @@
>  #include "erofs/io.h"
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
> +#include "erofs/exclude.h"
>  
>  #ifdef HAVE_LIBUUID
>  #include <uuid/uuid.h>
> @@ -30,6 +31,7 @@
>  
>  static struct option long_options[] = {
>  	{"help", no_argument, 0, 1},
> +	{"exclude-path", required_argument, NULL, 2},
>  	{0, 0, 0, 0},
>  };
>  
> @@ -50,12 +52,13 @@ static void usage(void)
>  {
>  	fputs("usage: [options] FILE DIRECTORY\n\n"
>  	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
> -	      " -zX[,Y]   X=compressor (Y=compression level, optional)\n"
> -	      " -d#       set output message level to # (maximum 9)\n"
> -	      " -x#       set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> -	      " -EX[,...] X=extended options\n"
> -	      " -T#       set a fixed UNIX timestamp # to all files\n"
> -	      " --help    display this help and exit\n"
> +	      " -zX[,Y]          X=compressor (Y=compression level, optional)\n"
> +	      " -d#              set output message level to # (maximum 9)\n"
> +	      " -x#              set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> +	      " -EX[,...]        X=extended options\n"
> +	      " -T#              set a fixed UNIX timestamp # to all files\n"
> +	      " --exclude-path=X avoid including file X (X = exact literal path)\n"
> +	      " --help           display this help and exit\n"
>  	      "\nAvailable compressors are: ", stderr);
>  	print_available_compressors(stderr, ", ");
>  }
> @@ -178,6 +181,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>  			}
>  			break;
>  
> +		case 2:
> +			opt = erofs_parse_exclude_path(optarg);
> +			if (opt) {
> +				erofs_err("failed to parse exclude path: %s",
> +					  erofs_strerror(opt));
> +				return opt;
> +			}
> +			break;
> +
>  		case 1:
>  			usage();
>  			exit(0);
> @@ -372,6 +384,7 @@ int main(int argc, char **argv)
>  	}
>  
>  	erofs_show_config();
> +	erofs_exclude_set_root(cfg.c_src_path);
>  
>  	sb_bh = erofs_buffer_init();
>  	if (IS_ERR(sb_bh)) {
> @@ -428,6 +441,7 @@ int main(int argc, char **argv)
>  exit:
>  	z_erofs_compress_exit();
>  	dev_close();
> +	erofs_cleanup_exclude_rules();
>  	erofs_exit_configure();
>  
>  	if (err) {
> 

^ permalink raw reply	[flat|nested] 4+ messages in thread

end of thread, other threads:[~2020-02-16  5:34 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-02-03 15:38 [PATCH v2] erofs-utils: introduce exclude dirs and files Li Guifu
2020-02-04 17:02 ` Gao Xiang via Linux-erofs
     [not found]   ` <108bd2aa-182f-c5d3-53eb-54ad0e313b76@foxmail.com>
2020-02-10 16:34     ` Gao Xiang via Linux-erofs
2020-02-16  5:27   ` Li GuiFu via Linux-erofs

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).