[PATCH 0/4] erofs-utils: code-refactoring for erofs compressor

[PATCH 0/4] erofs-utils: code-refactoring for erofs compressor

[Guo Xuenan via Linux-erofs]

simplify and unify erofs compressor code.

Test by following script:
```shell
#!/bin/bash

FSCK=`which fsck.erofs`
MKFS=`which mkfs.erofs`
DPFS=`which dump.erofs`
SRC_DIR=./src
IMGDIR=./image/

mkdir -p ${IMGDIR}
create_erofsimg()
{
	local img=$1
	echo "MKFS [mkfs.erofs $@]"
	$MKFS $@ ${SRC_DIR}
}

#test mkfs.erofs
create_erofsimg  ${IMGDIR}/erofs-raw32.img -E force-inode-compact || exit -11
create_erofsimg  ${IMGDIR}/erofs-raw64.img -E force-inode-extended || exit -12
create_erofsimg  ${IMGDIR}/erofs-lzma-4k.img -zlzma,9 || exit -13
create_erofsimg  ${IMGDIR}/erofs-lzma-bigpcluster-8k.img -zlzma,9 -C 8192 || exit -14
create_erofsimg  ${IMGDIR}/erofs-lzma-rand-bigpcluster-1M.img -zlzma,9 -C 1048576 --random-pclusterblks || exit -15
create_erofsimg  ${IMGDIR}/erofs-lz4-4k.img -zlz4hc,9 || exit -16
create_erofsimg  ${IMGDIR}/erofs-lz4-rand-bigpcluster-1M.img -zlz4hc,9 -C 1048576 --random-pclusterblks || exit -17
create_erofsimg  ${IMGDIR}/erofs-rand-alg-1M.img -zlz4:lzma,9:lz4hc,9 -C 1048576 --random-pclusterblks --random-algorithms || exit -18
create_erofsimg  ${IMGDIR}/erofs-rand-frag-1M.img -zlz4:lzma,9:lz4hc,9 -C 8192 -Efragments,8192 --random-pclusterblks --random-algorithms || exit -19
create_erofsimg  ${IMGDIR}/erofs-rand-frag-dedupe-1M.img -zlz4:lzma,9:lz4hc,9 -C 1048576 -Efragments,1048576 -Ededupe --random-algorithms || exit -20

#test {dump,fsck}.erofs
for img in `ls ${IMGDIR}/*.img`
do
	echo "FSCK_DUMP==================$img====================="
	mkdir -p ${img}_dir && $FSCK $img && $FSCK --extract=${img}_dir $img
	$DPFS $img
done
ls -F ${IMGDIR}/
```

Output:
--------------------------------------------
MKFS [mkfs.erofs ./image//erofs-raw32.img -E force-inode-compact]
mkfs.erofs 1.6-g566da452-dirty
Build completed.
MKFS [mkfs.erofs ./image//erofs-raw64.img -E force-inode-extended]
mkfs.erofs 1.6-g566da452-dirty
Build completed.
MKFS [mkfs.erofs ./image//erofs-lzma-4k.img -zlzma,9]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
MKFS [mkfs.erofs ./image//erofs-lzma-bigpcluster-8k.img -zlzma,9 -C 8192]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
MKFS [mkfs.erofs ./image//erofs-lzma-rand-bigpcluster-1M.img -zlzma,9 -C 1048576 --random-pclusterblks]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
MKFS [mkfs.erofs ./image//erofs-lz4-4k.img -zlz4hc,9]
mkfs.erofs 1.6-g566da452-dirty
Build completed.
MKFS [mkfs.erofs ./image//erofs-lz4-rand-bigpcluster-1M.img -zlz4hc,9 -C 1048576 --random-pclusterblks]
mkfs.erofs 1.6-g566da452-dirty
Build completed.
MKFS [mkfs.erofs ./image//erofs-rand-alg-1M.img -zlz4:lzma,9:lz4hc,9 -C 1048576 --random-pclusterblks --random-algorithms]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
MKFS [mkfs.erofs ./image//erofs-rand-frag-1M.img -zlz4:lzma,9:lz4hc,9 -C 8192 -Efragments,8192 --random-pclusterblks --random-algorithms]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: main() Line[790] EXPERIMENTAL compressed fragments feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
MKFS [mkfs.erofs ./image//erofs-rand-frag-dedupe-1M.img -zlz4:lzma,9:lz4hc,9 -C 1048576 -Efragments,1048576 -Ededupe --random-algorithms]
mkfs.erofs 1.6-g566da452-dirty
<W> erofs: main() Line[790] EXPERIMENTAL compressed fragments feature in use. Use at your own risk!
<W> erofs: main() Line[793] EXPERIMENTAL data deduplication feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1105] EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!
<W> erofs: z_erofs_compress_init() Line[1106] Note that it may take more time since the compressor is still single-threaded for now.
Build completed.
FSCK_DUMP==================./image//erofs-lz4-4k.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          36
Filesystem lz4_max_distance:                  65535
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:51 2023
Filesystem features:                          sb_csum mtime 0padding 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-lz4-rand-bigpcluster-1M.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lz4, lz4hc
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:51 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-lzma-4k.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:50 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-lzma-bigpcluster-8k.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:50 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-lzma-rand-bigpcluster-1M.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:50 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-rand-alg-1M.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            6
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lz4, lz4hc, lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:51 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-rand-frag-1M.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            2
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem packed nid:                        71
Filesystem compr_algs:                        lz4, lz4hc, lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       8
Filesystem created:                           Thu Jun 15 18:11:51 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster fragments dedupe 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-rand-frag-dedupe-1M.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            2
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem packed nid:                        71
Filesystem compr_algs:                        lz4, lz4hc, lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       8
Filesystem created:                           Thu Jun 15 18:11:51 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster fragments dedupe 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-raw32.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            17
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          36
Filesystem lz4_max_distance:                  65535
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:50 2023
Filesystem features:                          sb_csum mtime 
Filesystem UUID:                              not available
FSCK_DUMP==================./image//erofs-raw64.img=====================
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            17
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          36
Filesystem lz4_max_distance:                  65535
Filesystem sb_extslots:                       0
Filesystem inode count:                       7
Filesystem created:                           Thu Jun 15 18:11:50 2023
Filesystem features:                          sb_csum mtime 
Filesystem UUID:                              not available
erofs-lz4-4k.img			erofs-lzma-4k.img_dir/			 erofs-rand-alg-1M.img		erofs-rand-frag-dedupe-1M.img_dir/
erofs-lz4-4k.img_dir/			erofs-lzma-bigpcluster-8k.img		 erofs-rand-alg-1M.img_dir/	erofs-raw32.img
erofs-lz4-rand-bigpcluster-1M.img	erofs-lzma-bigpcluster-8k.img_dir/	 erofs-rand-frag-1M.img		erofs-raw32.img_dir/
erofs-lz4-rand-bigpcluster-1M.img_dir/	erofs-lzma-rand-bigpcluster-1M.img	 erofs-rand-frag-1M.img_dir/	erofs-raw64.img
erofs-lzma-4k.img			erofs-lzma-rand-bigpcluster-1M.img_dir/  erofs-rand-frag-dedupe-1M.img	erofs-raw64.img_dir/

Guo Xuenan (4):
  erofs-utils: lib: refactor erofs compressors init
  erofs-utils: lib: unify all identical compressor print function
  erofs-utils: simplify erofs compressor init and exit
  erofs-utils: dump: add some superblock fields

 dump/main.c              |  13 +++
 fsck/main.c              |  17 +---
 include/erofs/compress.h |   7 +-
 include/erofs/internal.h |   1 +
 lib/compress.c           |  87 ++++++++------------
 lib/compressor.c         | 173 ++++++++++++++++++++++++++++-----------
 lib/compressor.h         |  23 ++++--
 lib/compressor_liblzma.c |   4 -
 lib/compressor_lz4.c     |   2 -
 lib/compressor_lz4hc.c   |   3 -
 lib/super.c              |   5 ++
 mkfs/main.c              |  18 +---
 12 files changed, 207 insertions(+), 146 deletions(-)

-- 
2.31.1

[PATCH v2 1/4] erofs-utils: lib: refactor erofs compressors init

[Guo Xuenan via Linux-erofs]

using struct erofs_compressors_cfg for erofs compressor
global configuration.

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
---
 lib/compressor.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/compressor.h | 14 ++++++++++
 2 files changed, 82 insertions(+)

diff --git a/lib/compressor.c b/lib/compressor.c
index 52eb761..88a2fb0 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -22,6 +22,74 @@ static const struct erofs_compressor *compressors[] = {
 #endif
 };
 
+/* for compressors type configuration */
+static struct erofs_supported_algothrim {
+	int algtype;
+	const char *name;
+} erofs_supported_algothrims[] = {
+	{ Z_EROFS_COMPRESSION_LZ4, "lz4"},
+	{ Z_EROFS_COMPRESSION_LZ4, "lz4hc"},
+	{ Z_EROFS_COMPRESSION_LZMA, "lzma"},
+};
+
+/* global compressors configuration */
+static struct erofs_compressors_cfg erofs_ccfg;
+
+static void erofs_init_compressor(struct compressor *compressor,
+	const struct erofs_compressor *alg)
+{
+
+	compressor->handle.alg = alg;
+
+	/* should be written in "minimum compression ratio * 100" */
+	compressor->handle.compress_threshold = 100;
+
+	/* optimize for 4k size page */
+	compressor->handle.destsize_alignsize = erofs_blksiz();
+	compressor->handle.destsize_redzone_begin = erofs_blksiz() - 16;
+	compressor->handle.destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
+
+	if (alg && alg->init)
+		alg->init(&compressor->handle);
+}
+
+static void erofs_compressor_register(const char *name, const struct erofs_compressor *alg)
+{
+	int i;
+
+	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; i++) {
+		if (!strcmp(erofs_ccfg.compressors[i].name, name)) {
+			erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
+			return;
+		}
+	}
+
+	erofs_ccfg.compressors[i].name = name;
+	erofs_ccfg.compressors[i].algorithmtype = erofs_supported_algothrims[i].algtype;
+	erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
+	erofs_ccfg.erofs_ccfg_num = ++i;
+}
+
+void erofs_register_compressors(void)
+{
+	int i;
+
+	erofs_ccfg.erofs_ccfg_num = 0;
+	for (i = 0; i < ARRAY_SIZE(erofs_supported_algothrims); i++) {
+		erofs_compressor_register(erofs_supported_algothrims[i].name, NULL);
+	}
+
+#if LZ4_ENABLED
+	erofs_compressor_register("lz4", &erofs_compressor_lz4);
+#if LZ4HC_ENABLED
+	erofs_compressor_register("lz4hc", &erofs_compressor_lz4hc);
+#endif
+#endif
+#if HAVE_LIBLZMA
+	erofs_compressor_register("lzma", &erofs_compressor_lzma);
+#endif
+}
+
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize, bool inblocks)
diff --git a/lib/compressor.h b/lib/compressor.h
index cf063f1..1e760b6 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -8,6 +8,7 @@
 #define __EROFS_LIB_COMPRESSOR_H
 
 #include "erofs/defs.h"
+#include "erofs/config.h"
 
 struct erofs_compress;
 
@@ -40,6 +41,18 @@ struct erofs_compress {
 	void *private_data;
 };
 
+struct compressor {
+	const char *name;
+	struct erofs_compress handle;
+	unsigned int algorithmtype;
+	bool enable;
+};
+
+struct erofs_compressors_cfg {
+	struct compressor compressors[EROFS_MAX_COMPR_CFGS];
+	int erofs_ccfg_num;
+};
+
 /* list of compression algorithms */
 extern const struct erofs_compressor erofs_compressor_lz4;
 extern const struct erofs_compressor erofs_compressor_lz4hc;
@@ -52,5 +65,6 @@ int erofs_compress_destsize(const struct erofs_compress *c,
 int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
 int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
 int erofs_compressor_exit(struct erofs_compress *c);
+void erofs_register_compressors(void);
 
 #endif
-- 
2.31.1

[PATCH v2 2/4] erofs-utils: lib: unify all identical compressor print function

[Guo Xuenan via Linux-erofs]

{dump,fsck}.erofs use the same compressor print function,
available compressors means which algothrims are currently
supported by binary tools.
supported compressors including all algothrims that are ready
for your erofs binary tools.

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
---
 fsck/main.c              | 15 +-----------
 include/erofs/compress.h |  3 ++-
 lib/compressor.c         | 51 ++++++++++++++++++++++++++--------------
 mkfs/main.c              | 15 +-----------
 4 files changed, 38 insertions(+), 46 deletions(-)

diff --git a/fsck/main.c b/fsck/main.c
index f816bec..e559050 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -49,19 +49,6 @@ static struct option long_options[] = {
 	{0, 0, 0, 0},
 };
 
-static void print_available_decompressors(FILE *f, const char *delim)
-{
-	unsigned int i = 0;
-	const char *s;
-
-	while ((s = z_erofs_list_available_compressors(i)) != NULL) {
-		if (i++)
-			fputs(delim, f);
-		fputs(s, f);
-	}
-	fputc('\n', f);
-}
-
 static void usage(void)
 {
 	fputs("usage: [options] IMAGE\n\n"
@@ -84,7 +71,7 @@ static void usage(void)
 	      " --no-preserve-owner    extract as yourself\n"
 	      " --no-preserve-perms    apply user's umask when extracting permissions\n"
 	      "\nSupported algorithms are: ", stderr);
-	print_available_decompressors(stderr, ", ");
+	erofs_print_available_compressors(stderr);
 }
 
 static void erofsfsck_print_version(void)
diff --git a/include/erofs/compress.h b/include/erofs/compress.h
index 08af9e3..f1b9bbd 100644
--- a/include/erofs/compress.h
+++ b/include/erofs/compress.h
@@ -22,7 +22,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode, int fd);
 int z_erofs_compress_init(struct erofs_buffer_head *bh);
 int z_erofs_compress_exit(void);
 
-const char *z_erofs_list_available_compressors(unsigned int i);
+void erofs_print_available_compressors(FILE *f);
+void erofs_print_supported_compressors(FILE *f, unsigned int mask);
 
 static inline bool erofs_is_packed_inode(struct erofs_inode *inode)
 {
diff --git a/lib/compressor.c b/lib/compressor.c
index 88a2fb0..da8d1b9 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -10,18 +10,6 @@
 
 #define EROFS_CONFIG_COMPR_DEF_BOUNDARY		(128)
 
-static const struct erofs_compressor *compressors[] = {
-#if LZ4_ENABLED
-#if LZ4HC_ENABLED
-		&erofs_compressor_lz4hc,
-#endif
-		&erofs_compressor_lz4,
-#endif
-#if HAVE_LIBLZMA
-		&erofs_compressor_lzma,
-#endif
-};
-
 /* for compressors type configuration */
 static struct erofs_supported_algothrim {
 	int algtype;
@@ -119,9 +107,38 @@ int erofs_compress_destsize(const struct erofs_compress *c,
 	return ret;
 }
 
-const char *z_erofs_list_available_compressors(unsigned int i)
+void erofs_print_supported_compressors(FILE *f, unsigned int mask)
 {
-	return i >= ARRAY_SIZE(compressors) ? NULL : compressors[i]->name;
+	unsigned int i = 0;
+	int comma = false;
+	const char *s;
+
+	while (i < erofs_ccfg.erofs_ccfg_num) {
+		s = erofs_ccfg.compressors[i].name;
+		if (s && (mask & (1 << erofs_ccfg.compressors[i++].algorithmtype))) {
+			if (comma)
+				fputs(", ", f);
+			else
+				comma = true;
+			fputs(s, f);
+		}
+	}
+	fputc('\n', f);
+}
+
+void erofs_print_available_compressors(FILE *f)
+{
+	unsigned int i = 0;
+	const char *s;
+
+	while (i < erofs_ccfg.erofs_ccfg_num &&
+			erofs_ccfg.compressors[i].handle.alg &&
+			(s = erofs_ccfg.compressors[i].name)) {
+		if (i++)
+			fputs(", ", f);
+		fputs(s, f);
+	}
+	fputc('\n', f);
 }
 
 int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
@@ -154,11 +171,11 @@ int erofs_compressor_init(struct erofs_compress *c, char *alg_name)
 	}
 
 	ret = -EINVAL;
-	for (i = 0; i < ARRAY_SIZE(compressors); ++i) {
-		if (alg_name && strcmp(alg_name, compressors[i]->name))
+	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; ++i) {
+		if (alg_name && strcmp(alg_name, erofs_ccfg.compressors[i].name))
 			continue;
 
-		ret = compressors[i]->init(c);
+		ret = erofs_ccfg.compressors[i].handle.alg->init(c);
 		if (!ret) {
 			DBG_BUGON(!c->alg);
 			return 0;
diff --git a/mkfs/main.c b/mkfs/main.c
index ac208e5..9433a75 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -66,19 +66,6 @@ static struct option long_options[] = {
 	{0, 0, 0, 0},
 };
 
-static void print_available_compressors(FILE *f, const char *delim)
-{
-	unsigned int i = 0;
-	const char *s;
-
-	while ((s = z_erofs_list_available_compressors(i)) != NULL) {
-		if (i++)
-			fputs(delim, f);
-		fputs(s, f);
-	}
-	fputc('\n', f);
-}
-
 static void usage(void)
 {
 	fputs("usage: [options] FILE DIRECTORY\n\n"
@@ -126,7 +113,7 @@ static void usage(void)
 	      " --block-list-file=X   X=block_list file\n"
 #endif
 	      "\nAvailable compressors are: ", stderr);
-	print_available_compressors(stderr, ", ");
+	erofs_print_available_compressors(stderr);
 }
 
 static unsigned int pclustersize_packed, pclustersize_max;
-- 
2.31.1

[PATCH v2 3/4] erofs-utils: simplify erofs compressor init and exit

[Guo Xuenan via Linux-erofs]

since the compressor is independent in erofs-utils, rework the
compessor code to make adding a new algothrim easier.

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
---
 dump/main.c              |  2 +
 fsck/main.c              |  2 +
 include/erofs/compress.h |  4 +-
 lib/compress.c           | 87 +++++++++++++++++-----------------------
 lib/compressor.c         | 76 +++++++++++++++++------------------
 lib/compressor.h         | 13 ++----
 lib/compressor_liblzma.c |  4 --
 lib/compressor_lz4.c     |  2 -
 lib/compressor_lz4hc.c   |  3 --
 mkfs/main.c              |  3 +-
 10 files changed, 83 insertions(+), 113 deletions(-)

diff --git a/dump/main.c b/dump/main.c
index ae1ffa0..758e14a 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -633,6 +633,7 @@ int main(int argc, char **argv)
 	int err;
 
 	erofs_init_configure();
+	erofs_register_compressors();
 	err = erofsdump_parse_options_cfg(argc, argv);
 	if (err) {
 		if (err == -EINVAL)
@@ -677,5 +678,6 @@ exit_dev_close:
 exit:
 	blob_closeall();
 	erofs_exit_configure();
+	erofs_unregister_compressors();
 	return err;
 }
diff --git a/fsck/main.c b/fsck/main.c
index e559050..03f26a1 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -790,6 +790,7 @@ int main(int argc, char *argv[])
 	int err;
 
 	erofs_init_configure();
+	erofs_register_compressors();
 
 	fsckcfg.physical_blocks = 0;
 	fsckcfg.logical_blocks = 0;
@@ -870,6 +871,7 @@ exit_dev_close:
 exit:
 	blob_closeall();
 	erofs_exit_configure();
+	erofs_unregister_compressors();
 	return err ? 1 : 0;
 }
 
diff --git a/include/erofs/compress.h b/include/erofs/compress.h
index f1b9bbd..9e0f5ed 100644
--- a/include/erofs/compress.h
+++ b/include/erofs/compress.h
@@ -20,10 +20,10 @@ void z_erofs_drop_inline_pcluster(struct erofs_inode *inode);
 int erofs_write_compressed_file(struct erofs_inode *inode, int fd);
 
 int z_erofs_compress_init(struct erofs_buffer_head *bh);
-int z_erofs_compress_exit(void);
-
 void erofs_print_available_compressors(FILE *f);
 void erofs_print_supported_compressors(FILE *f, unsigned int mask);
+void erofs_register_compressors(void);
+void erofs_unregister_compressors(void);
 
 static inline bool erofs_is_packed_inode(struct erofs_inode *inode)
 {
diff --git a/lib/compress.c b/lib/compress.c
index 14d228f..60dc3e4 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -21,19 +21,12 @@
 #include "erofs/compress_hints.h"
 #include "erofs/fragments.h"
 
-/* compressing configuration specified by users */
-struct erofs_compress_cfg {
-	struct erofs_compress handle;
-	unsigned int algorithmtype;
-	bool enable;
-} erofs_ccfg[EROFS_MAX_COMPR_CFGS];
-
 struct z_erofs_vle_compress_ctx {
 	u8 queue[EROFS_CONFIG_COMPR_MAX_SZ * 2];
 	struct z_erofs_inmem_extent e;	/* (lookahead) extent */
 
 	struct erofs_inode *inode;
-	struct erofs_compress_cfg *ccfg;
+	struct compressor *compressor;
 
 	u8 *metacur;
 	unsigned int head, tail;
@@ -382,7 +375,7 @@ static int vle_compress_one(struct z_erofs_vle_compress_ctx *ctx)
 	static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_MAX_BLOCK_SIZE];
 	struct erofs_inode *inode = ctx->inode;
 	char *const dst = dstbuf + erofs_blksiz();
-	struct erofs_compress *const h = &ctx->ccfg->handle;
+	struct erofs_compress *const h = &ctx->compressor->handle;
 	unsigned int len = ctx->tail - ctx->head;
 	bool is_packed_inode = erofs_is_packed_inode(inode);
 	bool final = !ctx->remaining;
@@ -842,6 +835,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode, int fd)
 	static struct z_erofs_vle_compress_ctx ctx;
 	erofs_blk_t blkaddr, compressed_blocks;
 	unsigned int legacymetasize;
+	int ccfg_idx;
 	int ret;
 	u8 *compressmeta = malloc(vle_compressmeta_capacity(inode->i_size));
 
@@ -874,18 +868,18 @@ int erofs_write_compressed_file(struct erofs_inode *inode, int fd)
 	if (cfg.c_fragments && !cfg.c_dedupe)
 		inode->z_advise |= Z_EROFS_ADVISE_INTERLACED_PCLUSTER;
 
+	ccfg_idx = erofs_get_compressor_by_name(cfg.c_compr_alg[0]);
 #ifndef NDEBUG
 	if (cfg.c_random_algorithms) {
 		while (1) {
-			inode->z_algorithmtype[0] =
-				rand() % EROFS_MAX_COMPR_CFGS;
-			if (erofs_ccfg[inode->z_algorithmtype[0]].enable)
+			ccfg_idx = rand() % erofs_ccfg.erofs_ccfg_num;
+			if (erofs_ccfg.compressors[ccfg_idx].enable)
 				break;
 		}
 	}
 #endif
-	ctx.ccfg = &erofs_ccfg[inode->z_algorithmtype[0]];
-	inode->z_algorithmtype[0] = ctx.ccfg[0].algorithmtype;
+	ctx.compressor = &erofs_ccfg.compressors[ccfg_idx];
+	inode->z_algorithmtype[0] = ctx.compressor->algorithmtype;
 	inode->z_algorithmtype[1] = 0;
 
 	inode->idata_size = 0;
@@ -1020,15 +1014,6 @@ err_free_meta:
 	return ret;
 }
 
-static int erofs_get_compress_algorithm_id(const char *name)
-{
-	if (!strcmp(name, "lz4") || !strcmp(name, "lz4hc"))
-		return Z_EROFS_COMPRESSION_LZ4;
-	if (!strcmp(name, "lzma"))
-		return Z_EROFS_COMPRESSION_LZMA;
-	return -ENOTSUP;
-}
-
 int z_erofs_build_compr_cfgs(struct erofs_buffer_head *sb_bh)
 {
 	struct erofs_buffer_head *bh = sb_bh;
@@ -1085,29 +1070,42 @@ int z_erofs_build_compr_cfgs(struct erofs_buffer_head *sb_bh)
 
 int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
 {
-	int i, ret;
+	int i, k, ret;
 
+	/* enable all compressors for mkfs.erofs */
 	for (i = 0; cfg.c_compr_alg[i]; ++i) {
-		ret = erofs_compressor_init(&erofs_ccfg[i].handle,
-					     cfg.c_compr_alg[i]);
-		if (ret)
-			return ret;
+		struct compressor *compressor = NULL;
 
-		ret = erofs_compressor_setlevel(&erofs_ccfg[i].handle,
-						cfg.c_compr_level[i]);
-		if (ret)
-			return ret;
+		k = erofs_get_compressor_by_name(cfg.c_compr_alg[i]);
+		if (k < 0) {
+			erofs_warn("Cannot find valid compressor for %s!", cfg.c_compr_alg[i]);
+			continue;
+		}
 
-		ret = erofs_get_compress_algorithm_id(cfg.c_compr_alg[i]);
-		if (ret < 0)
+		if (!erofs_ccfg.compressors[k].handle.alg) {
+			erofs_err("compressor %s not available!", cfg.c_compr_alg[i]);
+			return -EINVAL;
+		}
+
+		compressor = &erofs_ccfg.compressors[k];
+		ret = erofs_compressor_setlevel(&compressor->handle, cfg.c_compr_level[i]);
+		if (ret) {
+			erofs_err("compressor %s invalid compress level %d!",
+				cfg.c_compr_alg[i], cfg.c_compr_level[i]);
 			return ret;
-		erofs_ccfg[i].algorithmtype = ret;
-		erofs_ccfg[i].enable = true;
-		sbi.available_compr_algs |= 1 << ret;
-		if (ret != Z_EROFS_COMPRESSION_LZ4)
+		}
+
+		erofs_ccfg.compressors[k].enable = true;
+		sbi.available_compr_algs |= 1 << erofs_ccfg.compressors[k].algorithmtype;
+		if (erofs_ccfg.compressors[k].algorithmtype != Z_EROFS_COMPRESSION_LZ4)
 			erofs_sb_set_compr_cfgs();
 	}
 
+	if (sbi.available_compr_algs & 1 << Z_EROFS_COMPRESSION_LZMA) {
+		erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
+		erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
+	}
+
 	/*
 	 * if primary algorithm is empty (e.g. compression off),
 	 * clear 0PADDING feature for old kernel compatibility.
@@ -1138,20 +1136,7 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
 	}
 
 	if (erofs_sb_has_compr_cfgs()) {
-		sbi.available_compr_algs |= 1 << ret;
 		return z_erofs_build_compr_cfgs(sb_bh);
 	}
 	return 0;
 }
-
-int z_erofs_compress_exit(void)
-{
-	int i, ret;
-
-	for (i = 0; cfg.c_compr_alg[i]; ++i) {
-		ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
-		if (ret)
-			return ret;
-	}
-	return 0;
-}
diff --git a/lib/compressor.c b/lib/compressor.c
index da8d1b9..c0e8c51 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -20,8 +20,13 @@ static struct erofs_supported_algothrim {
 	{ Z_EROFS_COMPRESSION_LZMA, "lzma"},
 };
 
+/* list of compression algorithms */
+extern const struct erofs_compressor erofs_compressor_lz4;
+extern const struct erofs_compressor erofs_compressor_lz4hc;
+extern const struct erofs_compressor erofs_compressor_lzma;
+
 /* global compressors configuration */
-static struct erofs_compressors_cfg erofs_ccfg;
+struct erofs_compressors_cfg erofs_ccfg;
 
 static void erofs_init_compressor(struct compressor *compressor,
 	const struct erofs_compressor *alg)
@@ -78,6 +83,36 @@ void erofs_register_compressors(void)
 #endif
 }
 
+int erofs_get_compressor_by_name(const char *name)
+{
+	int i;
+
+	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; i++) {
+		if (!strcmp(erofs_ccfg.compressors[i].name, name)) {
+			return i;
+		}
+	}
+	return -EINVAL;
+}
+
+static void erofs_compressor_unregister(struct erofs_compress *c)
+{
+	const struct erofs_compressor *alg = c->alg;
+
+	if (alg && alg->exit)
+		alg->exit(c);
+	c->alg = NULL;
+}
+
+void erofs_unregister_compressors(void)
+{
+	int i;
+
+	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; ++i) {
+		erofs_compressor_unregister(&erofs_ccfg.compressors[i].handle);
+	}
+}
+
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize, bool inblocks)
@@ -152,42 +187,3 @@ int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
 	c->compression_level = 0;
 	return 0;
 }
-
-int erofs_compressor_init(struct erofs_compress *c, char *alg_name)
-{
-	int ret, i;
-
-	/* should be written in "minimum compression ratio * 100" */
-	c->compress_threshold = 100;
-
-	/* optimize for 4k size page */
-	c->destsize_alignsize = erofs_blksiz();
-	c->destsize_redzone_begin = erofs_blksiz() - 16;
-	c->destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
-
-	if (!alg_name) {
-		c->alg = NULL;
-		return 0;
-	}
-
-	ret = -EINVAL;
-	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; ++i) {
-		if (alg_name && strcmp(alg_name, erofs_ccfg.compressors[i].name))
-			continue;
-
-		ret = erofs_ccfg.compressors[i].handle.alg->init(c);
-		if (!ret) {
-			DBG_BUGON(!c->alg);
-			return 0;
-		}
-	}
-	erofs_err("Cannot find a valid compressor %s", alg_name);
-	return ret;
-}
-
-int erofs_compressor_exit(struct erofs_compress *c)
-{
-	if (c->alg && c->alg->exit)
-		return c->alg->exit(c);
-	return 0;
-}
diff --git a/lib/compressor.h b/lib/compressor.h
index 1e760b6..b7d052c 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -13,8 +13,6 @@
 struct erofs_compress;
 
 struct erofs_compressor {
-	const char *name;
-
 	int default_level;
 	int best_level;
 
@@ -52,19 +50,14 @@ struct erofs_compressors_cfg {
 	struct compressor compressors[EROFS_MAX_COMPR_CFGS];
 	int erofs_ccfg_num;
 };
-
-/* list of compression algorithms */
-extern const struct erofs_compressor erofs_compressor_lz4;
-extern const struct erofs_compressor erofs_compressor_lz4hc;
-extern const struct erofs_compressor erofs_compressor_lzma;
+/* global compressors config */
+extern struct erofs_compressors_cfg erofs_ccfg;
 
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize, bool inblocks);
 
 int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
-int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
-int erofs_compressor_exit(struct erofs_compress *c);
-void erofs_register_compressors(void);
+int erofs_get_compressor_by_name(const char *name);
 
 #endif
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
index f274dce..5b4e83e 100644
--- a/lib/compressor_liblzma.c
+++ b/lib/compressor_liblzma.c
@@ -88,19 +88,15 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c)
 {
 	struct erofs_liblzma_context *ctx;
 
-	c->alg = &erofs_compressor_lzma;
 	ctx = malloc(sizeof(*ctx));
 	if (!ctx)
 		return -ENOMEM;
 	ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
 	c->private_data = ctx;
-	erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
-	erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
 	return 0;
 }
 
 const struct erofs_compressor erofs_compressor_lzma = {
-	.name = "lzma",
 	.default_level = LZMA_PRESET_DEFAULT,
 	.best_level = 109,
 	.init = erofs_compressor_liblzma_init,
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index b6f6e7e..1182869 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -32,13 +32,11 @@ static int compressor_lz4_exit(struct erofs_compress *c)
 
 static int compressor_lz4_init(struct erofs_compress *c)
 {
-	c->alg = &erofs_compressor_lz4;
 	sbi.lz4_max_distance = LZ4_DISTANCE_MAX;
 	return 0;
 }
 
 const struct erofs_compressor erofs_compressor_lz4 = {
-	.name = "lz4",
 	.default_level = 0,
 	.best_level = 0,
 	.init = compressor_lz4_init,
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index eec1c84..81b91d6 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -38,8 +38,6 @@ static int compressor_lz4hc_exit(struct erofs_compress *c)
 
 static int compressor_lz4hc_init(struct erofs_compress *c)
 {
-	c->alg = &erofs_compressor_lz4hc;
-
 	c->private_data = LZ4_createStreamHC();
 	if (!c->private_data)
 		return -ENOMEM;
@@ -60,7 +58,6 @@ static int compressor_lz4hc_setlevel(struct erofs_compress *c,
 }
 
 const struct erofs_compressor erofs_compressor_lz4hc = {
-	.name = "lz4hc",
 	.default_level = LZ4HC_CLEVEL_DEFAULT,
 	.best_level = LZ4HC_CLEVEL_MAX,
 	.init = compressor_lz4hc_init,
diff --git a/mkfs/main.c b/mkfs/main.c
index 9433a75..5f8c720 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -717,6 +717,7 @@ int main(int argc, char **argv)
 
 	erofs_init_configure();
 	erofs_mkfs_default_options();
+	erofs_register_compressors();
 
 	err = mkfs_parse_options_cfg(argc, argv);
 	erofs_show_progs(argc, argv);
@@ -916,7 +917,7 @@ int main(int argc, char **argv)
 	if (!err && erofs_sb_has_sb_chksum())
 		err = erofs_mkfs_superblock_csum_set();
 exit:
-	z_erofs_compress_exit();
+	erofs_unregister_compressors();
 	z_erofs_dedupe_exit();
 #ifdef WITH_ANDROID
 	erofs_droid_blocklist_fclose();
-- 
2.31.1

[PATCH v2 4/4] erofs-utils: dump: add some superblock fields

[Guo Xuenan via Linux-erofs]

dump.erofs show compression algorithms and sb_extslots,
and update feature information.

The proposed super block info is shown as below:
Filesystem magic number:                      0xE0F5E1E2
Filesystem blocks:                            4624
Filesystem inode metadata start block:        0
Filesystem shared xattr metadata start block: 0
Filesystem root nid:                          37
Filesystem compr_algs:                        lz4, lz4hc, lzma
Filesystem sb_extslots:                       0
Filesystem inode count:                       6131
Filesystem created:                           Wed Jun  7 17:15:44 2023
Filesystem features:                          sb_csum mtime 0padding compr_cfgs big_pcluster
Filesystem UUID:                              not available

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
---
 dump/main.c              | 11 +++++++++++
 include/erofs/internal.h |  1 +
 lib/super.c              |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/dump/main.c b/dump/main.c
index 758e14a..bb8d9ef 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -94,12 +94,14 @@ static struct erofsdump_feature feature_lists[] = {
 	{ true, EROFS_FEATURE_COMPAT_SB_CHKSUM, "sb_csum" },
 	{ true, EROFS_FEATURE_COMPAT_MTIME, "mtime" },
 	{ false, EROFS_FEATURE_INCOMPAT_ZERO_PADDING, "0padding" },
+	{ false, EROFS_FEATURE_INCOMPAT_COMPR_CFGS, "compr_cfgs" },
 	{ false, EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER, "big_pcluster" },
 	{ false, EROFS_FEATURE_INCOMPAT_CHUNKED_FILE, "chunked_file" },
 	{ false, EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, "device_table" },
 	{ false, EROFS_FEATURE_INCOMPAT_ZTAILPACKING, "ztailpacking" },
 	{ false, EROFS_FEATURE_INCOMPAT_FRAGMENTS, "fragments" },
 	{ false, EROFS_FEATURE_INCOMPAT_DEDUPE, "dedupe" },
+	{ false, EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES, "xattr_prefixes" },
 };
 
 static int erofsdump_readdir(struct erofs_dir_context *ctx);
@@ -609,6 +611,15 @@ static void erofsdump_show_superblock(void)
 	if (erofs_sb_has_fragments() && sbi.packed_nid > 0)
 		fprintf(stdout, "Filesystem packed nid:                        %llu\n",
 			sbi.packed_nid | 0ULL);
+	if (erofs_sb_has_compr_cfgs()) {
+		fprintf(stdout, "Filesystem compr_algs:                        ");
+		erofs_print_supported_compressors(stdout, sbi.available_compr_algs);
+	} else {
+		fprintf(stdout, "Filesystem lz4_max_distance:                  %u\n",
+			sbi.lz4_max_distance | 0U);
+	}
+	fprintf(stdout, "Filesystem sb_extslots:                       %u\n",
+			sbi.extslots | 0U);
 	fprintf(stdout, "Filesystem inode count:                       %llu\n",
 			sbi.inos | 0ULL);
 	fprintf(stdout, "Filesystem created:                           %s",
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index ab964d4..95e49aa 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -74,6 +74,7 @@ struct erofs_sb_info {
 	u32 feature_incompat;
 	u64 build_time;
 	u32 build_time_nsec;
+	u8 extslots;
 
 	unsigned char islotbits;
 	unsigned char blkszbits;
diff --git a/lib/super.c b/lib/super.c
index 820c883..102a527 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -105,11 +105,16 @@ int erofs_read_superblock(void)
 	sbi.packed_nid = le64_to_cpu(dsb->packed_nid);
 	sbi.inos = le64_to_cpu(dsb->inos);
 	sbi.checksum = le32_to_cpu(dsb->checksum);
+	sbi.extslots = dsb->sb_extslots;
 
 	sbi.build_time = le64_to_cpu(dsb->build_time);
 	sbi.build_time_nsec = le32_to_cpu(dsb->build_time_nsec);
 
 	memcpy(&sbi.uuid, dsb->uuid, sizeof(dsb->uuid));
+	if (erofs_sb_has_compr_cfgs())
+		sbi.available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
+	else
+		sbi.lz4_max_distance = le16_to_cpu(dsb->u1.lz4_max_distance);
 	return erofs_init_devices(&sbi, dsb);
 }
 
-- 
2.31.1

Re: [PATCH v2 1/4] erofs-utils: lib: refactor erofs compressors init

[Gao Xiang]

On 2023/6/15 18:17, Guo Xuenan via Linux-erofs wrote:
> using struct erofs_compressors_cfg for erofs compressor
> global configuration.
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> ---
>   lib/compressor.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
>   lib/compressor.h | 14 ++++++++++
>   2 files changed, 82 insertions(+)
> 
> diff --git a/lib/compressor.c b/lib/compressor.c
> index 52eb761..88a2fb0 100644
> --- a/lib/compressor.c
> +++ b/lib/compressor.c
> @@ -22,6 +22,74 @@ static const struct erofs_compressor *compressors[] = {
>   #endif
>   };
>   
> +/* for compressors type configuration */

This comment might be unnecessary.

> +static struct erofs_supported_algothrim {

				^ algorithm




> +	int algtype;
> +	const char *name;
> +} erofs_supported_algothrims[] = {
> +	{ Z_EROFS_COMPRESSION_LZ4, "lz4"},
> +	{ Z_EROFS_COMPRESSION_LZ4, "lz4hc"},
> +	{ Z_EROFS_COMPRESSION_LZMA, "lzma"},
> +};
> +
> +/* global compressors configuration */


Let's avoid this comment as well.


> +static struct erofs_compressors_cfg erofs_ccfg;
> +
> +static void erofs_init_compressor(struct compressor *compressor,
> +	const struct erofs_compressor *alg)
> +{
> +
> +	compressor->handle.alg = alg;
> +
> +	/* should be written in "minimum compression ratio * 100" */
> +	compressor->handle.compress_threshold = 100;
> +
> +	/* optimize for 4k size page */
> +	compressor->handle.destsize_alignsize = erofs_blksiz();
> +	compressor->handle.destsize_redzone_begin = erofs_blksiz() - 16;
> +	compressor->handle.destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
> +
> +	if (alg && alg->init)
> +		alg->init(&compressor->handle);
> +}
> +
> +static void erofs_compressor_register(const char *name, const struct erofs_compressor *alg)
> +{
> +	int i;
> +
> +	for (i = 0; i < erofs_ccfg.erofs_ccfg_num; i++) {
> +		if (!strcmp(erofs_ccfg.compressors[i].name, name)) {
> +			erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
> +			return;
> +		}
> +	}
> +
> +	erofs_ccfg.compressors[i].name = name;
> +	erofs_ccfg.compressors[i].algorithmtype = erofs_supported_algothrims[i].algtype;
> +	erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
> +	erofs_ccfg.erofs_ccfg_num = ++i;
> +}
> +
> +void erofs_register_compressors(void)
> +{
> +	int i;
> +
> +	erofs_ccfg.erofs_ccfg_num = 0;
> +	for (i = 0; i < ARRAY_SIZE(erofs_supported_algothrims); i++) {
> +		erofs_compressor_register(erofs_supported_algothrims[i].name, NULL);
> +	}
> +
> +#if LZ4_ENABLED
> +	erofs_compressor_register("lz4", &erofs_compressor_lz4);
> +#if LZ4HC_ENABLED
> +	erofs_compressor_register("lz4hc", &erofs_compressor_lz4hc);
> +#endif
> +#endif
> +#if HAVE_LIBLZMA
> +	erofs_compressor_register("lzma", &erofs_compressor_lzma);
> +#endif
> +}
> +
>   int erofs_compress_destsize(const struct erofs_compress *c,
>   			    const void *src, unsigned int *srcsize,
>   			    void *dst, unsigned int dstsize, bool inblocks)
> diff --git a/lib/compressor.h b/lib/compressor.h
> index cf063f1..1e760b6 100644
> --- a/lib/compressor.h
> +++ b/lib/compressor.h
> @@ -8,6 +8,7 @@
>   #define __EROFS_LIB_COMPRESSOR_H
>   
>   #include "erofs/defs.h"
> +#include "erofs/config.h"
>   
>   struct erofs_compress;
>   
> @@ -40,6 +41,18 @@ struct erofs_compress {
>   	void *private_data;
>   };
>   
> +struct compressor {
> +	const char *name;
> +	struct erofs_compress handle;
> +	unsigned int algorithmtype;
> +	bool enable;

could we just use a struct erofs_supported_algothrim * to replace
name and algorithmtype?

Also, please use `struct erofs_compressor` here.

Thanks,
Gao Xiang

Re: [PATCH v2 2/4] erofs-utils: lib: unify all identical compressor print function

[Gao Xiang]

On 2023/6/15 18:17, Guo Xuenan via Linux-erofs wrote:
> {dump,fsck}.erofs use the same compressor print function,
> available compressors means which algothrims are currently
> supported by binary tools.
> supported compressors including all algothrims that are ready
> for your erofs binary tools.
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> ---
>   fsck/main.c              | 15 +-----------
>   include/erofs/compress.h |  3 ++-
>   lib/compressor.c         | 51 ++++++++++++++++++++++++++--------------
>   mkfs/main.c              | 15 +-----------
>   4 files changed, 38 insertions(+), 46 deletions(-)
> 
> diff --git a/fsck/main.c b/fsck/main.c
> index f816bec..e559050 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -49,19 +49,6 @@ static struct option long_options[] = {
>   	{0, 0, 0, 0},
>   };
>   
> -static void print_available_decompressors(FILE *f, const char *delim)
> -{
> -	unsigned int i = 0;
> -	const char *s;
> -
> -	while ((s = z_erofs_list_available_compressors(i)) != NULL) {
> -		if (i++)
> -			fputs(delim, f);
> -		fputs(s, f);
> -	}
> -	fputc('\n', f);
> -}
> -
>   static void usage(void)
>   {
>   	fputs("usage: [options] IMAGE\n\n"
> @@ -84,7 +71,7 @@ static void usage(void)
>   	      " --no-preserve-owner    extract as yourself\n"
>   	      " --no-preserve-perms    apply user's umask when extracting permissions\n"
>   	      "\nSupported algorithms are: ", stderr);
> -	print_available_decompressors(stderr, ", ");
> +	erofs_print_available_compressors(stderr);
>   }
>   
>   static void erofsfsck_print_version(void)
> diff --git a/include/erofs/compress.h b/include/erofs/compress.h
> index 08af9e3..f1b9bbd 100644
> --- a/include/erofs/compress.h
> +++ b/include/erofs/compress.h
> @@ -22,7 +22,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode, int fd);
>   int z_erofs_compress_init(struct erofs_buffer_head *bh);
>   int z_erofs_compress_exit(void);
>   
> -const char *z_erofs_list_available_compressors(unsigned int i);
> +void erofs_print_available_compressors(FILE *f);
> +void erofs_print_supported_compressors(FILE *f, unsigned int mask);
>   
>   static inline bool erofs_is_packed_inode(struct erofs_inode *inode)
>   {
> diff --git a/lib/compressor.c b/lib/compressor.c
> index 88a2fb0..da8d1b9 100644
> --- a/lib/compressor.c
> +++ b/lib/compressor.c
> @@ -10,18 +10,6 @@
>   
>   #define EROFS_CONFIG_COMPR_DEF_BOUNDARY		(128)
>   
> -static const struct erofs_compressor *compressors[] = {
> -#if LZ4_ENABLED
> -#if LZ4HC_ENABLED
> -		&erofs_compressor_lz4hc,
> -#endif
> -		&erofs_compressor_lz4,
> -#endif
> -#if HAVE_LIBLZMA
> -		&erofs_compressor_lzma,
> -#endif
> -};
> -
>   /* for compressors type configuration */
>   static struct erofs_supported_algothrim {
>   	int algtype;
> @@ -119,9 +107,38 @@ int erofs_compress_destsize(const struct erofs_compress *c,
>   	return ret;
>   }
>   
> -const char *z_erofs_list_available_compressors(unsigned int i)
> +void erofs_print_supported_compressors(FILE *f, unsigned int mask)
>   {
> -	return i >= ARRAY_SIZE(compressors) ? NULL : compressors[i]->name;
> +	unsigned int i = 0;
> +	int comma = false;
> +	const char *s;
> +
> +	while (i < erofs_ccfg.erofs_ccfg_num) {
> +		s = erofs_ccfg.compressors[i].name;
> +		if (s && (mask & (1 << erofs_ccfg.compressors[i++].algorithmtype))) {
> +			if (comma)
> +				fputs(", ", f);
> +			else
> +				comma = true;
> +			fputs(s, f);
> +		}
> +	}
> +	fputc('\n', f);
> +}
> +
> +void erofs_print_available_compressors(FILE *f)
> +{

Should just erofs_print_supported_compressors(f, ~0) and avoid this helper?

Thanks,
Gao Xiang

Re: [PATCH v2 1/4] erofs-utils: lib: refactor erofs compressors init

[Guo Xuenan]

On 2023/6/15 18:44, Gao Xiang wrote:
>
>
> On 2023/6/15 18:17, Guo Xuenan via Linux-erofs wrote:
>> using struct erofs_compressors_cfg for erofs compressor
>> global configuration.
>>
>> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
>> ---
>>   lib/compressor.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
>>   lib/compressor.h | 14 ++++++++++
>>   2 files changed, 82 insertions(+)
>>
>> diff --git a/lib/compressor.c b/lib/compressor.c
>> index 52eb761..88a2fb0 100644
>> --- a/lib/compressor.c
>> +++ b/lib/compressor.c
>> @@ -22,6 +22,74 @@ static const struct erofs_compressor 
>> *compressors[] = {
>>   #endif
>>   };
>>   +/* for compressors type configuration */
>
> This comment might be unnecessary.
>
OKay
>> +static struct erofs_supported_algothrim {
>
>                 ^ algorithm
>
>
>
>
>> +    int algtype;
>> +    const char *name;
>> +} erofs_supported_algothrims[] = {
>> +    { Z_EROFS_COMPRESSION_LZ4, "lz4"},
>> +    { Z_EROFS_COMPRESSION_LZ4, "lz4hc"},
>> +    { Z_EROFS_COMPRESSION_LZMA, "lzma"},
>> +};
>> +
>> +/* global compressors configuration */
>
>
> Let's avoid this comment as well.
>
>
>> +static struct erofs_compressors_cfg erofs_ccfg;
>> +
>> +static void erofs_init_compressor(struct compressor *compressor,
>> +    const struct erofs_compressor *alg)
>> +{
>> +
>> +    compressor->handle.alg = alg;
>> +
>> +    /* should be written in "minimum compression ratio * 100" */
>> +    compressor->handle.compress_threshold = 100;
>> +
>> +    /* optimize for 4k size page */
>> +    compressor->handle.destsize_alignsize = erofs_blksiz();
>> +    compressor->handle.destsize_redzone_begin = erofs_blksiz() - 16;
>> +    compressor->handle.destsize_redzone_end = 
>> EROFS_CONFIG_COMPR_DEF_BOUNDARY;
>> +
>> +    if (alg && alg->init)
>> +        alg->init(&compressor->handle);
>> +}
>> +
>> +static void erofs_compressor_register(const char *name, const struct 
>> erofs_compressor *alg)
>> +{
>> +    int i;
>> +
>> +    for (i = 0; i < erofs_ccfg.erofs_ccfg_num; i++) {
>> +        if (!strcmp(erofs_ccfg.compressors[i].name, name)) {
>> + erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
>> +            return;
>> +        }
>> +    }
>> +
>> +    erofs_ccfg.compressors[i].name = name;
>> +    erofs_ccfg.compressors[i].algorithmtype = 
>> erofs_supported_algothrims[i].algtype;
>> +    erofs_init_compressor(&erofs_ccfg.compressors[i], alg);
>> +    erofs_ccfg.erofs_ccfg_num = ++i;
>> +}
>> +
>> +void erofs_register_compressors(void)
>> +{
>> +    int i;
>> +
>> +    erofs_ccfg.erofs_ccfg_num = 0;
>> +    for (i = 0; i < ARRAY_SIZE(erofs_supported_algothrims); i++) {
>> + erofs_compressor_register(erofs_supported_algothrims[i].name, NULL);
>> +    }
>> +
>> +#if LZ4_ENABLED
>> +    erofs_compressor_register("lz4", &erofs_compressor_lz4);
>> +#if LZ4HC_ENABLED
>> +    erofs_compressor_register("lz4hc", &erofs_compressor_lz4hc);
>> +#endif
>> +#endif
>> +#if HAVE_LIBLZMA
>> +    erofs_compressor_register("lzma", &erofs_compressor_lzma);
>> +#endif
>> +}
>> +
>>   int erofs_compress_destsize(const struct erofs_compress *c,
>>                   const void *src, unsigned int *srcsize,
>>                   void *dst, unsigned int dstsize, bool inblocks)
>> diff --git a/lib/compressor.h b/lib/compressor.h
>> index cf063f1..1e760b6 100644
>> --- a/lib/compressor.h
>> +++ b/lib/compressor.h
>> @@ -8,6 +8,7 @@
>>   #define __EROFS_LIB_COMPRESSOR_H
>>     #include "erofs/defs.h"
>> +#include "erofs/config.h"
>>     struct erofs_compress;
>>   @@ -40,6 +41,18 @@ struct erofs_compress {
>>       void *private_data;
>>   };
>>   +struct compressor {
>> +    const char *name;
>> +    struct erofs_compress handle;
>> +    unsigned int algorithmtype;
>> +    bool enable;
>
> could we just use a struct erofs_supported_algothrim * to replace
> name and algorithmtype?
>
Yes, of course, your suggestion is better!
> Also, please use `struct erofs_compressor` here.
struct erofs_compressor has been used  lib/compressor.h:15
> Thanks,
> Gao Xiang
>
-- 
Best regards
Guo Xuenan

Re: [PATCH v2 2/4] erofs-utils: lib: unify all identical compressor print function

[Guo Xuenan]

[-- Attachment #1: Type: text/plain, Size: 4683 bytes --]


On 2023/6/15 18:46, Gao Xiang wrote:
>
>
> On 2023/6/15 18:17, Guo Xuenan via Linux-erofs wrote:
>> {dump,fsck}.erofs use the same compressor print function,
>> available compressors means which algothrims are currently
>> supported by binary tools.
>> supported compressors including all algothrims that are ready
>> for your erofs binary tools.
>>
>> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
>> ---
>>   fsck/main.c              | 15 +-----------
>>   include/erofs/compress.h |  3 ++-
>>   lib/compressor.c         | 51 ++++++++++++++++++++++++++--------------
>>   mkfs/main.c              | 15 +-----------
>>   4 files changed, 38 insertions(+), 46 deletions(-)
>>
>> diff --git a/fsck/main.c b/fsck/main.c
>> index f816bec..e559050 100644
>> --- a/fsck/main.c
>> +++ b/fsck/main.c
>> @@ -49,19 +49,6 @@ static struct option long_options[] = {
>>       {0, 0, 0, 0},
>>   };
>>   -static void print_available_decompressors(FILE *f, const char *delim)
>> -{
>> -    unsigned int i = 0;
>> -    const char *s;
>> -
>> -    while ((s = z_erofs_list_available_compressors(i)) != NULL) {
>> -        if (i++)
>> -            fputs(delim, f);
>> -        fputs(s, f);
>> -    }
>> -    fputc('\n', f);
>> -}
>> -
>>   static void usage(void)
>>   {
>>       fputs("usage: [options] IMAGE\n\n"
>> @@ -84,7 +71,7 @@ static void usage(void)
>>             " --no-preserve-owner    extract as yourself\n"
>>             " --no-preserve-perms    apply user's umask when 
>> extracting permissions\n"
>>             "\nSupported algorithms are: ", stderr);
>> -    print_available_decompressors(stderr, ", ");
>> +    erofs_print_available_compressors(stderr);
>>   }
>>     static void erofsfsck_print_version(void)
>> diff --git a/include/erofs/compress.h b/include/erofs/compress.h
>> index 08af9e3..f1b9bbd 100644
>> --- a/include/erofs/compress.h
>> +++ b/include/erofs/compress.h
>> @@ -22,7 +22,8 @@ int erofs_write_compressed_file(struct erofs_inode 
>> *inode, int fd);
>>   int z_erofs_compress_init(struct erofs_buffer_head *bh);
>>   int z_erofs_compress_exit(void);
>>   -const char *z_erofs_list_available_compressors(unsigned int i);
>> +void erofs_print_available_compressors(FILE *f);
>> +void erofs_print_supported_compressors(FILE *f, unsigned int mask);
>>     static inline bool erofs_is_packed_inode(struct erofs_inode *inode)
>>   {
>> diff --git a/lib/compressor.c b/lib/compressor.c
>> index 88a2fb0..da8d1b9 100644
>> --- a/lib/compressor.c
>> +++ b/lib/compressor.c
>> @@ -10,18 +10,6 @@
>>     #define EROFS_CONFIG_COMPR_DEF_BOUNDARY        (128)
>>   -static const struct erofs_compressor *compressors[] = {
>> -#if LZ4_ENABLED
>> -#if LZ4HC_ENABLED
>> -        &erofs_compressor_lz4hc,
>> -#endif
>> -        &erofs_compressor_lz4,
>> -#endif
>> -#if HAVE_LIBLZMA
>> -        &erofs_compressor_lzma,
>> -#endif
>> -};
>> -
>>   /* for compressors type configuration */
>>   static struct erofs_supported_algothrim {
>>       int algtype;
>> @@ -119,9 +107,38 @@ int erofs_compress_destsize(const struct 
>> erofs_compress *c,
>>       return ret;
>>   }
>>   -const char *z_erofs_list_available_compressors(unsigned int i)
>> +void erofs_print_supported_compressors(FILE *f, unsigned int mask)
>>   {
>> -    return i >= ARRAY_SIZE(compressors) ? NULL : compressors[i]->name;
>> +    unsigned int i = 0;
>> +    int comma = false;
>> +    const char *s;
>> +
>> +    while (i < erofs_ccfg.erofs_ccfg_num) {
>> +        s = erofs_ccfg.compressors[i].name;
>> +        if (s && (mask & (1 << 
>> erofs_ccfg.compressors[i++].algorithmtype))) {
>> +            if (comma)
>> +                fputs(", ", f);
>> +            else
>> +                comma = true;
>> +            fputs(s, f);
>> +        }
>> +    }
>> +    fputc('\n', f);
>> +}
>> +
>> +void erofs_print_available_compressors(FILE *f)
>> +{
>
> Should just erofs_print_supported_compressors(f, ~0) and avoid this 
> helper?
>
As commit message of this patch explained, available compressors means 
which algothrims are
currentlyavailable to user in binary tools. I mean fsck/mkfs.erofs 
binary tools may only support
lz4 compression.erofs_print_available_compressors should only print lz4; 
but for dump.erofs ,
which is not used tomake erofs image, there is a bit difference here. 
dump.erofs should identify
all supportedalgorithms.
> Thanks,
> Gao Xiang
>
-- 
Best regards
Guo Xuenan

[-- Attachment #2: Type: text/html, Size: 9239 bytes --]

Re: [PATCH v2 1/4] erofs-utils: lib: refactor erofs compressors init

[Gao Xiang]

On 2023/6/15 19:37, Guo Xuenan wrote:
> 

...

> Yes, of course, your suggestion is better!
>> Also, please use `struct erofs_compressor` here.
> struct erofs_compressor has been used  lib/compressor.h:15

So we might need another name for this.

Thanks,
Gao Xiang

Re: [PATCH v2 2/4] erofs-utils: lib: unify all identical compressor print function

[Gao Xiang]

On 2023/6/15 20:01, Guo Xuenan wrote:
> 
> On 2023/6/15 18:46, Gao Xiang wrote:

...


>>> +
>>> +void erofs_print_available_compressors(FILE *f)
>>> +{
>>
>> Should just erofs_print_supported_compressors(f, ~0) and avoid this helper?
>>
> As commit message of this patch explained, available compressors means which algothrims are
> currentlyavailable to user in binary tools. I mean fsck/mkfs.erofs binary tools may only support
> lz4 compression.erofs_print_available_compressors should only print lz4; but for dump.erofs ,
> which is not used tomake erofs image, there is a bit difference here. dump.erofs should identify
> all supportedalgorithms.

Oh, that makes sense, sorry about the noise.

>> Thanks,
>> Gao Xiang
>>
> -- 
> Best regards
> Guo Xuenan
> 

[PATCH v2 1/4] erofs-utils: lib: refactor erofs compressors init

[Guo Xuenan via Linux-erofs]

using struct erofs_compressors_cfg for erofs compressor
global configuration.

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
---
 lib/compressor.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/compressor.h |  2 ++
 2 files changed, 63 insertions(+)

diff --git a/lib/compressor.c b/lib/compressor.c
index 4333f26..6288297 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -26,6 +26,67 @@ static const struct erofs_compressor *compressors[] = {
 #endif
 };
 
+static struct erofs_supported_algorithm {
+	bool enable;
+	unsigned int algtype;
+	const char *name;
+	struct erofs_compress handle;
+} erofs_supported_algorithms[] = {
+	{ .algtype = Z_EROFS_COMPRESSION_LZ4, .name = "lz4" },
+	{ .algtype = Z_EROFS_COMPRESSION_LZ4, .name = "lz4hc" },
+	{ .algtype = Z_EROFS_COMPRESSION_LZMA, .name = "lzma" },
+	{ .algtype = Z_EROFS_COMPRESSION_DEFLATE, .name = "deflate" },
+	{ .algtype = Z_EROFS_COMPRESSION_DEFLATE, .name = "libdeflate" },
+};
+
+static void erofs_init_compressor(struct erofs_sb_info *sbi,
+		struct erofs_compress *handle, const struct erofs_compressor *alg)
+{
+
+	handle->alg = alg;
+
+	/* should be written in "minimum compression ratio * 100" */
+	handle->compress_threshold = 100;
+
+	/* optimize for 4k size page */
+	handle->destsize_alignsize = erofs_blksiz(sbi);
+	handle->destsize_redzone_begin = erofs_blksiz(sbi) - 16;
+	handle->destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
+
+	if (alg && alg->init)
+		alg->init(handle);
+}
+
+static void erofs_compressor_register(struct erofs_sb_info *sbi,
+		const char *name, const struct erofs_compressor *alg)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(erofs_supported_algorithms); i++) {
+		if (!strcmp(erofs_supported_algorithms[i].name, name)) {
+			erofs_init_compressor(sbi, &erofs_supported_algorithms[i].handle, alg);
+			return;
+		}
+	}
+}
+
+void erofs_register_compressors(struct erofs_sb_info *sbi)
+{
+#if LZ4_ENABLED
+	erofs_compressor_register(sbi, "lz4", &erofs_compressor_lz4);
+#if LZ4HC_ENABLED
+	erofs_compressor_register(sbi, "lz4hc", &erofs_compressor_lz4hc);
+#endif
+#endif
+#if HAVE_LIBLZMA
+	erofs_compressor_register(sbi, "lzma", &erofs_compressor_lzma);
+#endif
+	erofs_compressor_register(sbi, "deflate", &erofs_compressor_deflate);
+#if HAVE_LIBDEFLATE
+	erofs_compressor_register(sbi, "libdeflate", &erofs_compressor_libdeflate);
+#endif
+}
+
 int erofs_compress_destsize(const struct erofs_compress *c,
 			    const void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize, bool inblocks)
diff --git a/lib/compressor.h b/lib/compressor.h
index 08a3988..66ea933 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -8,6 +8,7 @@
 #define __EROFS_LIB_COMPRESSOR_H
 
 #include "erofs/defs.h"
+#include "erofs/config.h"
 
 struct erofs_compress;
 
@@ -56,5 +57,6 @@ int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
 int erofs_compressor_init(struct erofs_sb_info *sbi,
 		struct erofs_compress *c, char *alg_name);
 int erofs_compressor_exit(struct erofs_compress *c);
+void erofs_register_compressors(struct erofs_sb_info *sbi);
 
 #endif
-- 
2.34.3