[PATCH v2 0/3] erofs-utils: prepare for per-(sub)file compression strategies

[PATCH v2 0/3] erofs-utils: prepare for per-(sub)file compression strategies

[Gao Xiang]

Hi all,

This patchset mainly provides a new helper z_erofs_get_max_pclusterblks()
to prepare for per-(sub)file compression strategies, valid pclustersize
can be returned according to detailed data type or access patterns.

In order to do that, compression header is now generated on the per-file
basis as well, which will be also needed for parallel compression in
the future.

randomizing pclusterblks support in debugging mode is also added to
randomize each pcluster size for big pcluster selftest.

Thanks,
Gao Xiang

Gao Xiang (3):
  erofs-utils: prepare for per-(sub)file compress strategies
  erofs-utils: introduce --enable-debug
  erofs-utils: support randomizing pclusterblks in debugging mode

 configure.ac           | 12 ++++++
 include/erofs/config.h |  3 ++
 lib/compress.c         | 85 ++++++++++++++++++++++++++++--------------
 mkfs/main.c            | 50 ++++++++++++++++---------
 4 files changed, 105 insertions(+), 45 deletions(-)

-- 
2.20.1

[PATCH v2 1/3] erofs-utils: prepare for per-(sub)file compress strategies

[Gao Xiang]

In order to adjust pclustersize on the per-(sub)file basis,
generating per-file map headers are needed instead.

In addition to that, we could use COMPACT_4B on the per-file
basis as well after this patch.

Signed-off-by: Gao Xiang <xiang@kernel.org>
---
 lib/compress.c | 81 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 54 insertions(+), 27 deletions(-)

diff --git a/lib/compress.c b/lib/compress.c
index e146416890f0..2f83198202ba 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -22,7 +22,7 @@
 static struct erofs_compress compresshandle;
 static int compressionlevel;
 
-static struct z_erofs_map_header mapheader;
+static unsigned int algorithmtype[2];
 
 struct z_erofs_vle_compress_ctx {
 	u8 *metacur;
@@ -149,12 +149,16 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,
 	return count;
 }
 
+/* TODO: apply per-(sub)file strategies here */
+static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
+{
+	return cfg.c_physical_clusterblks;
+}
+
 static int vle_compress_one(struct erofs_inode *inode,
 			    struct z_erofs_vle_compress_ctx *ctx,
 			    bool final)
 {
-	const unsigned int pclusterblks = cfg.c_physical_clusterblks;
-	const unsigned int pclustersize = pclusterblks * EROFS_BLKSIZ;
 	struct erofs_compress *const h = &compresshandle;
 	unsigned int len = ctx->tail - ctx->head;
 	unsigned int count;
@@ -163,6 +167,8 @@ static int vle_compress_one(struct erofs_inode *inode,
 	char *const dst = dstbuf + EROFS_BLKSIZ;
 
 	while (len) {
+		const unsigned int pclustersize =
+			z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
 		bool raw;
 
 		if (len <= pclustersize) {
@@ -342,13 +348,14 @@ static void *write_compacted_indexes(u8 *out,
 int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
 					erofs_blk_t blkaddr,
 					unsigned int legacymetasize,
-					unsigned int logical_clusterbits)
+					void *compressmeta)
 {
 	const unsigned int mpos = Z_EROFS_VLE_EXTENT_ALIGN(inode->inode_isize +
 							   inode->xattr_isize) +
 				  sizeof(struct z_erofs_map_header);
 	const unsigned int totalidx = (legacymetasize -
 				       Z_EROFS_LEGACY_MAP_HEADER_SIZE) / 8;
+	const unsigned int logical_clusterbits = inode->z_logical_clusterbits;
 	u8 *out, *in;
 	struct z_erofs_compressindex_vec cv[16];
 	/* # of 8-byte units so that it can be aligned with 32 bytes */
@@ -379,9 +386,9 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
 		compacted_4b_end = totalidx;
 	}
 
-	out = in = inode->compressmeta;
+	out = in = compressmeta;
 
-	out += sizeof(mapheader);
+	out += sizeof(struct z_erofs_map_header);
 	in += Z_EROFS_LEGACY_MAP_HEADER_SIZE;
 
 	dummy_head = false;
@@ -428,11 +435,26 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
 					      4, logical_clusterbits, true,
 					      &dummy_head);
 	}
-	inode->extent_isize = out - (u8 *)inode->compressmeta;
-	inode->datalayout = EROFS_INODE_FLAT_COMPRESSION;
+	inode->extent_isize = out - (u8 *)compressmeta;
 	return 0;
 }
 
+static void z_erofs_write_mapheader(struct erofs_inode *inode,
+				    void *compressmeta)
+{
+	struct z_erofs_map_header h = {
+		.h_advise = cpu_to_le16(inode->z_advise),
+		.h_algorithmtype = inode->z_algorithmtype[1] << 4 |
+				   inode->z_algorithmtype[0],
+		/* lclustersize */
+		.h_clusterbits = inode->z_logical_clusterbits - 12,
+	};
+
+	memset(compressmeta, 0, Z_EROFS_LEGACY_MAP_HEADER_SIZE);
+	/* write out map header */
+	memcpy(compressmeta, &h, sizeof(struct z_erofs_map_header));
+}
+
 int erofs_write_compressed_file(struct erofs_inode *inode)
 {
 	struct erofs_buffer_head *bh;
@@ -459,9 +481,25 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 		goto err_close;
 	}
 
-	memset(compressmeta, 0, Z_EROFS_LEGACY_MAP_HEADER_SIZE);
-	/* write out compressed header */
-	memcpy(compressmeta, &mapheader, sizeof(mapheader));
+	/* initialize per-file compression setting */
+	inode->z_advise = 0;
+	if (!cfg.c_legacy_compress) {
+		inode->z_advise |= Z_EROFS_ADVISE_COMPACTED_2B;
+		inode->datalayout = EROFS_INODE_FLAT_COMPRESSION;
+	} else {
+		inode->datalayout = EROFS_INODE_FLAT_COMPRESSION_LEGACY;
+	}
+
+	if (cfg.c_physical_clusterblks > 1) {
+		inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_1;
+		if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION)
+			inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_2;
+	}
+	inode->z_algorithmtype[0] = algorithmtype[0];
+	inode->z_algorithmtype[1] = algorithmtype[1];
+	inode->z_logical_clusterbits = LOG_BLOCK_SIZE;
+
+	z_erofs_write_mapheader(inode, compressmeta);
 
 	blkaddr = erofs_mapbh(bh->block);	/* start_blkaddr */
 	ctx.blkaddr = blkaddr;
@@ -516,19 +554,19 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	 *       when both mkfs & kernel support compression inline.
 	 */
 	erofs_bdrop(bh, false);
-	inode->compressmeta = compressmeta;
 	inode->idata_size = 0;
 	inode->u.i_blocks = compressed_blocks;
 
 	legacymetasize = ctx.metacur - compressmeta;
-	if (cfg.c_legacy_compress) {
+	if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
 		inode->extent_isize = legacymetasize;
-		inode->datalayout = EROFS_INODE_FLAT_COMPRESSION_LEGACY;
 	} else {
 		ret = z_erofs_convert_to_compacted_format(inode, blkaddr,
-							  legacymetasize, 12);
+							  legacymetasize,
+							  compressmeta);
 		DBG_BUGON(ret);
 	}
+	inode->compressmeta = compressmeta;
 	return 0;
 
 err_bdrop:
@@ -580,7 +618,6 @@ int z_erofs_build_compr_cfgs(struct erofs_buffer_head *sb_bh)
 
 int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
 {
-	unsigned int algorithmtype[2];
 	/* initialize for primary compression algorithm */
 	int ret = erofs_compressor_init(&compresshandle,
 					cfg.c_compr_alg_master);
@@ -603,16 +640,13 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
 		compresshandle.alg->default_level :
 		cfg.c_compr_level_master;
 
-	/* figure out mapheader */
+	/* figure out primary algorithm */
 	ret = erofs_get_compress_algorithm_id(cfg.c_compr_alg_master);
 	if (ret < 0)
 		return ret;
 
 	algorithmtype[0] = ret;	/* primary algorithm (head 0) */
 	algorithmtype[1] = 0;	/* secondary algorithm (head 1) */
-	mapheader.h_advise = 0;
-	if (!cfg.c_legacy_compress)
-		mapheader.h_advise |= Z_EROFS_ADVISE_COMPACTED_2B;
 	/*
 	 * if big pcluster is enabled, an extra CBLKCNT lcluster index needs
 	 * to be loaded in order to get those compressed block counts.
@@ -625,15 +659,8 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
 			return -EINVAL;
 		}
 		erofs_sb_set_big_pcluster();
-		mapheader.h_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_1;
-		if (!cfg.c_legacy_compress)
-			mapheader.h_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_2;
-
 		erofs_warn("EXPERIMENTAL big pcluster feature in use. Use at your own risk!");
 	}
-	mapheader.h_algorithmtype = algorithmtype[1] << 4 |
-					  algorithmtype[0];
-	mapheader.h_clusterbits = LOG_BLOCK_SIZE - 12;
 
 	if (erofs_sb_has_compr_cfgs()) {
 		sbi.available_compr_algs |= 1 << ret;
-- 
2.20.1

[PATCH v2 2/3] erofs-utils: introduce --enable-debug

[Gao Xiang]

This adds --enable-debug to explicitly specify debugging mode.

It's disabled by default, which means NDEBUG macro is defined instead.

Signed-off-by: Gao Xiang <xiang@kernel.org>
---
 configure.ac | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/configure.ac b/configure.ac
index 28926c303c5c..f626064cc55c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -59,6 +59,12 @@ AC_DEFUN([EROFS_UTILS_PARSE_DIRECTORY],
  fi
 ])
 
+AC_ARG_ENABLE([debug],
+    [AS_HELP_STRING([--enable-debug],
+                    [enable debugging mode @<:@default=no@:>@])],
+    [enable_debug="$enableval"],
+    [enable_debug="no"])
+
 AC_ARG_ENABLE(lz4,
    [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])],
    [enable_lz4="$enableval"], [enable_lz4="yes"])
@@ -150,6 +156,12 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
 # Checks for library functions.
 AC_CHECK_FUNCS([backtrace fallocate gettimeofday memset realpath strdup strerror strrchr strtoull])
 
+# Configure debug mode
+AS_IF([test "x$enable_debug" != "xno"], [], [
+  dnl Turn off all assert checking.
+  CPPFLAGS="$CPPFLAGS -DNDEBUG"
+])
+
 # Configure libuuid
 AS_IF([test "x$with_uuid" != "xno"], [
   PKG_CHECK_MODULES([libuuid], [uuid])
-- 
2.20.1

[PATCH v2 3/3] erofs-utils: support randomizing pclusterblks in debugging mode

[Gao Xiang]

It's used for big pcluster selftest.

Signed-off-by: Gao Xiang <xiang@kernel.org>
---
 include/erofs/config.h |  3 +++
 lib/compress.c         |  4 ++++
 mkfs/main.c            | 50 +++++++++++++++++++++++++++---------------
 3 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/include/erofs/config.h b/include/erofs/config.h
index e2f6541f1d1f..21bd25e886e6 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -40,6 +40,9 @@ struct erofs_configure {
 	int c_dbg_lvl;
 	bool c_dry_run;
 	bool c_legacy_compress;
+#ifndef NDEBUG
+	bool c_random_pclusterblks;
+#endif
 	char c_timeinherit;
 
 #ifdef HAVE_LIBSELINUX
diff --git a/lib/compress.c b/lib/compress.c
index 2f83198202ba..1b847ce27c2f 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -152,6 +152,10 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,
 /* TODO: apply per-(sub)file strategies here */
 static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
 {
+#ifndef NDEBUG
+	if (cfg.c_random_pclusterblks)
+		return 1 + rand() % cfg.c_physical_clusterblks;
+#endif
 	return cfg.c_physical_clusterblks;
 }
 
diff --git a/mkfs/main.c b/mkfs/main.c
index 3e0f64eb2d31..b2a4cba1d2f5 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -39,6 +39,9 @@ static struct option long_options[] = {
 	{"force-uid", required_argument, NULL, 5},
 	{"force-gid", required_argument, NULL, 6},
 	{"all-root", no_argument, NULL, 7},
+#ifndef NDEBUG
+	{"random-pclusterblks", no_argument, NULL, 8},
+#endif
 #ifdef WITH_ANDROID
 	{"mount-point", required_argument, NULL, 10},
 	{"product-out", required_argument, NULL, 11},
@@ -64,29 +67,32 @@ static void usage(void)
 {
 	fputs("usage: [options] FILE DIRECTORY\n\n"
 	      "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
-	      " -zX[,Y]            X=compressor (Y=compression level, optional)\n"
-	      " -C#                specify the size of compress physical cluster in bytes\n"
-	      " -d#                set output message level to # (maximum 9)\n"
-	      " -x#                set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
-	      " -EX[,...]          X=extended options\n"
-	      " -T#                set a fixed UNIX timestamp # to all files\n"
+	      " -zX[,Y]               X=compressor (Y=compression level, optional)\n"
+	      " -C#                   specify the size of compress physical cluster in bytes\n"
+	      " -d#                   set output message level to # (maximum 9)\n"
+	      " -x#                   set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
+	      " -EX[,...]             X=extended options\n"
+	      " -T#                   set a fixed UNIX timestamp # to all files\n"
 #ifdef HAVE_LIBUUID
-	      " -UX                use a given filesystem UUID\n"
+	      " -UX                   use a given filesystem UUID\n"
 #endif
-	      " --exclude-path=X   avoid including file X (X = exact literal path)\n"
-	      " --exclude-regex=X  avoid including files that match X (X = regular expression)\n"
+	      " --exclude-path=X      avoid including file X (X = exact literal path)\n"
+	      " --exclude-regex=X     avoid including files that match X (X = regular expression)\n"
 #ifdef HAVE_LIBSELINUX
-	      " --file-contexts=X  specify a file contexts file to setup selinux labels\n"
+	      " --file-contexts=X     specify a file contexts file to setup selinux labels\n"
+#endif
+	      " --force-uid=#         set all file uids to # (# = UID)\n"
+	      " --force-gid=#         set all file gids to # (# = GID)\n"
+	      " --all-root            make all files owned by root\n"
+	      " --help                display this help and exit\n"
+#ifndef NDEBUG
+	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
 #endif
-	      " --force-uid=#      set all file uids to # (# = UID)\n"
-	      " --force-gid=#      set all file gids to # (# = GID)\n"
-	      " --all-root         make all files owned by root\n"
-	      " --help             display this help and exit\n"
 #ifdef WITH_ANDROID
 	      "\nwith following android-specific options:\n"
-	      " --mount-point=X    X=prefix of target fs path (default: /)\n"
-	      " --product-out=X    X=product_out directory\n"
-	      " --fs-config-file=X X=fs_config file\n"
+	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
+	      " --product-out=X       X=product_out directory\n"
+	      " --fs-config-file=X    X=fs_config file\n"
 #endif
 	      "\nAvailable compressors are: ", stderr);
 	print_available_compressors(stderr, ", ");
@@ -257,6 +263,11 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 7:
 			cfg.c_uid = cfg.c_gid = 0;
 			break;
+#ifndef NDEBUG
+		case 8:
+			cfg.c_random_pclusterblks = true;
+			break;
+#endif
 #ifdef WITH_ANDROID
 		case 10:
 			cfg.mount_point = optarg;
@@ -523,7 +534,10 @@ int main(int argc, char **argv)
 
 	erofs_show_config();
 	erofs_set_fs_root(cfg.c_src_path);
-
+#ifndef NDEBUG
+	if (cfg.c_random_pclusterblks)
+		srand(time(NULL));
+#endif
 	sb_bh = erofs_buffer_init();
 	if (IS_ERR(sb_bh)) {
 		err = PTR_ERR(sb_bh);
-- 
2.20.1

Re: [PATCH v2 3/3] erofs-utils: support randomizing pclusterblks in debugging mode

[Christopher Hess via Linux-erofs]

Fix code source 

Sent from my iPhone