* [PATCH v2 1/3] erofs-utils: prepare for per-(sub)file compress strategies
2021-05-22 4:34 [PATCH v2 0/3] erofs-utils: prepare for per-(sub)file compression strategies Gao Xiang
@ 2021-05-22 4:35 ` Gao Xiang
2021-05-22 4:35 ` [PATCH v2 2/3] erofs-utils: introduce --enable-debug Gao Xiang
2021-05-22 4:35 ` [PATCH v2 3/3] erofs-utils: support randomizing pclusterblks in debugging mode Gao Xiang
2 siblings, 0 replies; 4+ messages in thread
From: Gao Xiang @ 2021-05-22 4:35 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
In order to adjust pclustersize on the per-(sub)file basis,
generating per-file map headers are needed instead.
In addition to that, we could use COMPACT_4B on the per-file
basis as well after this patch.
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
lib/compress.c | 81 +++++++++++++++++++++++++++++++++-----------------
1 file changed, 54 insertions(+), 27 deletions(-)
diff --git a/lib/compress.c b/lib/compress.c
index e146416890f0..2f83198202ba 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -22,7 +22,7 @@
static struct erofs_compress compresshandle;
static int compressionlevel;
-static struct z_erofs_map_header mapheader;
+static unsigned int algorithmtype[2];
struct z_erofs_vle_compress_ctx {
u8 *metacur;
@@ -149,12 +149,16 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,
return count;
}
+/* TODO: apply per-(sub)file strategies here */
+static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
+{
+ return cfg.c_physical_clusterblks;
+}
+
static int vle_compress_one(struct erofs_inode *inode,
struct z_erofs_vle_compress_ctx *ctx,
bool final)
{
- const unsigned int pclusterblks = cfg.c_physical_clusterblks;
- const unsigned int pclustersize = pclusterblks * EROFS_BLKSIZ;
struct erofs_compress *const h = &compresshandle;
unsigned int len = ctx->tail - ctx->head;
unsigned int count;
@@ -163,6 +167,8 @@ static int vle_compress_one(struct erofs_inode *inode,
char *const dst = dstbuf + EROFS_BLKSIZ;
while (len) {
+ const unsigned int pclustersize =
+ z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
bool raw;
if (len <= pclustersize) {
@@ -342,13 +348,14 @@ static void *write_compacted_indexes(u8 *out,
int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
erofs_blk_t blkaddr,
unsigned int legacymetasize,
- unsigned int logical_clusterbits)
+ void *compressmeta)
{
const unsigned int mpos = Z_EROFS_VLE_EXTENT_ALIGN(inode->inode_isize +
inode->xattr_isize) +
sizeof(struct z_erofs_map_header);
const unsigned int totalidx = (legacymetasize -
Z_EROFS_LEGACY_MAP_HEADER_SIZE) / 8;
+ const unsigned int logical_clusterbits = inode->z_logical_clusterbits;
u8 *out, *in;
struct z_erofs_compressindex_vec cv[16];
/* # of 8-byte units so that it can be aligned with 32 bytes */
@@ -379,9 +386,9 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
compacted_4b_end = totalidx;
}
- out = in = inode->compressmeta;
+ out = in = compressmeta;
- out += sizeof(mapheader);
+ out += sizeof(struct z_erofs_map_header);
in += Z_EROFS_LEGACY_MAP_HEADER_SIZE;
dummy_head = false;
@@ -428,11 +435,26 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
4, logical_clusterbits, true,
&dummy_head);
}
- inode->extent_isize = out - (u8 *)inode->compressmeta;
- inode->datalayout = EROFS_INODE_FLAT_COMPRESSION;
+ inode->extent_isize = out - (u8 *)compressmeta;
return 0;
}
+static void z_erofs_write_mapheader(struct erofs_inode *inode,
+ void *compressmeta)
+{
+ struct z_erofs_map_header h = {
+ .h_advise = cpu_to_le16(inode->z_advise),
+ .h_algorithmtype = inode->z_algorithmtype[1] << 4 |
+ inode->z_algorithmtype[0],
+ /* lclustersize */
+ .h_clusterbits = inode->z_logical_clusterbits - 12,
+ };
+
+ memset(compressmeta, 0, Z_EROFS_LEGACY_MAP_HEADER_SIZE);
+ /* write out map header */
+ memcpy(compressmeta, &h, sizeof(struct z_erofs_map_header));
+}
+
int erofs_write_compressed_file(struct erofs_inode *inode)
{
struct erofs_buffer_head *bh;
@@ -459,9 +481,25 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
goto err_close;
}
- memset(compressmeta, 0, Z_EROFS_LEGACY_MAP_HEADER_SIZE);
- /* write out compressed header */
- memcpy(compressmeta, &mapheader, sizeof(mapheader));
+ /* initialize per-file compression setting */
+ inode->z_advise = 0;
+ if (!cfg.c_legacy_compress) {
+ inode->z_advise |= Z_EROFS_ADVISE_COMPACTED_2B;
+ inode->datalayout = EROFS_INODE_FLAT_COMPRESSION;
+ } else {
+ inode->datalayout = EROFS_INODE_FLAT_COMPRESSION_LEGACY;
+ }
+
+ if (cfg.c_physical_clusterblks > 1) {
+ inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_1;
+ if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION)
+ inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_2;
+ }
+ inode->z_algorithmtype[0] = algorithmtype[0];
+ inode->z_algorithmtype[1] = algorithmtype[1];
+ inode->z_logical_clusterbits = LOG_BLOCK_SIZE;
+
+ z_erofs_write_mapheader(inode, compressmeta);
blkaddr = erofs_mapbh(bh->block); /* start_blkaddr */
ctx.blkaddr = blkaddr;
@@ -516,19 +554,19 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
* when both mkfs & kernel support compression inline.
*/
erofs_bdrop(bh, false);
- inode->compressmeta = compressmeta;
inode->idata_size = 0;
inode->u.i_blocks = compressed_blocks;
legacymetasize = ctx.metacur - compressmeta;
- if (cfg.c_legacy_compress) {
+ if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
inode->extent_isize = legacymetasize;
- inode->datalayout = EROFS_INODE_FLAT_COMPRESSION_LEGACY;
} else {
ret = z_erofs_convert_to_compacted_format(inode, blkaddr,
- legacymetasize, 12);
+ legacymetasize,
+ compressmeta);
DBG_BUGON(ret);
}
+ inode->compressmeta = compressmeta;
return 0;
err_bdrop:
@@ -580,7 +618,6 @@ int z_erofs_build_compr_cfgs(struct erofs_buffer_head *sb_bh)
int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
{
- unsigned int algorithmtype[2];
/* initialize for primary compression algorithm */
int ret = erofs_compressor_init(&compresshandle,
cfg.c_compr_alg_master);
@@ -603,16 +640,13 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
compresshandle.alg->default_level :
cfg.c_compr_level_master;
- /* figure out mapheader */
+ /* figure out primary algorithm */
ret = erofs_get_compress_algorithm_id(cfg.c_compr_alg_master);
if (ret < 0)
return ret;
algorithmtype[0] = ret; /* primary algorithm (head 0) */
algorithmtype[1] = 0; /* secondary algorithm (head 1) */
- mapheader.h_advise = 0;
- if (!cfg.c_legacy_compress)
- mapheader.h_advise |= Z_EROFS_ADVISE_COMPACTED_2B;
/*
* if big pcluster is enabled, an extra CBLKCNT lcluster index needs
* to be loaded in order to get those compressed block counts.
@@ -625,15 +659,8 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
return -EINVAL;
}
erofs_sb_set_big_pcluster();
- mapheader.h_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_1;
- if (!cfg.c_legacy_compress)
- mapheader.h_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_2;
-
erofs_warn("EXPERIMENTAL big pcluster feature in use. Use at your own risk!");
}
- mapheader.h_algorithmtype = algorithmtype[1] << 4 |
- algorithmtype[0];
- mapheader.h_clusterbits = LOG_BLOCK_SIZE - 12;
if (erofs_sb_has_compr_cfgs()) {
sbi.available_compr_algs |= 1 << ret;
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v2 2/3] erofs-utils: introduce --enable-debug
2021-05-22 4:34 [PATCH v2 0/3] erofs-utils: prepare for per-(sub)file compression strategies Gao Xiang
2021-05-22 4:35 ` [PATCH v2 1/3] erofs-utils: prepare for per-(sub)file compress strategies Gao Xiang
@ 2021-05-22 4:35 ` Gao Xiang
2021-05-22 4:35 ` [PATCH v2 3/3] erofs-utils: support randomizing pclusterblks in debugging mode Gao Xiang
2 siblings, 0 replies; 4+ messages in thread
From: Gao Xiang @ 2021-05-22 4:35 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
This adds --enable-debug to explicitly specify debugging mode.
It's disabled by default, which means NDEBUG macro is defined instead.
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
configure.ac | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/configure.ac b/configure.ac
index 28926c303c5c..f626064cc55c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -59,6 +59,12 @@ AC_DEFUN([EROFS_UTILS_PARSE_DIRECTORY],
fi
])
+AC_ARG_ENABLE([debug],
+ [AS_HELP_STRING([--enable-debug],
+ [enable debugging mode @<:@default=no@:>@])],
+ [enable_debug="$enableval"],
+ [enable_debug="no"])
+
AC_ARG_ENABLE(lz4,
[AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])],
[enable_lz4="$enableval"], [enable_lz4="yes"])
@@ -150,6 +156,12 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
# Checks for library functions.
AC_CHECK_FUNCS([backtrace fallocate gettimeofday memset realpath strdup strerror strrchr strtoull])
+# Configure debug mode
+AS_IF([test "x$enable_debug" != "xno"], [], [
+ dnl Turn off all assert checking.
+ CPPFLAGS="$CPPFLAGS -DNDEBUG"
+])
+
# Configure libuuid
AS_IF([test "x$with_uuid" != "xno"], [
PKG_CHECK_MODULES([libuuid], [uuid])
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v2 3/3] erofs-utils: support randomizing pclusterblks in debugging mode
2021-05-22 4:34 [PATCH v2 0/3] erofs-utils: prepare for per-(sub)file compression strategies Gao Xiang
2021-05-22 4:35 ` [PATCH v2 1/3] erofs-utils: prepare for per-(sub)file compress strategies Gao Xiang
2021-05-22 4:35 ` [PATCH v2 2/3] erofs-utils: introduce --enable-debug Gao Xiang
@ 2021-05-22 4:35 ` Gao Xiang
2 siblings, 0 replies; 4+ messages in thread
From: Gao Xiang @ 2021-05-22 4:35 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
It's used for big pcluster selftest.
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
include/erofs/config.h | 3 +++
lib/compress.c | 4 ++++
mkfs/main.c | 50 +++++++++++++++++++++++++++---------------
3 files changed, 39 insertions(+), 18 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index e2f6541f1d1f..21bd25e886e6 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -40,6 +40,9 @@ struct erofs_configure {
int c_dbg_lvl;
bool c_dry_run;
bool c_legacy_compress;
+#ifndef NDEBUG
+ bool c_random_pclusterblks;
+#endif
char c_timeinherit;
#ifdef HAVE_LIBSELINUX
diff --git a/lib/compress.c b/lib/compress.c
index 2f83198202ba..1b847ce27c2f 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -152,6 +152,10 @@ static int write_uncompressed_extent(struct z_erofs_vle_compress_ctx *ctx,
/* TODO: apply per-(sub)file strategies here */
static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
{
+#ifndef NDEBUG
+ if (cfg.c_random_pclusterblks)
+ return 1 + rand() % cfg.c_physical_clusterblks;
+#endif
return cfg.c_physical_clusterblks;
}
diff --git a/mkfs/main.c b/mkfs/main.c
index 3e0f64eb2d31..b2a4cba1d2f5 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -39,6 +39,9 @@ static struct option long_options[] = {
{"force-uid", required_argument, NULL, 5},
{"force-gid", required_argument, NULL, 6},
{"all-root", no_argument, NULL, 7},
+#ifndef NDEBUG
+ {"random-pclusterblks", no_argument, NULL, 8},
+#endif
#ifdef WITH_ANDROID
{"mount-point", required_argument, NULL, 10},
{"product-out", required_argument, NULL, 11},
@@ -64,29 +67,32 @@ static void usage(void)
{
fputs("usage: [options] FILE DIRECTORY\n\n"
"Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
- " -zX[,Y] X=compressor (Y=compression level, optional)\n"
- " -C# specify the size of compress physical cluster in bytes\n"
- " -d# set output message level to # (maximum 9)\n"
- " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
- " -EX[,...] X=extended options\n"
- " -T# set a fixed UNIX timestamp # to all files\n"
+ " -zX[,Y] X=compressor (Y=compression level, optional)\n"
+ " -C# specify the size of compress physical cluster in bytes\n"
+ " -d# set output message level to # (maximum 9)\n"
+ " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
+ " -EX[,...] X=extended options\n"
+ " -T# set a fixed UNIX timestamp # to all files\n"
#ifdef HAVE_LIBUUID
- " -UX use a given filesystem UUID\n"
+ " -UX use a given filesystem UUID\n"
#endif
- " --exclude-path=X avoid including file X (X = exact literal path)\n"
- " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
+ " --exclude-path=X avoid including file X (X = exact literal path)\n"
+ " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
- " --file-contexts=X specify a file contexts file to setup selinux labels\n"
+ " --file-contexts=X specify a file contexts file to setup selinux labels\n"
+#endif
+ " --force-uid=# set all file uids to # (# = UID)\n"
+ " --force-gid=# set all file gids to # (# = GID)\n"
+ " --all-root make all files owned by root\n"
+ " --help display this help and exit\n"
+#ifndef NDEBUG
+ " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
#endif
- " --force-uid=# set all file uids to # (# = UID)\n"
- " --force-gid=# set all file gids to # (# = GID)\n"
- " --all-root make all files owned by root\n"
- " --help display this help and exit\n"
#ifdef WITH_ANDROID
"\nwith following android-specific options:\n"
- " --mount-point=X X=prefix of target fs path (default: /)\n"
- " --product-out=X X=product_out directory\n"
- " --fs-config-file=X X=fs_config file\n"
+ " --mount-point=X X=prefix of target fs path (default: /)\n"
+ " --product-out=X X=product_out directory\n"
+ " --fs-config-file=X X=fs_config file\n"
#endif
"\nAvailable compressors are: ", stderr);
print_available_compressors(stderr, ", ");
@@ -257,6 +263,11 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
case 7:
cfg.c_uid = cfg.c_gid = 0;
break;
+#ifndef NDEBUG
+ case 8:
+ cfg.c_random_pclusterblks = true;
+ break;
+#endif
#ifdef WITH_ANDROID
case 10:
cfg.mount_point = optarg;
@@ -523,7 +534,10 @@ int main(int argc, char **argv)
erofs_show_config();
erofs_set_fs_root(cfg.c_src_path);
-
+#ifndef NDEBUG
+ if (cfg.c_random_pclusterblks)
+ srand(time(NULL));
+#endif
sb_bh = erofs_buffer_init();
if (IS_ERR(sb_bh)) {
err = PTR_ERR(sb_bh);
--
2.20.1
^ permalink raw reply related [flat|nested] 4+ messages in thread