* [PATCH v4] erofs-utils: introduce segment compression
@ 2020-06-18 16:26 Li Guifu via Linux-erofs
2020-06-18 23:05 ` Gao Xiang
2020-06-19 17:51 ` [PATCH v5] " Li Guifu via Linux-erofs
0 siblings, 2 replies; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-06-18 16:26 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
Changes since v3 suggest by Gao Xiang<hsiangkao@gmx.com>:
- add 'S#' parameter to custome compression segment size
- move limit logic to size decrease
include/erofs/config.h | 1 +
lib/compress.c | 8 ++++++--
lib/config.c | 1 +
mkfs/main.c | 16 +++++++++++++++-
4 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..9125c1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ unsigned int c_compr_seg_size; /* max segment compress size */
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..eb024aa 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ unsigned int comprlimits;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -163,8 +164,7 @@ static int vle_compress_one(struct erofs_inode *inode,
goto nocompression;
break;
}
-
- count = len;
+ count = min(len, ctx->comprlimits);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -202,6 +202,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->comprlimits -= count;
+ if (!ctx->comprlimits)
+ ctx->comprlimits = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +425,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.comprlimits = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..1c39403 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = 1024U * EROFS_BLKSIZ;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..036d818 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# set the max input stream size # to one compress\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ cfg.c_compr_seg_size = strtol(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (cfg.c_compr_seg_size % EROFS_BLKSIZ != 0) {
+ erofs_err("segment size:%u should be align to %u",
+ cfg.c_compr_seg_size, EROFS_BLKSIZ);
+ return -EINVAL;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v4] erofs-utils: introduce segment compression
2020-06-18 16:26 [PATCH v4] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
@ 2020-06-18 23:05 ` Gao Xiang
2020-06-19 17:51 ` [PATCH v5] " Li Guifu via Linux-erofs
1 sibling, 0 replies; 11+ messages in thread
From: Gao Xiang @ 2020-06-18 23:05 UTC (permalink / raw)
To: Li Guifu; +Cc: linux-erofs
Hi Guifu,
On Fri, Jun 19, 2020 at 12:26:57AM +0800, Li Guifu via Linux-erofs wrote:
> Support segment compression which seperates files in several logic
> units (segments) and each segment is compressed independently.
>
> Advantages:
> - more friendly for data differencing;
> - it can also be used for parallel compression in the same file later.
>
> Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
> ---
> Changes since v3 suggest by Gao Xiang<hsiangkao@gmx.com>:
> - add 'S#' parameter to custome compression segment size
> - move limit logic to size decrease
>
> include/erofs/config.h | 1 +
> lib/compress.c | 8 ++++++--
> lib/config.c | 1 +
> mkfs/main.c | 16 +++++++++++++++-
Just do a quick response for this, and will test it later.
First, You might need to update the manpage as well.
> 4 files changed, 23 insertions(+), 3 deletions(-)
>
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index 2f09749..9125c1e 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -36,6 +36,7 @@ struct erofs_configure {
> char *c_src_path;
> char *c_compr_alg_master;
> int c_compr_level_master;
u64 c_compr_segsize;
> + unsigned int c_compr_seg_size; /* max segment compress size */
> int c_force_inodeversion;
> /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
> int c_inline_xattr_tolerance;
> diff --git a/lib/compress.c b/lib/compress.c
> index 6cc68ed..eb024aa 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
>
> erofs_blk_t blkaddr; /* pointing to the next blkaddr */
> u16 clusterofs;
> + unsigned int comprlimits;
How about the name "segavail"; ?
u64 segavail;
> };
>
> #define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
> @@ -163,8 +164,7 @@ static int vle_compress_one(struct erofs_inode *inode,
> goto nocompression;
> break;
> }
I think we might add "if (segavail < EROFS_BLKSIZE) goto nocompression;"
since it seems better.
> -
> - count = len;
> + count = min(len, ctx->comprlimits);
> ret = erofs_compress_destsize(h, compressionlevel,
> ctx->queue + ctx->head,
> &count, dst, EROFS_BLKSIZ);
> @@ -202,6 +202,9 @@ nocompression:
>
> ++ctx->blkaddr;
> len -= count;
> + ctx->comprlimits -= count;
> + if (!ctx->comprlimits)
> + ctx->comprlimits = cfg.c_compr_seg_size;
>
> if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
> const unsigned int qh_aligned =
> @@ -422,6 +425,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> ctx.head = ctx.tail = 0;
> ctx.clusterofs = 0;
> remaining = inode->i_size;
> + ctx.comprlimits = cfg.c_compr_seg_size;
>
> while (remaining) {
> const u64 readcount = min_t(u64, remaining,
> diff --git a/lib/config.c b/lib/config.c
> index da0c260..1c39403 100644
> --- a/lib/config.c
> +++ b/lib/config.c
> @@ -23,6 +23,7 @@ void erofs_init_configure(void)
> cfg.c_force_inodeversion = 0;
> cfg.c_inline_xattr_tolerance = 2;
> cfg.c_unix_timestamp = -1;
> + cfg.c_compr_seg_size = 1024U * EROFS_BLKSIZ;
We don't need that limit by default, so
cfg.c_compr_segsize = -1ULL;
> }
>
> void erofs_show_config(void)
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 94bf1e6..036d818 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -61,6 +61,7 @@ static void usage(void)
> " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> " -EX[,...] X=extended options\n"
> " -T# set a fixed UNIX timestamp # to all files\n"
> + " -S# set the max input stream size # to one compress\n"
> " --exclude-path=X avoid including file X (X = exact literal path)\n"
> " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
> #ifdef HAVE_LIBSELINUX
> @@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> char *endptr;
> int opt, i;
>
> - while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
> + while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
> long_options, NULL)) != -1) {
> switch (opt) {
> case 'z':
> @@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> return -EINVAL;
> }
> break;
> + case 'S':
> + cfg.c_compr_seg_size = strtol(optarg, &endptr, 0);
> + if (*endptr != '\0') {
> + erofs_err("invalid compress segment size %s",
> + optarg);
> + return -EINVAL;
> + }
> + if (cfg.c_compr_seg_size % EROFS_BLKSIZ != 0) {
> + erofs_err("segment size:%u should be align to %u",
> + cfg.c_compr_seg_size, EROFS_BLKSIZ);
> + return -EINVAL;
> + }
if (!cfg.c_compr_segsize)
cfg.c_compr_segsize = -1ULL;
else if (cfg.c_compr_segsize % EROFS_BLKSIZ) {
erofs_err("segmentsize %u should be aligned with blocksize %u",
cfg.c_compr_seg_size, EROFS_BLKSIZ);
return -EINVAL;
}
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v5] erofs-utils: introduce segment compression
2020-06-18 16:26 [PATCH v4] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
2020-06-18 23:05 ` Gao Xiang
@ 2020-06-19 17:51 ` Li Guifu via Linux-erofs
2020-06-19 23:03 ` Gao Xiang
2020-06-21 10:51 ` [PATCH v6] erofs-utils: introduce segment limits compression Li Guifu via Linux-erofs
1 sibling, 2 replies; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-06-19 17:51 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
include/erofs/config.h | 1 +
lib/compress.c | 16 ++++++++++++++--
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 16 +++++++++++++++-
5 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..995664d 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ u64 c_compr_seg_size;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..383ee00 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -158,13 +159,19 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ count = min_t(u64, len, ctx->segavail);
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+ goto nocompression;
+ }
+
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
- count = len;
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -174,8 +181,9 @@ static int vle_compress_one(struct erofs_inode *inode,
inode->i_srcpath,
erofs_strerror(ret));
}
+ count = len;
nocompression:
- ret = write_uncompressed_block(ctx, &len, dst);
+ ret = write_uncompressed_block(ctx, &count, dst);
if (ret < 0)
return ret;
count = ret;
@@ -202,6 +210,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->segavail -= count;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +433,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..de982e1 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = UINT64_MAX;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..b12cb22 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set the max input stream size at one compression. The default is unsigned 64bit MAX.
+It must be algin to EROFS block size(4096).
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..96cc053 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# set the max input stream size # at one compression\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ cfg.c_compr_seg_size = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0' || !cfg.c_compr_seg_size) {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (cfg.c_compr_seg_size % EROFS_BLKSIZ) {
+ erofs_err("segment size:%"PRIu64" should be align to %u",
+ cfg.c_compr_seg_size, EROFS_BLKSIZ);
+ return -EINVAL;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v5] erofs-utils: introduce segment compression
2020-06-19 17:51 ` [PATCH v5] " Li Guifu via Linux-erofs
@ 2020-06-19 23:03 ` Gao Xiang
2020-06-21 10:51 ` [PATCH v6] erofs-utils: introduce segment limits compression Li Guifu via Linux-erofs
1 sibling, 0 replies; 11+ messages in thread
From: Gao Xiang @ 2020-06-19 23:03 UTC (permalink / raw)
To: Li Guifu; +Cc: linux-erofs
On Sat, Jun 20, 2020 at 01:51:33AM +0800, Li Guifu via Linux-erofs wrote:
> Support segment compression which seperates files in several logic
> units (segments) and each segment is compressed independently.
>
> Advantages:
> - more friendly for data differencing;
> - it can also be used for parallel compression in the same file later.
>
> Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
> ---
> include/erofs/config.h | 1 +
> lib/compress.c | 16 ++++++++++++++--
> lib/config.c | 1 +
> man/mkfs.erofs.1 | 4 ++++
> mkfs/main.c | 16 +++++++++++++++-
> 5 files changed, 35 insertions(+), 3 deletions(-)
>
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index 2f09749..995664d 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -36,6 +36,7 @@ struct erofs_configure {
> char *c_src_path;
> char *c_compr_alg_master;
> int c_compr_level_master;
> + u64 c_compr_seg_size;
Could you please move this variable up a bit? Thanks.
char *c_compr_alg_master;
u64 c_compr_seg_size;
int c_compr_level_master;
> int c_force_inodeversion;
> /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
> int c_inline_xattr_tolerance;
> diff --git a/lib/compress.c b/lib/compress.c
> index 6cc68ed..383ee00 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
>
> erofs_blk_t blkaddr; /* pointing to the next blkaddr */
> u16 clusterofs;
> + u64 segavail;
> };
>
> #define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
> @@ -158,13 +159,19 @@ static int vle_compress_one(struct erofs_inode *inode,
> while (len) {
> bool raw;
unsigned int limit;
>
> + count = min_t(u64, len, ctx->segavail);
kill this line.
> + if (ctx->segavail <= EROFS_BLKSIZ) {
> + if (len < ctx->segavail && !final)
> + break;
limit = ctx->segavail;
> + goto nocompression;
> + }
> +
> if (len <= EROFS_BLKSIZ) {
> if (final)
> goto nocompression;
> break;
> }
>
> - count = len;
count = min_t(u64, len, ctx->segavail);
> ret = erofs_compress_destsize(h, compressionlevel,
> ctx->queue + ctx->head,
> &count, dst, EROFS_BLKSIZ);
> @@ -174,8 +181,9 @@ static int vle_compress_one(struct erofs_inode *inode,
> inode->i_srcpath,
> erofs_strerror(ret));
> }
> + count = len;
kill this line and add limit = EROFS_BLKSIZ;
> nocompression:
> - ret = write_uncompressed_block(ctx, &len, dst);
> + ret = write_uncompressed_block(ctx, &count, dst);
ret = write_uncompressed_block(ctx, &count, limit, dst);
and update write_uncompressed_block as well.
> if (ret < 0)
> return ret;
> count = ret;
> @@ -202,6 +210,9 @@ nocompression:
>
> ++ctx->blkaddr;
> len -= count;
> + ctx->segavail -= count;
> + if (!ctx->segavail)
> + ctx->segavail = cfg.c_compr_seg_size;
>
> if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
> const unsigned int qh_aligned =
> @@ -422,6 +433,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> ctx.head = ctx.tail = 0;
> ctx.clusterofs = 0;
> remaining = inode->i_size;
> + ctx.segavail = cfg.c_compr_seg_size;
>
> while (remaining) {
> const u64 readcount = min_t(u64, remaining,
> diff --git a/lib/config.c b/lib/config.c
> index da0c260..de982e1 100644
> --- a/lib/config.c
> +++ b/lib/config.c
> @@ -23,6 +23,7 @@ void erofs_init_configure(void)
> cfg.c_force_inodeversion = 0;
> cfg.c_inline_xattr_tolerance = 2;
> cfg.c_unix_timestamp = -1;
> + cfg.c_compr_seg_size = UINT64_MAX;
cfg.c_compr_seg_size = -1;
since it is a very simple way to assign UINT_MAX by implicit sign
extension without taking care for the specific data type.
> }
>
> void erofs_show_config(void)
> diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> index 891c5a8..b12cb22 100644
> --- a/man/mkfs.erofs.1
> +++ b/man/mkfs.erofs.1
> @@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
> Set all files to the given UNIX timestamp. Reproducible builds requires setting
> all to a specific one.
> .TP
> +.BI "\-S " #
> +Set the max input stream size at one compression. The default is unsigned 64bit MAX.
> +It must be algin to EROFS block size(4096).
it's hard for end users to type "max 64-bit unsigned value"...
I'd suggest "Set max input stream size for each individual segment (disabled if 0).
The default value is 0. It should be aligned with blocksize."
> +.TP
> .BI "\-\-exclude-path=" path
> Ignore file that matches the exact literal path.
> You may give multiple `--exclude-path' options.
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 94bf1e6..96cc053 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -61,6 +61,7 @@ static void usage(void)
> " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> " -EX[,...] X=extended options\n"
> " -T# set a fixed UNIX timestamp # to all files\n"
> + " -S# set the max input stream size # at one compression\n"
-S# Set max input stream size # for each individual segment\n
> " --exclude-path=X avoid including file X (X = exact literal path)\n"
> " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
> #ifdef HAVE_LIBSELINUX
> @@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> char *endptr;
> int opt, i;
>
> - while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
> + while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
> long_options, NULL)) != -1) {
> switch (opt) {
> case 'z':
> @@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> return -EINVAL;
> }
> break;
> + case 'S':
> + cfg.c_compr_seg_size = strtoll(optarg, &endptr, 0);
> + if (*endptr != '\0' || !cfg.c_compr_seg_size) {
Disable this if cfg.c_compr_seg_size == 0
> + erofs_err("invalid compress segment size %s",
> + optarg);
> + return -EINVAL;
> + }
> + if (cfg.c_compr_seg_size % EROFS_BLKSIZ) {
> + erofs_err("segment size:%"PRIu64" should be align to %u",
Could you follow my advice in the previous reply?
Although I'm not good at English, but I don't think the above message is _reasonable_.
Thanks,
Gao Xiang
> + cfg.c_compr_seg_size, EROFS_BLKSIZ);
> + return -EINVAL;
> + }
> + break;
> case 2:
> opt = erofs_parse_exclude_path(optarg, false);
> if (opt) {
> --
> 2.17.1
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v6] erofs-utils: introduce segment limits compression
2020-06-19 17:51 ` [PATCH v5] " Li Guifu via Linux-erofs
2020-06-19 23:03 ` Gao Xiang
@ 2020-06-21 10:51 ` Li Guifu via Linux-erofs
2020-06-21 12:27 ` [PATCH v8] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
1 sibling, 1 reply; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-06-21 10:51 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
Changes since v3 suggest by Gao Xiang<hsiangkao@gmx.com>:
- add a limits varialbe to give the limits size in the write_uncompress_block
- Set comments more readable
include/erofs/config.h | 1 +
lib/compress.c | 29 +++++++++++++++++++++--------
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 18 +++++++++++++++++-
5 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..8a79895 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,23 +117,21 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int limits,
char *dst)
{
int ret;
- unsigned int count;
+ unsigned int count = min(limits, *len); /* write uncompressed data */
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
+ count += ctx->clusterofs;
ctx->clusterofs = 0;
}
- /* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +156,22 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int limits = EROFS_BLKSIZ;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ limits = ctx->segavail;
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -175,7 +182,7 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, &len, dst);
+ ret = write_uncompressed_block(ctx, &len, limits, dst);
if (ret < 0)
return ret;
count = ret;
@@ -203,6 +210,11 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (count >= ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
+ else
+ ctx->segavail -= count;
+
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
round_down(ctx->head, EROFS_BLKSIZ);
@@ -422,6 +434,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..fbb2914 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = -1UL;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..0b613e4 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set max input stream size for each individual segment (disabled if 0).
+The default value is 0. It should be aligned with blocksize.
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..0e26f4f 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# Set max input stream size # for each individual segment\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ cfg.c_compr_seg_size = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (!cfg.c_compr_seg_size) {
+ cfg.c_compr_seg_size = -1UL;
+ } else if (cfg.c_compr_seg_size % EROFS_BLKSIZ) {
+ erofs_err("segment size:%"PRIu64" should be align to %u",
+ cfg.c_compr_seg_size, EROFS_BLKSIZ);
+ return -EINVAL;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v8] erofs-utils: introduce segment compression
2020-06-21 10:51 ` [PATCH v6] erofs-utils: introduce segment limits compression Li Guifu via Linux-erofs
@ 2020-06-21 12:27 ` Li Guifu via Linux-erofs
2020-06-21 14:37 ` Gao Xiang via Linux-erofs
0 siblings, 1 reply; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-06-21 12:27 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
include/erofs/config.h | 1 +
lib/compress.c | 47 ++++++++++++++++++++++++++----------------
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 18 +++++++++++++++-
5 files changed, 52 insertions(+), 19 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..6c0708d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,23 +117,11 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
- char *dst)
+ unsigned int count, char *dst)
{
int ret;
- unsigned int count;
-
- /* reset clusterofs to 0 if permitted */
- if (!erofs_sb_has_lz4_0padding() &&
- ctx->head >= ctx->clusterofs) {
- ctx->head -= ctx->clusterofs;
- *len += ctx->clusterofs;
- ctx->clusterofs = 0;
- }
-
- /* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
+ DBG_BUGON(count > EROFS_BLKSIZ);
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +146,22 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int limit = EROFS_BLKSIZ;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ limit = ctx->segavail;
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -175,10 +172,18 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, &len, dst);
+ /* reset clusterofs to 0 if permitted */
+ if (!erofs_sb_has_lz4_0padding() &&
+ ctx->head >= ctx->clusterofs) {
+ ctx->head -= ctx->clusterofs;
+ len += ctx->clusterofs;
+ limit += ctx->clusterofs;
+ ctx->clusterofs = 0;
+ }
+ count = min(limit, len);
+ ret = write_uncompressed_block(ctx, count, dst);
if (ret < 0)
return ret;
- count = ret;
raw = true;
} else {
/* write compressed data */
@@ -203,6 +208,11 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (count >= ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
+ else
+ ctx->segavail -= count;
+
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
round_down(ctx->head, EROFS_BLKSIZ);
@@ -422,6 +432,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..721ff61 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = -1;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..0b613e4 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set max input stream size for each individual segment (disabled if 0).
+The default value is 0. It should be aligned with blocksize.
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..dcb01cc 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# Set max input stream size # for each individual segment\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ cfg.c_compr_seg_size = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (!cfg.c_compr_seg_size) {
+ cfg.c_compr_seg_size = -1;
+ } else if (cfg.c_compr_seg_size % EROFS_BLKSIZ) {
+ erofs_err("segment size:%"PRIu64" should be align with %u",
+ cfg.c_compr_seg_size, EROFS_BLKSIZ);
+ return -EINVAL;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v8] erofs-utils: introduce segment compression
2020-06-21 12:27 ` [PATCH v8] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
@ 2020-06-21 14:37 ` Gao Xiang via Linux-erofs
2020-06-30 17:27 ` [PATCH v10] " Li Guifu via Linux-erofs
0 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2020-06-21 14:37 UTC (permalink / raw)
To: Li Guifu; +Cc: linux-erofs
On Sun, Jun 21, 2020 at 08:27:45PM +0800, Li Guifu via Linux-erofs wrote:
...
> nocompression:
> - ret = write_uncompressed_block(ctx, &len, dst);
> + /* reset clusterofs to 0 if permitted */
> + if (!erofs_sb_has_lz4_0padding() &&
> + ctx->head >= ctx->clusterofs) {
> + ctx->head -= ctx->clusterofs;
> + len += ctx->clusterofs;
> + limit += ctx->clusterofs;
> + ctx->clusterofs = 0;
TL;DR: it seems still buggy here.
Pls rethink carefully and send a usable patch then...
Thanks,
Gao Xiang
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH v10] erofs-utils: introduce segment compression
2020-06-21 14:37 ` Gao Xiang via Linux-erofs
@ 2020-06-30 17:27 ` Li Guifu via Linux-erofs
2020-07-05 8:32 ` [PATCH v11] " Li Guifu via Linux-erofs
0 siblings, 1 reply; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-06-30 17:27 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
include/erofs/config.h | 1 +
lib/compress.c | 29 +++++++++++++++++++++++------
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 16 +++++++++++++++-
5 files changed, 44 insertions(+), 7 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..2ea5809 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int *ucomproft,
char *dst)
{
int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
+ *ucomproft = ctx->clusterofs;
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
ctx->clusterofs = 0;
+ count = min(EROFS_BLKSIZ, *len);
+ } else {
+ *ucomproft = 0;
+ count = min_t(u64, ctx->segavail, *len);
+ if (count > EROFS_BLKSIZ)
+ count = EROFS_BLKSIZ;
}
/* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int ucomproft = 0;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -175,7 +188,7 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, &len, dst);
+ ret = write_uncompressed_block(ctx, &len, &ucomproft, dst);
if (ret < 0)
return ret;
count = ret;
@@ -202,6 +215,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->segavail -= count - ucomproft;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +438,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..721ff61 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = -1;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..2a4ef71 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set max input stream size for each individual segment (disabled if 0).
+The default value is 0. It has be aligned with blocksize.
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..cded973 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# Set max input stream size # for each individual segment\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ i = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (!i) {
+ cfg.c_compr_seg_size = -1;
+ } else {
+ cfg.c_compr_seg_size = i * EROFS_BLKSIZ;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v11] erofs-utils: introduce segment compression
2020-06-30 17:27 ` [PATCH v10] " Li Guifu via Linux-erofs
@ 2020-07-05 8:32 ` Li Guifu via Linux-erofs
2020-07-05 18:20 ` Gao Xiang via Linux-erofs
0 siblings, 1 reply; 11+ messages in thread
From: Li Guifu via Linux-erofs @ 2020-07-05 8:32 UTC (permalink / raw)
To: linux-erofs; +Cc: Li Guifu
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
---
Changes from v10
- chang variable uncomprofs to clusterofs which only used
when write uncompress block
include/erofs/config.h | 1 +
lib/compress.c | 30 ++++++++++++++++++++++++------
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 16 +++++++++++++++-
5 files changed, 45 insertions(+), 7 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..a2a278c 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int *clusterofs,
char *dst)
{
int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
+ *clusterofs = ctx->clusterofs;
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
ctx->clusterofs = 0;
+ count = min(EROFS_BLKSIZ, *len);
+ } else {
+ *clusterofs = 0;
+ count = min_t(u64, ctx->segavail, *len);
+ if (count > EROFS_BLKSIZ)
+ count = EROFS_BLKSIZ;
}
/* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int clusterofs;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -175,11 +188,12 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, &len, dst);
+ ret = write_uncompressed_block(ctx, &len, &clusterofs, dst);
if (ret < 0)
return ret;
count = ret;
raw = true;
+ ctx->segavail -= count - clusterofs;
} else {
/* write compressed data */
erofs_dbg("Writing %u compressed data to block %u",
@@ -194,6 +208,7 @@ nocompression:
if (ret)
return ret;
raw = false;
+ ctx->segavail -= count;
}
ctx->head += count;
@@ -202,6 +217,8 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +439,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..721ff61 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = -1;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..2a4ef71 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set max input stream size for each individual segment (disabled if 0).
+The default value is 0. It has be aligned with blocksize.
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..cded973 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# Set max input stream size # for each individual segment\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ i = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (!i) {
+ cfg.c_compr_seg_size = -1;
+ } else {
+ cfg.c_compr_seg_size = i * EROFS_BLKSIZ;
+ }
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.17.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v11] erofs-utils: introduce segment compression
2020-07-05 8:32 ` [PATCH v11] " Li Guifu via Linux-erofs
@ 2020-07-05 18:20 ` Gao Xiang via Linux-erofs
2020-10-07 15:04 ` Gao Xiang via Linux-erofs
0 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2020-07-05 18:20 UTC (permalink / raw)
To: Li Guifu, Li Guifu; +Cc: linux-erofs
Hi Guifu,
On Sun, Jul 05, 2020 at 04:32:30PM +0800, Li Guifu via Linux-erofs wrote:
> Support segment compression which seperates files in several logic
> units (segments) and each segment is compressed independently.
>
> Advantages:
> - more friendly for data differencing;
> - it can also be used for parallel compression in the same file later.
>
> Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
> ---
> Changes from v10
> - chang variable uncomprofs to clusterofs which only used
> when write uncompress block
Could you please test the following patch if you're available?
Does it work?
From 0436ed04717853351e13d68db6f170f60e25fc12 Mon Sep 17 00:00:00 2001
From: Li Guifu <bluce.lee@aliyun.com>
Date: Sun, 5 Jul 2020 16:32:30 +0800
Subject: [PATCH v12] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
---
include/erofs/config.h | 2 ++
lib/compress.c | 38 ++++++++++++++++++++++++++++++--------
lib/config.c | 1 +
man/mkfs.erofs.1 | 4 ++++
mkfs/main.c | 12 +++++++++++-
5 files changed, 48 insertions(+), 9 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..b149633 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,8 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_segsize;
+
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..4216fa7 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -124,24 +125,33 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
- ctx->head >= ctx->clusterofs) {
+ ctx->clusterofs && ctx->head >= ctx->clusterofs) {
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
+
+ ctx->segavail += ctx->clusterofs;
+ DBG_BUGON(ctx->segavail > cfg.c_compr_segsize);
+
+ DBG_BUGON(ctx->segavail < EROFS_BLKSIZ);
+ /* so only *len will be the candidate instead of segavail */
+ count = *len;
+
ctx->clusterofs = 0;
+ } else {
+ count = min_t(u64, ctx->segavail, *len);
}
- /* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
+ if (count > EROFS_BLKSIZ)
+ count = EROFS_BLKSIZ;
+ /* fill zero if the uncompressed block isn't full */
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
erofs_dbg("Writing %u uncompressed data to block %u",
count, ctx->blkaddr);
ret = blk_write(dst, ctx->blkaddr, 1);
- if (ret)
- return ret;
- return count;
+ return ret ? ret : count;
}
static int vle_compress_one(struct erofs_inode *inode,
@@ -158,13 +168,20 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ goto nocompression;
+ }
+
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
@@ -195,8 +212,12 @@ nocompression:
return ret;
raw = false;
}
-
ctx->head += count;
+ DBG_BUGON(ctx->segavail < count);
+ ctx->segavail -= count;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_segsize;
+
/* write compression indexes for this blkaddr */
vle_write_indexes(ctx, count, raw);
@@ -421,6 +442,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE;
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
+ ctx.segavail = cfg.c_compr_segsize;
remaining = inode->i_size;
while (remaining) {
diff --git a/lib/config.c b/lib/config.c
index da0c260..9d4bea1 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_segsize = -1;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..8d0fc10 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -48,6 +48,10 @@ Forcely generate compact inodes (32-byte inodes) to output.
Forcely generate extended inodes (64-byte inodes) to output.
.RE
.TP
+.BI "\-S " #
+Set maximum blocks for each individual compress segment.
+The default is 0 (disabled).
+.TP
.BI "\-T " #
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..0265ae9 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# Set maximum blocks for each individual compress segment\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ i = strtoll(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid blocks per compress segment %s",
+ optarg);
+ return -EINVAL;
+ }
+ cfg.c_compr_segsize = i ? blknr_to_addr(i) : -1;
+ break;
case 2:
opt = erofs_parse_exclude_path(optarg, false);
if (opt) {
--
2.24.0
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH v11] erofs-utils: introduce segment compression
2020-07-05 18:20 ` Gao Xiang via Linux-erofs
@ 2020-10-07 15:04 ` Gao Xiang via Linux-erofs
0 siblings, 0 replies; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2020-10-07 15:04 UTC (permalink / raw)
To: Li Guifu, Li Guifu, linux-erofs
Guifu, some progress on this as well?
On Mon, Jul 06, 2020 at 02:20:50AM +0800, Gao Xiang via Linux-erofs wrote:
> Hi Guifu,
>
> On Sun, Jul 05, 2020 at 04:32:30PM +0800, Li Guifu via Linux-erofs wrote:
> > Support segment compression which seperates files in several logic
> > units (segments) and each segment is compressed independently.
> >
> > Advantages:
> > - more friendly for data differencing;
> > - it can also be used for parallel compression in the same file later.
> >
> > Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
> > ---
> > Changes from v10
> > - chang variable uncomprofs to clusterofs which only used
> > when write uncompress block
>
> Could you please test the following patch if you're available?
> Does it work?
>
> From 0436ed04717853351e13d68db6f170f60e25fc12 Mon Sep 17 00:00:00 2001
> From: Li Guifu <bluce.lee@aliyun.com>
> Date: Sun, 5 Jul 2020 16:32:30 +0800
> Subject: [PATCH v12] erofs-utils: introduce segment compression
>
> Support segment compression which seperates files in several logic
> units (segments) and each segment is compressed independently.
>
> Advantages:
> - more friendly for data differencing;
> - it can also be used for parallel compression in the same file later.
>
> Signed-off-by: Li Guifu <bluce.lee@aliyun.com>
> Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
> ---
> include/erofs/config.h | 2 ++
> lib/compress.c | 38 ++++++++++++++++++++++++++++++--------
> lib/config.c | 1 +
> man/mkfs.erofs.1 | 4 ++++
> mkfs/main.c | 12 +++++++++++-
> 5 files changed, 48 insertions(+), 9 deletions(-)
>
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index 2f09749..b149633 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -35,6 +35,8 @@ struct erofs_configure {
> char *c_img_path;
> char *c_src_path;
> char *c_compr_alg_master;
> + u64 c_compr_segsize;
> +
> int c_compr_level_master;
> int c_force_inodeversion;
> /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
> diff --git a/lib/compress.c b/lib/compress.c
> index 6cc68ed..4216fa7 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
>
> erofs_blk_t blkaddr; /* pointing to the next blkaddr */
> u16 clusterofs;
> + u64 segavail;
> };
>
> #define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
> @@ -124,24 +125,33 @@ static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
>
> /* reset clusterofs to 0 if permitted */
> if (!erofs_sb_has_lz4_0padding() &&
> - ctx->head >= ctx->clusterofs) {
> + ctx->clusterofs && ctx->head >= ctx->clusterofs) {
> ctx->head -= ctx->clusterofs;
> *len += ctx->clusterofs;
> +
> + ctx->segavail += ctx->clusterofs;
> + DBG_BUGON(ctx->segavail > cfg.c_compr_segsize);
> +
> + DBG_BUGON(ctx->segavail < EROFS_BLKSIZ);
> + /* so only *len will be the candidate instead of segavail */
> + count = *len;
> +
> ctx->clusterofs = 0;
> + } else {
> + count = min_t(u64, ctx->segavail, *len);
> }
>
> - /* write uncompressed data */
> - count = min(EROFS_BLKSIZ, *len);
> + if (count > EROFS_BLKSIZ)
> + count = EROFS_BLKSIZ;
>
> + /* fill zero if the uncompressed block isn't full */
> memcpy(dst, ctx->queue + ctx->head, count);
> memset(dst + count, 0, EROFS_BLKSIZ - count);
>
> erofs_dbg("Writing %u uncompressed data to block %u",
> count, ctx->blkaddr);
> ret = blk_write(dst, ctx->blkaddr, 1);
> - if (ret)
> - return ret;
> - return count;
> + return ret ? ret : count;
> }
>
> static int vle_compress_one(struct erofs_inode *inode,
> @@ -158,13 +168,20 @@ static int vle_compress_one(struct erofs_inode *inode,
> while (len) {
> bool raw;
>
> + if (ctx->segavail <= EROFS_BLKSIZ) {
> + if (len < ctx->segavail && !final)
> + break;
> +
> + goto nocompression;
> + }
> +
> if (len <= EROFS_BLKSIZ) {
> if (final)
> goto nocompression;
> break;
> }
>
> - count = len;
> + count = min_t(u64, len, ctx->segavail);
> ret = erofs_compress_destsize(h, compressionlevel,
> ctx->queue + ctx->head,
> &count, dst, EROFS_BLKSIZ);
> @@ -195,8 +212,12 @@ nocompression:
> return ret;
> raw = false;
> }
> -
> ctx->head += count;
> + DBG_BUGON(ctx->segavail < count);
> + ctx->segavail -= count;
> + if (!ctx->segavail)
> + ctx->segavail = cfg.c_compr_segsize;
> +
> /* write compression indexes for this blkaddr */
> vle_write_indexes(ctx, count, raw);
>
> @@ -421,6 +442,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE;
> ctx.head = ctx.tail = 0;
> ctx.clusterofs = 0;
> + ctx.segavail = cfg.c_compr_segsize;
> remaining = inode->i_size;
>
> while (remaining) {
> diff --git a/lib/config.c b/lib/config.c
> index da0c260..9d4bea1 100644
> --- a/lib/config.c
> +++ b/lib/config.c
> @@ -23,6 +23,7 @@ void erofs_init_configure(void)
> cfg.c_force_inodeversion = 0;
> cfg.c_inline_xattr_tolerance = 2;
> cfg.c_unix_timestamp = -1;
> + cfg.c_compr_segsize = -1;
> }
>
> void erofs_show_config(void)
> diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> index 891c5a8..8d0fc10 100644
> --- a/man/mkfs.erofs.1
> +++ b/man/mkfs.erofs.1
> @@ -48,6 +48,10 @@ Forcely generate compact inodes (32-byte inodes) to output.
> Forcely generate extended inodes (64-byte inodes) to output.
> .RE
> .TP
> +.BI "\-S " #
> +Set maximum blocks for each individual compress segment.
> +The default is 0 (disabled).
> +.TP
> .BI "\-T " #
> Set all files to the given UNIX timestamp. Reproducible builds requires setting
> all to a specific one.
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 94bf1e6..0265ae9 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -61,6 +61,7 @@ static void usage(void)
> " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> " -EX[,...] X=extended options\n"
> " -T# set a fixed UNIX timestamp # to all files\n"
> + " -S# Set maximum blocks for each individual compress segment\n"
> " --exclude-path=X avoid including file X (X = exact literal path)\n"
> " --exclude-regex=X avoid including files that match X (X = regular expression)\n"
> #ifdef HAVE_LIBSELINUX
> @@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> char *endptr;
> int opt, i;
>
> - while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
> + while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
> long_options, NULL)) != -1) {
> switch (opt) {
> case 'z':
> @@ -188,6 +189,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> return -EINVAL;
> }
> break;
> + case 'S':
> + i = strtoll(optarg, &endptr, 0);
> + if (*endptr != '\0') {
> + erofs_err("invalid blocks per compress segment %s",
> + optarg);
> + return -EINVAL;
> + }
> + cfg.c_compr_segsize = i ? blknr_to_addr(i) : -1;
> + break;
> case 2:
> opt = erofs_parse_exclude_path(optarg, false);
> if (opt) {
> --
> 2.24.0
>
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
end of thread, other threads:[~2020-10-07 15:04 UTC | newest]
Thread overview: 11+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-06-18 16:26 [PATCH v4] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
2020-06-18 23:05 ` Gao Xiang
2020-06-19 17:51 ` [PATCH v5] " Li Guifu via Linux-erofs
2020-06-19 23:03 ` Gao Xiang
2020-06-21 10:51 ` [PATCH v6] erofs-utils: introduce segment limits compression Li Guifu via Linux-erofs
2020-06-21 12:27 ` [PATCH v8] erofs-utils: introduce segment compression Li Guifu via Linux-erofs
2020-06-21 14:37 ` Gao Xiang via Linux-erofs
2020-06-30 17:27 ` [PATCH v10] " Li Guifu via Linux-erofs
2020-07-05 8:32 ` [PATCH v11] " Li Guifu via Linux-erofs
2020-07-05 18:20 ` Gao Xiang via Linux-erofs
2020-10-07 15:04 ` Gao Xiang via Linux-erofs
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).