* [PATCH 1/4] erofs: introduce erofs_sb_has_xxx() helpers
2021-03-27 3:49 ` [PATCH 0/4] erofs: introduce on-disk compression configurations Gao Xiang via Linux-erofs
@ 2021-03-27 3:49 ` Gao Xiang via Linux-erofs
2021-03-27 9:33 ` Chao Yu
2021-03-27 3:49 ` [PATCH 2/4] erofs: support adjust lz4 history window size Gao Xiang via Linux-erofs
` (2 subsequent siblings)
3 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-03-27 3:49 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Chao Yu; +Cc: LKML
From: Gao Xiang <hsiangkao@redhat.com>
Introduce erofs_sb_has_xxx() to make long checks short, especially
for later big pcluster & LZMA features.
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
---
fs/erofs/decompressor.c | 3 +--
fs/erofs/internal.h | 9 +++++++++
fs/erofs/super.c | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 34e73ff76f89..80e8871aef71 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -124,8 +124,7 @@ static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)
support_0padding = false;
/* decompression inplace is only safe when 0padding is enabled */
- if (EROFS_SB(rq->sb)->feature_incompat &
- EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) {
+ if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) {
support_0padding = true;
while (!src[inputmargin & ~PAGE_MASK])
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 30e63b73a675..d29fc0c56032 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -218,6 +218,15 @@ static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
}
+#define EROFS_FEATURE_FUNCS(name, compat, feature) \
+static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
+{ \
+ return sbi->feature_##compat & EROFS_FEATURE_##feature; \
+}
+
+EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
+EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
+
/* atomic flag definitions */
#define EROFS_I_EA_INITED_BIT 0
#define EROFS_I_Z_INITED_BIT 1
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 0445d09b6331..991b99eaf22a 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -149,7 +149,7 @@ static int erofs_read_superblock(struct super_block *sb)
}
sbi->feature_compat = le32_to_cpu(dsb->feature_compat);
- if (sbi->feature_compat & EROFS_FEATURE_COMPAT_SB_CHKSUM) {
+ if (erofs_sb_has_sb_chksum(sbi)) {
ret = erofs_superblock_csum_verify(sb, data);
if (ret)
goto out;
--
2.20.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 2/4] erofs: support adjust lz4 history window size
2021-03-27 3:49 ` [PATCH 0/4] erofs: introduce on-disk compression configurations Gao Xiang via Linux-erofs
2021-03-27 3:49 ` [PATCH 1/4] erofs: introduce erofs_sb_has_xxx() helpers Gao Xiang via Linux-erofs
@ 2021-03-27 3:49 ` Gao Xiang via Linux-erofs
2021-03-27 9:34 ` Chao Yu
2021-03-27 3:49 ` [PATCH 3/4] erofs: introduce on-disk lz4 fs configurations Gao Xiang via Linux-erofs
2021-03-27 3:49 ` [PATCH 4/4] erofs: add on-disk compression configurations Gao Xiang via Linux-erofs
3 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-03-27 3:49 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Chao Yu; +Cc: Guo Weichao, LKML
From: Huang Jianan <huangjianan@oppo.com>
lz4 uses LZ4_DISTANCE_MAX to record history preservation. When
using rolling decompression, a block with a higher compression
ratio will cause a larger memory allocation (up to 64k). It may
cause a large resource burden in extreme cases on devices with
small memory and a large number of concurrent IOs. So appropriately
reducing this value can improve performance.
Decreasing this value will reduce the compression ratio (except
when input_size <LZ4_DISTANCE_MAX). But considering that erofs
currently only supports 4k output, reducing this value will not
significantly reduce the compression benefits.
The maximum value of LZ4_DISTANCE_MAX defined by lz4 is 64k, and
we can only reduce this value. For the old kernel, it just can't
reduce the memory allocation during rolling decompression without
affecting the decompression result.
Signed-off-by: Huang Jianan <huangjianan@oppo.com>
Signed-off-by: Guo Weichao <guoweichao@oppo.com>
[ Gao Xiang: introduce struct erofs_sb_lz4_info for configurations. ]
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
---
fs/erofs/decompressor.c | 21 +++++++++++++++++----
fs/erofs/erofs_fs.h | 3 ++-
fs/erofs/internal.h | 19 +++++++++++++++++++
fs/erofs/super.c | 4 +++-
4 files changed, 41 insertions(+), 6 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 80e8871aef71..93411e9df9b6 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -28,6 +28,17 @@ struct z_erofs_decompressor {
char *name;
};
+int z_erofs_load_lz4_config(struct super_block *sb,
+ struct erofs_super_block *dsb)
+{
+ u16 distance = le16_to_cpu(dsb->lz4_max_distance);
+
+ EROFS_SB(sb)->lz4.max_distance_pages = distance ?
+ DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
+ LZ4_MAX_DISTANCE_PAGES;
+ return 0;
+}
+
static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct list_head *pagepool)
{
@@ -36,6 +47,8 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
BITS_PER_LONG)] = { 0 };
+ unsigned int lz4_max_distance_pages =
+ EROFS_SB(rq->sb)->lz4.max_distance_pages;
void *kaddr = NULL;
unsigned int i, j, top;
@@ -44,14 +57,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct page *const page = rq->out[i];
struct page *victim;
- if (j >= LZ4_MAX_DISTANCE_PAGES)
+ if (j >= lz4_max_distance_pages)
j = 0;
/* 'valid' bounced can only be tested after a complete round */
if (test_bit(j, bounced)) {
- DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
- DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
- availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
+ DBG_BUGON(i < lz4_max_distance_pages);
+ DBG_BUGON(top >= lz4_max_distance_pages);
+ availables[top++] = rq->out[i - lz4_max_distance_pages];
}
if (page) {
diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
index 9ad1615f4474..b27d0e4e4ab5 100644
--- a/fs/erofs/erofs_fs.h
+++ b/fs/erofs/erofs_fs.h
@@ -39,7 +39,8 @@ struct erofs_super_block {
__u8 uuid[16]; /* 128-bit uuid for volume */
__u8 volume_name[16]; /* volume name */
__le32 feature_incompat;
- __u8 reserved2[44];
+ __le16 lz4_max_distance;
+ __u8 reserved2[42];
};
/*
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index d29fc0c56032..1de60992c3dd 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -59,6 +59,12 @@ struct erofs_fs_context {
unsigned int mount_opt;
};
+/* all filesystem-wide lz4 configurations */
+struct erofs_sb_lz4_info {
+ /* # of pages needed for EROFS lz4 rolling decompression */
+ u16 max_distance_pages;
+};
+
struct erofs_sb_info {
#ifdef CONFIG_EROFS_FS_ZIP
/* list for all registered superblocks, mainly for shrinker */
@@ -72,6 +78,8 @@ struct erofs_sb_info {
/* pseudo inode to manage cached pages */
struct inode *managed_cache;
+
+ struct erofs_sb_lz4_info lz4;
#endif /* CONFIG_EROFS_FS_ZIP */
u32 blocks;
u32 meta_blkaddr;
@@ -432,6 +440,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
struct erofs_workgroup *egrp);
int erofs_try_to_free_cached_page(struct address_space *mapping,
struct page *page);
+int z_erofs_load_lz4_config(struct super_block *sb,
+ struct erofs_super_block *dsb);
#else
static inline void erofs_shrinker_register(struct super_block *sb) {}
static inline void erofs_shrinker_unregister(struct super_block *sb) {}
@@ -439,6 +449,15 @@ static inline int erofs_init_shrinker(void) { return 0; }
static inline void erofs_exit_shrinker(void) {}
static inline int z_erofs_init_zip_subsystem(void) { return 0; }
static inline void z_erofs_exit_zip_subsystem(void) {}
+static inline int z_erofs_load_lz4_config(struct super_block *sb,
+ struct erofs_super_block *dsb)
+{
+ if (dsb->lz4_max_distance) {
+ erofs_err(sb, "lz4 algorithm isn't enabled");
+ return -EINVAL;
+ }
+ return 0;
+}
#endif /* !CONFIG_EROFS_FS_ZIP */
#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 991b99eaf22a..3212e4f73f85 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -187,7 +187,9 @@ static int erofs_read_superblock(struct super_block *sb)
ret = -EFSCORRUPTED;
goto out;
}
- ret = 0;
+
+ /* parse on-disk compression configurations */
+ ret = z_erofs_load_lz4_config(sb, dsb);
out:
kunmap(page);
put_page(page);
--
2.20.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 2/4] erofs: support adjust lz4 history window size
2021-03-27 3:49 ` [PATCH 2/4] erofs: support adjust lz4 history window size Gao Xiang via Linux-erofs
@ 2021-03-27 9:34 ` Chao Yu
2021-03-27 10:15 ` Gao Xiang
0 siblings, 1 reply; 11+ messages in thread
From: Chao Yu @ 2021-03-27 9:34 UTC (permalink / raw)
To: Gao Xiang, linux-erofs, Chao Yu; +Cc: Guo Weichao, LKML
On 2021/3/27 11:49, Gao Xiang wrote:
> From: Huang Jianan <huangjianan@oppo.com>
>
> lz4 uses LZ4_DISTANCE_MAX to record history preservation. When
> using rolling decompression, a block with a higher compression
> ratio will cause a larger memory allocation (up to 64k). It may
> cause a large resource burden in extreme cases on devices with
> small memory and a large number of concurrent IOs. So appropriately
> reducing this value can improve performance.
>
> Decreasing this value will reduce the compression ratio (except
> when input_size <LZ4_DISTANCE_MAX). But considering that erofs
> currently only supports 4k output, reducing this value will not
> significantly reduce the compression benefits.
>
> The maximum value of LZ4_DISTANCE_MAX defined by lz4 is 64k, and
> we can only reduce this value. For the old kernel, it just can't
> reduce the memory allocation during rolling decompression without
> affecting the decompression result.
>
> Signed-off-by: Huang Jianan <huangjianan@oppo.com>
> Signed-off-by: Guo Weichao <guoweichao@oppo.com>
> [ Gao Xiang: introduce struct erofs_sb_lz4_info for configurations. ]
> Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
> ---
> fs/erofs/decompressor.c | 21 +++++++++++++++++----
> fs/erofs/erofs_fs.h | 3 ++-
> fs/erofs/internal.h | 19 +++++++++++++++++++
> fs/erofs/super.c | 4 +++-
> 4 files changed, 41 insertions(+), 6 deletions(-)
>
> diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
> index 80e8871aef71..93411e9df9b6 100644
> --- a/fs/erofs/decompressor.c
> +++ b/fs/erofs/decompressor.c
> @@ -28,6 +28,17 @@ struct z_erofs_decompressor {
> char *name;
> };
>
> +int z_erofs_load_lz4_config(struct super_block *sb,
> + struct erofs_super_block *dsb)
> +{
> + u16 distance = le16_to_cpu(dsb->lz4_max_distance);
> +
> + EROFS_SB(sb)->lz4.max_distance_pages = distance ?
> + DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
> + LZ4_MAX_DISTANCE_PAGES;
> + return 0;
> +}
> +
> static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> struct list_head *pagepool)
> {
> @@ -36,6 +47,8 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
> unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
> BITS_PER_LONG)] = { 0 };
> + unsigned int lz4_max_distance_pages =
> + EROFS_SB(rq->sb)->lz4.max_distance_pages;
> void *kaddr = NULL;
> unsigned int i, j, top;
>
> @@ -44,14 +57,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> struct page *const page = rq->out[i];
> struct page *victim;
>
> - if (j >= LZ4_MAX_DISTANCE_PAGES)
> + if (j >= lz4_max_distance_pages)
> j = 0;
>
> /* 'valid' bounced can only be tested after a complete round */
> if (test_bit(j, bounced)) {
> - DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
> - DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
> - availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
> + DBG_BUGON(i < lz4_max_distance_pages);
> + DBG_BUGON(top >= lz4_max_distance_pages);
> + availables[top++] = rq->out[i - lz4_max_distance_pages];
> }
>
> if (page) {
> diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
> index 9ad1615f4474..b27d0e4e4ab5 100644
> --- a/fs/erofs/erofs_fs.h
> +++ b/fs/erofs/erofs_fs.h
> @@ -39,7 +39,8 @@ struct erofs_super_block {
> __u8 uuid[16]; /* 128-bit uuid for volume */
> __u8 volume_name[16]; /* volume name */
> __le32 feature_incompat;
> - __u8 reserved2[44];
> + __le16 lz4_max_distance;
It missed to add comments, otherwise it looks good to me.
Reviewed-by: Chao Yu <yuchao0@huawei.com>
Thanks,
> + __u8 reserved2[42];
> };
>
> /*
> diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
> index d29fc0c56032..1de60992c3dd 100644
> --- a/fs/erofs/internal.h
> +++ b/fs/erofs/internal.h
> @@ -59,6 +59,12 @@ struct erofs_fs_context {
> unsigned int mount_opt;
> };
>
> +/* all filesystem-wide lz4 configurations */
> +struct erofs_sb_lz4_info {
> + /* # of pages needed for EROFS lz4 rolling decompression */
> + u16 max_distance_pages;
> +};
> +
> struct erofs_sb_info {
> #ifdef CONFIG_EROFS_FS_ZIP
> /* list for all registered superblocks, mainly for shrinker */
> @@ -72,6 +78,8 @@ struct erofs_sb_info {
>
> /* pseudo inode to manage cached pages */
> struct inode *managed_cache;
> +
> + struct erofs_sb_lz4_info lz4;
> #endif /* CONFIG_EROFS_FS_ZIP */
> u32 blocks;
> u32 meta_blkaddr;
> @@ -432,6 +440,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
> struct erofs_workgroup *egrp);
> int erofs_try_to_free_cached_page(struct address_space *mapping,
> struct page *page);
> +int z_erofs_load_lz4_config(struct super_block *sb,
> + struct erofs_super_block *dsb);
> #else
> static inline void erofs_shrinker_register(struct super_block *sb) {}
> static inline void erofs_shrinker_unregister(struct super_block *sb) {}
> @@ -439,6 +449,15 @@ static inline int erofs_init_shrinker(void) { return 0; }
> static inline void erofs_exit_shrinker(void) {}
> static inline int z_erofs_init_zip_subsystem(void) { return 0; }
> static inline void z_erofs_exit_zip_subsystem(void) {}
> +static inline int z_erofs_load_lz4_config(struct super_block *sb,
> + struct erofs_super_block *dsb)
> +{
> + if (dsb->lz4_max_distance) {
> + erofs_err(sb, "lz4 algorithm isn't enabled");
> + return -EINVAL;
> + }
> + return 0;
> +}
> #endif /* !CONFIG_EROFS_FS_ZIP */
>
> #define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */
> diff --git a/fs/erofs/super.c b/fs/erofs/super.c
> index 991b99eaf22a..3212e4f73f85 100644
> --- a/fs/erofs/super.c
> +++ b/fs/erofs/super.c
> @@ -187,7 +187,9 @@ static int erofs_read_superblock(struct super_block *sb)
> ret = -EFSCORRUPTED;
> goto out;
> }
> - ret = 0;
> +
> + /* parse on-disk compression configurations */
> + ret = z_erofs_load_lz4_config(sb, dsb);
> out:
> kunmap(page);
> put_page(page);
>
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 2/4] erofs: support adjust lz4 history window size
2021-03-27 9:34 ` Chao Yu
@ 2021-03-27 10:15 ` Gao Xiang
0 siblings, 0 replies; 11+ messages in thread
From: Gao Xiang @ 2021-03-27 10:15 UTC (permalink / raw)
To: Chao Yu; +Cc: LKML, Guo Weichao, linux-erofs
Hi Chao,
On Sat, Mar 27, 2021 at 05:34:33PM +0800, Chao Yu wrote:
> On 2021/3/27 11:49, Gao Xiang wrote:
> > From: Huang Jianan <huangjianan@oppo.com>
> >
> > lz4 uses LZ4_DISTANCE_MAX to record history preservation. When
> > using rolling decompression, a block with a higher compression
> > ratio will cause a larger memory allocation (up to 64k). It may
> > cause a large resource burden in extreme cases on devices with
> > small memory and a large number of concurrent IOs. So appropriately
> > reducing this value can improve performance.
> >
> > Decreasing this value will reduce the compression ratio (except
> > when input_size <LZ4_DISTANCE_MAX). But considering that erofs
> > currently only supports 4k output, reducing this value will not
> > significantly reduce the compression benefits.
> >
> > The maximum value of LZ4_DISTANCE_MAX defined by lz4 is 64k, and
> > we can only reduce this value. For the old kernel, it just can't
> > reduce the memory allocation during rolling decompression without
> > affecting the decompression result.
> >
> > Signed-off-by: Huang Jianan <huangjianan@oppo.com>
> > Signed-off-by: Guo Weichao <guoweichao@oppo.com>
> > [ Gao Xiang: introduce struct erofs_sb_lz4_info for configurations. ]
> > Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
> > ---
> > fs/erofs/decompressor.c | 21 +++++++++++++++++----
> > fs/erofs/erofs_fs.h | 3 ++-
> > fs/erofs/internal.h | 19 +++++++++++++++++++
> > fs/erofs/super.c | 4 +++-
> > 4 files changed, 41 insertions(+), 6 deletions(-)
> >
> > diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
> > index 80e8871aef71..93411e9df9b6 100644
> > --- a/fs/erofs/decompressor.c
> > +++ b/fs/erofs/decompressor.c
> > @@ -28,6 +28,17 @@ struct z_erofs_decompressor {
> > char *name;
> > };
> > +int z_erofs_load_lz4_config(struct super_block *sb,
> > + struct erofs_super_block *dsb)
> > +{
> > + u16 distance = le16_to_cpu(dsb->lz4_max_distance);
> > +
> > + EROFS_SB(sb)->lz4.max_distance_pages = distance ?
> > + DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
> > + LZ4_MAX_DISTANCE_PAGES;
> > + return 0;
> > +}
> > +
> > static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> > struct list_head *pagepool)
> > {
> > @@ -36,6 +47,8 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> > struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
> > unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
> > BITS_PER_LONG)] = { 0 };
> > + unsigned int lz4_max_distance_pages =
> > + EROFS_SB(rq->sb)->lz4.max_distance_pages;
> > void *kaddr = NULL;
> > unsigned int i, j, top;
> > @@ -44,14 +57,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
> > struct page *const page = rq->out[i];
> > struct page *victim;
> > - if (j >= LZ4_MAX_DISTANCE_PAGES)
> > + if (j >= lz4_max_distance_pages)
> > j = 0;
> > /* 'valid' bounced can only be tested after a complete round */
> > if (test_bit(j, bounced)) {
> > - DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
> > - DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
> > - availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
> > + DBG_BUGON(i < lz4_max_distance_pages);
> > + DBG_BUGON(top >= lz4_max_distance_pages);
> > + availables[top++] = rq->out[i - lz4_max_distance_pages];
> > }
> > if (page) {
> > diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
> > index 9ad1615f4474..b27d0e4e4ab5 100644
> > --- a/fs/erofs/erofs_fs.h
> > +++ b/fs/erofs/erofs_fs.h
> > @@ -39,7 +39,8 @@ struct erofs_super_block {
> > __u8 uuid[16]; /* 128-bit uuid for volume */
> > __u8 volume_name[16]; /* volume name */
> > __le32 feature_incompat;
> > - __u8 reserved2[44];
> > + __le16 lz4_max_distance;
>
> It missed to add comments, otherwise it looks good to me.
>
Yes, it needs some inline comment description, will update in
the next version.
> Reviewed-by: Chao Yu <yuchao0@huawei.com>
>
Thanks for the review!
Thanks,
Gao Xiang
> Thanks,
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH 3/4] erofs: introduce on-disk lz4 fs configurations
2021-03-27 3:49 ` [PATCH 0/4] erofs: introduce on-disk compression configurations Gao Xiang via Linux-erofs
2021-03-27 3:49 ` [PATCH 1/4] erofs: introduce erofs_sb_has_xxx() helpers Gao Xiang via Linux-erofs
2021-03-27 3:49 ` [PATCH 2/4] erofs: support adjust lz4 history window size Gao Xiang via Linux-erofs
@ 2021-03-27 3:49 ` Gao Xiang via Linux-erofs
2021-03-27 9:34 ` Chao Yu
2021-03-27 3:49 ` [PATCH 4/4] erofs: add on-disk compression configurations Gao Xiang via Linux-erofs
3 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-03-27 3:49 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Chao Yu; +Cc: LKML
From: Gao Xiang <hsiangkao@redhat.com>
Introduce z_erofs_lz4_cfgs to store all lz4 configurations.
Currently it's only max_distance, but will be used for new
features later.
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
---
fs/erofs/decompressor.c | 15 +++++++++++++--
fs/erofs/erofs_fs.h | 6 ++++++
fs/erofs/internal.h | 8 +++++---
fs/erofs/super.c | 2 +-
4 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 93411e9df9b6..97538ff24a19 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -29,9 +29,20 @@ struct z_erofs_decompressor {
};
int z_erofs_load_lz4_config(struct super_block *sb,
- struct erofs_super_block *dsb)
+ struct erofs_super_block *dsb,
+ struct z_erofs_lz4_cfgs *lz4, int size)
{
- u16 distance = le16_to_cpu(dsb->lz4_max_distance);
+ u16 distance;
+
+ if (lz4) {
+ if (size < sizeof(struct z_erofs_lz4_cfgs)) {
+ erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
+ return -EINVAL;
+ }
+ distance = le16_to_cpu(lz4->max_distance);
+ } else {
+ distance = le16_to_cpu(dsb->lz4_max_distance);
+ }
EROFS_SB(sb)->lz4.max_distance_pages = distance ?
DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
index b27d0e4e4ab5..1322ae63944b 100644
--- a/fs/erofs/erofs_fs.h
+++ b/fs/erofs/erofs_fs.h
@@ -196,6 +196,12 @@ enum {
Z_EROFS_COMPRESSION_MAX
};
+/* 14 bytes (+ length field = 16 bytes) */
+struct z_erofs_lz4_cfgs {
+ __le16 max_distance;
+ u8 reserved[12];
+} __packed;
+
/*
* bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
* e.g. for 4k logical cluster size, 4B if compacted 2B is off;
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 1de60992c3dd..46b977f348eb 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -441,7 +441,8 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
int erofs_try_to_free_cached_page(struct address_space *mapping,
struct page *page);
int z_erofs_load_lz4_config(struct super_block *sb,
- struct erofs_super_block *dsb);
+ struct erofs_super_block *dsb,
+ struct z_erofs_lz4_cfgs *lz4, int len);
#else
static inline void erofs_shrinker_register(struct super_block *sb) {}
static inline void erofs_shrinker_unregister(struct super_block *sb) {}
@@ -450,9 +451,10 @@ static inline void erofs_exit_shrinker(void) {}
static inline int z_erofs_init_zip_subsystem(void) { return 0; }
static inline void z_erofs_exit_zip_subsystem(void) {}
static inline int z_erofs_load_lz4_config(struct super_block *sb,
- struct erofs_super_block *dsb)
+ struct erofs_super_block *dsb,
+ struct z_erofs_lz4_cfgs *lz4, int len)
{
- if (dsb->lz4_max_distance) {
+ if (lz4 || dsb->lz4_max_distance) {
erofs_err(sb, "lz4 algorithm isn't enabled");
return -EINVAL;
}
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 3212e4f73f85..1ca8da3f2125 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -189,7 +189,7 @@ static int erofs_read_superblock(struct super_block *sb)
}
/* parse on-disk compression configurations */
- ret = z_erofs_load_lz4_config(sb, dsb);
+ ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
out:
kunmap(page);
put_page(page);
--
2.20.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH 4/4] erofs: add on-disk compression configurations
2021-03-27 3:49 ` [PATCH 0/4] erofs: introduce on-disk compression configurations Gao Xiang via Linux-erofs
` (2 preceding siblings ...)
2021-03-27 3:49 ` [PATCH 3/4] erofs: introduce on-disk lz4 fs configurations Gao Xiang via Linux-erofs
@ 2021-03-27 3:49 ` Gao Xiang via Linux-erofs
2021-03-27 9:46 ` Chao Yu
3 siblings, 1 reply; 11+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-03-27 3:49 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Chao Yu; +Cc: LKML
From: Gao Xiang <hsiangkao@redhat.com>
Add a bitmap for available compression algorithms and a variable-sized
on-disk table for compression options in preparation for upcoming big
pcluster and LZMA algorithm, which follows the end of super block.
To parse the compression options, the bitmap is scanned one by one.
For each available algorithm, there is data followed by 2-byte `length'
correspondingly (it's enough for most cases, or entire fs blocks should
be used.)
With such available algorithm bitmap, kernel itself can also refuse to
mount such filesystem if any unsupported compression algorithm exists.
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
---
fs/erofs/decompressor.c | 2 +-
fs/erofs/erofs_fs.h | 16 +++--
fs/erofs/internal.h | 5 +-
fs/erofs/super.c | 145 +++++++++++++++++++++++++++++++++++++++-
4 files changed, 161 insertions(+), 7 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 97538ff24a19..27aa6a99b371 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -41,7 +41,7 @@ int z_erofs_load_lz4_config(struct super_block *sb,
}
distance = le16_to_cpu(lz4->max_distance);
} else {
- distance = le16_to_cpu(dsb->lz4_max_distance);
+ distance = le16_to_cpu(dsb->u1.lz4_max_distance);
}
EROFS_SB(sb)->lz4.max_distance_pages = distance ?
diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
index 1322ae63944b..ef3f8a99aa5f 100644
--- a/fs/erofs/erofs_fs.h
+++ b/fs/erofs/erofs_fs.h
@@ -18,15 +18,18 @@
* be incompatible with this kernel version.
*/
#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
-#define EROFS_ALL_FEATURE_INCOMPAT EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
+#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
+#define EROFS_ALL_FEATURE_INCOMPAT \
+ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
+ EROFS_FEATURE_INCOMPAT_COMPR_CFGS)
-/* 128-byte erofs on-disk super block */
+/* erofs on-disk super block (currently 128 bytes) */
struct erofs_super_block {
__le32 magic; /* file system magic number */
__le32 checksum; /* crc32c(super_block) */
__le32 feature_compat;
__u8 blkszbits; /* support block_size == PAGE_SIZE only */
- __u8 reserved;
+ __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */
__le16 root_nid; /* nid of root directory */
__le64 inos; /* total valid ino # (== f_files - f_favail) */
@@ -39,7 +42,11 @@ struct erofs_super_block {
__u8 uuid[16]; /* 128-bit uuid for volume */
__u8 volume_name[16]; /* volume name */
__le32 feature_incompat;
- __le16 lz4_max_distance;
+ union {
+ /* bitmap for available compression algorithms */
+ __le16 available_compr_algs;
+ __le16 lz4_max_distance;
+ } __packed u1;
__u8 reserved2[42];
};
@@ -195,6 +202,7 @@ enum {
Z_EROFS_COMPRESSION_LZ4 = 0,
Z_EROFS_COMPRESSION_MAX
};
+#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1))
/* 14 bytes (+ length field = 16 bytes) */
struct z_erofs_lz4_cfgs {
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 46b977f348eb..f3fa895d809f 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -75,6 +75,7 @@ struct erofs_sb_info {
struct xarray managed_pslots;
unsigned int shrinker_run_no;
+ u16 available_compr_algs;
/* pseudo inode to manage cached pages */
struct inode *managed_cache;
@@ -90,6 +91,7 @@ struct erofs_sb_info {
/* inode slot unit size in bit shift */
unsigned char islotbits;
+ u32 sb_size; /* total superblock size */
u32 build_time_nsec;
u64 build_time;
@@ -233,6 +235,7 @@ static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
}
EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
+EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
/* atomic flag definitions */
@@ -454,7 +457,7 @@ static inline int z_erofs_load_lz4_config(struct super_block *sb,
struct erofs_super_block *dsb,
struct z_erofs_lz4_cfgs *lz4, int len)
{
- if (lz4 || dsb->lz4_max_distance) {
+ if (lz4 || dsb->u1.lz4_max_distance) {
erofs_err(sb, "lz4 algorithm isn't enabled");
return -EINVAL;
}
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 1ca8da3f2125..c5e3039f51bf 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -122,6 +122,138 @@ static bool check_layout_compatibility(struct super_block *sb,
return true;
}
+#ifdef CONFIG_EROFS_FS_ZIP
+/* read variable-sized metadata, offset will be aligned by 4-byte */
+static void *erofs_read_metadata(struct super_block *sb, struct page **pagep,
+ erofs_off_t *offset, int *lengthp)
+{
+ struct page *page = *pagep;
+ u8 *buffer, *ptr;
+ int len, i, cnt;
+ erofs_blk_t blk;
+
+ *offset = round_up(*offset, 4);
+ blk = erofs_blknr(*offset);
+
+ if (!page || page->index != blk) {
+ if (page) {
+ unlock_page(page);
+ put_page(page);
+ }
+ page = erofs_get_meta_page(sb, blk);
+ if (IS_ERR(page))
+ goto err_nullpage;
+ }
+
+ ptr = kmap(page);
+ len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]);
+ if (!len)
+ len = U16_MAX + 1;
+ buffer = kmalloc(len, GFP_KERNEL);
+ if (!buffer) {
+ buffer = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+ *offset += sizeof(__le16);
+ *lengthp = len;
+
+ for (i = 0; i < len; i += cnt) {
+ cnt = min(EROFS_BLKSIZ - (int)erofs_blkoff(*offset), len - i);
+ blk = erofs_blknr(*offset);
+
+ if (!page || page->index != blk) {
+ if (page) {
+ kunmap(page);
+ unlock_page(page);
+ put_page(page);
+ }
+ page = erofs_get_meta_page(sb, blk);
+ if (IS_ERR(page)) {
+ kfree(buffer);
+ goto err_nullpage;
+ }
+ ptr = kmap(page);
+ }
+ memcpy(buffer + i, ptr + erofs_blkoff(*offset), cnt);
+ *offset += cnt;
+ }
+ kunmap(page);
+out:
+ *pagep = page;
+ return buffer;
+err_nullpage:
+ *pagep = NULL;
+ return page;
+}
+
+static int erofs_load_compr_cfgs(struct super_block *sb,
+ struct erofs_super_block *dsb)
+{
+ struct erofs_sb_info *sbi;
+ struct page *page;
+ unsigned int algs, alg;
+ erofs_off_t offset;
+ int size, ret;
+
+ sbi = EROFS_SB(sb);
+ sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
+
+ if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
+ erofs_err(sb,
+"try to load compressed image with unsupported algorithms %x",
+ sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
+ return -EINVAL;
+ }
+
+ offset = EROFS_SUPER_OFFSET + sbi->sb_size;
+ page = NULL;
+ alg = 0;
+ ret = 0;
+
+ for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
+ void *data;
+
+ if (!(algs & 1))
+ continue;
+
+ data = erofs_read_metadata(sb, &page, &offset, &size);
+ if (IS_ERR(data)) {
+ ret = PTR_ERR(data);
+ goto err;
+ }
+
+ switch (alg) {
+ case Z_EROFS_COMPRESSION_LZ4:
+ ret = z_erofs_load_lz4_config(sb, dsb, data, size);
+ break;
+ default:
+ DBG_BUGON(1);
+ ret = -EFAULT;
+ }
+ kfree(data);
+ if (ret)
+ goto err;
+ }
+err:
+ if (page) {
+ unlock_page(page);
+ put_page(page);
+ }
+ return ret;
+}
+#else
+static int erofs_load_compr_cfgs(struct super_block *sb,
+ struct erofs_super_block *dsb)
+{
+ if (dsb->u1.available_compr_algs) {
+ erofs_err(sb,
+"try to load compressed image when compression is disabled");
+ return -EINVAL;
+ }
+ return 0;
+}
+#endif
+
static int erofs_read_superblock(struct super_block *sb)
{
struct erofs_sb_info *sbi;
@@ -166,6 +298,13 @@ static int erofs_read_superblock(struct super_block *sb)
if (!check_layout_compatibility(sb, dsb))
goto out;
+ sbi->sb_size = 128 + dsb->sb_extslots * 16;
+ if (sbi->sb_size > EROFS_BLKSIZ) {
+ erofs_err(sb, "invalid sb_extslots %u (more than a fs block)",
+ sbi->sb_size);
+ goto out;
+ }
+
sbi->blocks = le32_to_cpu(dsb->blocks);
sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
#ifdef CONFIG_EROFS_FS_XATTR
@@ -189,7 +328,11 @@ static int erofs_read_superblock(struct super_block *sb)
}
/* parse on-disk compression configurations */
- ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
+ if (erofs_sb_has_compr_cfgs(sbi))
+ ret = erofs_load_compr_cfgs(sb, dsb);
+ else
+ ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
+
out:
kunmap(page);
put_page(page);
--
2.20.1
^ permalink raw reply related [flat|nested] 11+ messages in thread* Re: [PATCH 4/4] erofs: add on-disk compression configurations
2021-03-27 3:49 ` [PATCH 4/4] erofs: add on-disk compression configurations Gao Xiang via Linux-erofs
@ 2021-03-27 9:46 ` Chao Yu
2021-03-27 10:20 ` Gao Xiang
0 siblings, 1 reply; 11+ messages in thread
From: Chao Yu @ 2021-03-27 9:46 UTC (permalink / raw)
To: Gao Xiang, linux-erofs, Chao Yu; +Cc: LKML
On 2021/3/27 11:49, Gao Xiang wrote:
> From: Gao Xiang <hsiangkao@redhat.com>
>
> Add a bitmap for available compression algorithms and a variable-sized
> on-disk table for compression options in preparation for upcoming big
> pcluster and LZMA algorithm, which follows the end of super block.
>
> To parse the compression options, the bitmap is scanned one by one.
> For each available algorithm, there is data followed by 2-byte `length'
> correspondingly (it's enough for most cases, or entire fs blocks should
> be used.)
>
> With such available algorithm bitmap, kernel itself can also refuse to
> mount such filesystem if any unsupported compression algorithm exists.
>
> Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
> ---
> fs/erofs/decompressor.c | 2 +-
> fs/erofs/erofs_fs.h | 16 +++--
> fs/erofs/internal.h | 5 +-
> fs/erofs/super.c | 145 +++++++++++++++++++++++++++++++++++++++-
> 4 files changed, 161 insertions(+), 7 deletions(-)
>
> diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
> index 97538ff24a19..27aa6a99b371 100644
> --- a/fs/erofs/decompressor.c
> +++ b/fs/erofs/decompressor.c
> @@ -41,7 +41,7 @@ int z_erofs_load_lz4_config(struct super_block *sb,
> }
> distance = le16_to_cpu(lz4->max_distance);
> } else {
> - distance = le16_to_cpu(dsb->lz4_max_distance);
> + distance = le16_to_cpu(dsb->u1.lz4_max_distance);
> }
>
> EROFS_SB(sb)->lz4.max_distance_pages = distance ?
> diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
> index 1322ae63944b..ef3f8a99aa5f 100644
> --- a/fs/erofs/erofs_fs.h
> +++ b/fs/erofs/erofs_fs.h
> @@ -18,15 +18,18 @@
> * be incompatible with this kernel version.
> */
> #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
> -#define EROFS_ALL_FEATURE_INCOMPAT EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
> +#define EROFS_ALL_FEATURE_INCOMPAT \
> + (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
> + EROFS_FEATURE_INCOMPAT_COMPR_CFGS)
>
> -/* 128-byte erofs on-disk super block */
> +/* erofs on-disk super block (currently 128 bytes) */
> struct erofs_super_block {
> __le32 magic; /* file system magic number */
> __le32 checksum; /* crc32c(super_block) */
> __le32 feature_compat;
> __u8 blkszbits; /* support block_size == PAGE_SIZE only */
> - __u8 reserved;
> + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */
>
> __le16 root_nid; /* nid of root directory */
> __le64 inos; /* total valid ino # (== f_files - f_favail) */
> @@ -39,7 +42,11 @@ struct erofs_super_block {
> __u8 uuid[16]; /* 128-bit uuid for volume */
> __u8 volume_name[16]; /* volume name */
> __le32 feature_incompat;
> - __le16 lz4_max_distance;
> + union {
> + /* bitmap for available compression algorithms */
> + __le16 available_compr_algs;
> + __le16 lz4_max_distance;
> + } __packed u1;
> __u8 reserved2[42];
> };
>
> @@ -195,6 +202,7 @@ enum {
> Z_EROFS_COMPRESSION_LZ4 = 0,
> Z_EROFS_COMPRESSION_MAX
> };
> +#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1))
>
> /* 14 bytes (+ length field = 16 bytes) */
> struct z_erofs_lz4_cfgs {
> diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
> index 46b977f348eb..f3fa895d809f 100644
> --- a/fs/erofs/internal.h
> +++ b/fs/erofs/internal.h
> @@ -75,6 +75,7 @@ struct erofs_sb_info {
> struct xarray managed_pslots;
>
> unsigned int shrinker_run_no;
> + u16 available_compr_algs;
>
> /* pseudo inode to manage cached pages */
> struct inode *managed_cache;
> @@ -90,6 +91,7 @@ struct erofs_sb_info {
> /* inode slot unit size in bit shift */
> unsigned char islotbits;
>
> + u32 sb_size; /* total superblock size */
> u32 build_time_nsec;
> u64 build_time;
>
> @@ -233,6 +235,7 @@ static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
> }
>
> EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
> +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
> EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
>
> /* atomic flag definitions */
> @@ -454,7 +457,7 @@ static inline int z_erofs_load_lz4_config(struct super_block *sb,
> struct erofs_super_block *dsb,
> struct z_erofs_lz4_cfgs *lz4, int len)
> {
> - if (lz4 || dsb->lz4_max_distance) {
> + if (lz4 || dsb->u1.lz4_max_distance) {
> erofs_err(sb, "lz4 algorithm isn't enabled");
> return -EINVAL;
> }
> diff --git a/fs/erofs/super.c b/fs/erofs/super.c
> index 1ca8da3f2125..c5e3039f51bf 100644
> --- a/fs/erofs/super.c
> +++ b/fs/erofs/super.c
> @@ -122,6 +122,138 @@ static bool check_layout_compatibility(struct super_block *sb,
> return true;
> }
>
> +#ifdef CONFIG_EROFS_FS_ZIP
> +/* read variable-sized metadata, offset will be aligned by 4-byte */
> +static void *erofs_read_metadata(struct super_block *sb, struct page **pagep,
> + erofs_off_t *offset, int *lengthp)
> +{
> + struct page *page = *pagep;
> + u8 *buffer, *ptr;
> + int len, i, cnt;
> + erofs_blk_t blk;
> +
> + *offset = round_up(*offset, 4);
> + blk = erofs_blknr(*offset);
> +
> + if (!page || page->index != blk) {
> + if (page) {
> + unlock_page(page);
> + put_page(page);
> + }
> + page = erofs_get_meta_page(sb, blk);
> + if (IS_ERR(page))
> + goto err_nullpage;
> + }
> +
> + ptr = kmap(page);
> + len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]);
> + if (!len)
> + len = U16_MAX + 1;
> + buffer = kmalloc(len, GFP_KERNEL);
> + if (!buffer) {
> + buffer = ERR_PTR(-ENOMEM);
Caller expects valid page w/o kmapped or a NULL page, right? it needs
to call kunmap() here? or out label can be relocated above kunmap()?
Thanks,
> + goto out;
> + }
> + *offset += sizeof(__le16);
> + *lengthp = len;
> +
> + for (i = 0; i < len; i += cnt) {
> + cnt = min(EROFS_BLKSIZ - (int)erofs_blkoff(*offset), len - i);
> + blk = erofs_blknr(*offset);
> +
> + if (!page || page->index != blk) {
> + if (page) {
> + kunmap(page);
> + unlock_page(page);
> + put_page(page);
> + }
> + page = erofs_get_meta_page(sb, blk);
> + if (IS_ERR(page)) {
> + kfree(buffer);
> + goto err_nullpage;
> + }
> + ptr = kmap(page);
> + }
> + memcpy(buffer + i, ptr + erofs_blkoff(*offset), cnt);
> + *offset += cnt;
> + }
> + kunmap(page);
> +out:
> + *pagep = page;
> + return buffer;
> +err_nullpage:
> + *pagep = NULL;
> + return page;
> +}
> +
> +static int erofs_load_compr_cfgs(struct super_block *sb,
> + struct erofs_super_block *dsb)
> +{
> + struct erofs_sb_info *sbi;
> + struct page *page;
> + unsigned int algs, alg;
> + erofs_off_t offset;
> + int size, ret;
> +
> + sbi = EROFS_SB(sb);
> + sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
> +
> + if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
> + erofs_err(sb,
> +"try to load compressed image with unsupported algorithms %x",
> + sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
> + return -EINVAL;
> + }
> +
> + offset = EROFS_SUPER_OFFSET + sbi->sb_size;
> + page = NULL;
> + alg = 0;
> + ret = 0;
> +
> + for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
> + void *data;
> +
> + if (!(algs & 1))
> + continue;
> +
> + data = erofs_read_metadata(sb, &page, &offset, &size);
> + if (IS_ERR(data)) {
> + ret = PTR_ERR(data);
> + goto err;
> + }
> +
> + switch (alg) {
> + case Z_EROFS_COMPRESSION_LZ4:
> + ret = z_erofs_load_lz4_config(sb, dsb, data, size);
> + break;
> + default:
> + DBG_BUGON(1);
> + ret = -EFAULT;
> + }
> + kfree(data);
> + if (ret)
> + goto err;
> + }
> +err:
> + if (page) {
> + unlock_page(page);
> + put_page(page);
> + }
> + return ret;
> +}
> +#else
> +static int erofs_load_compr_cfgs(struct super_block *sb,
> + struct erofs_super_block *dsb)
> +{
> + if (dsb->u1.available_compr_algs) {
> + erofs_err(sb,
> +"try to load compressed image when compression is disabled");
> + return -EINVAL;
> + }
> + return 0;
> +}
> +#endif
> +
> static int erofs_read_superblock(struct super_block *sb)
> {
> struct erofs_sb_info *sbi;
> @@ -166,6 +298,13 @@ static int erofs_read_superblock(struct super_block *sb)
> if (!check_layout_compatibility(sb, dsb))
> goto out;
>
> + sbi->sb_size = 128 + dsb->sb_extslots * 16;
> + if (sbi->sb_size > EROFS_BLKSIZ) {
> + erofs_err(sb, "invalid sb_extslots %u (more than a fs block)",
> + sbi->sb_size);
> + goto out;
> + }
> +
> sbi->blocks = le32_to_cpu(dsb->blocks);
> sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
> #ifdef CONFIG_EROFS_FS_XATTR
> @@ -189,7 +328,11 @@ static int erofs_read_superblock(struct super_block *sb)
> }
>
> /* parse on-disk compression configurations */
> - ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
> + if (erofs_sb_has_compr_cfgs(sbi))
> + ret = erofs_load_compr_cfgs(sb, dsb);
> + else
> + ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
> +
> out:
> kunmap(page);
> put_page(page);
>
^ permalink raw reply [flat|nested] 11+ messages in thread* Re: [PATCH 4/4] erofs: add on-disk compression configurations
2021-03-27 9:46 ` Chao Yu
@ 2021-03-27 10:20 ` Gao Xiang
0 siblings, 0 replies; 11+ messages in thread
From: Gao Xiang @ 2021-03-27 10:20 UTC (permalink / raw)
To: Chao Yu; +Cc: linux-erofs, LKML
Hi Chao,
On Sat, Mar 27, 2021 at 05:46:44PM +0800, Chao Yu wrote:
> On 2021/3/27 11:49, Gao Xiang wrote:
> > From: Gao Xiang <hsiangkao@redhat.com>
> >
> > Add a bitmap for available compression algorithms and a variable-sized
> > on-disk table for compression options in preparation for upcoming big
> > pcluster and LZMA algorithm, which follows the end of super block.
> >
> > To parse the compression options, the bitmap is scanned one by one.
> > For each available algorithm, there is data followed by 2-byte `length'
> > correspondingly (it's enough for most cases, or entire fs blocks should
> > be used.)
> >
> > With such available algorithm bitmap, kernel itself can also refuse to
> > mount such filesystem if any unsupported compression algorithm exists.
> >
> > Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
> > ---
> > fs/erofs/decompressor.c | 2 +-
> > fs/erofs/erofs_fs.h | 16 +++--
> > fs/erofs/internal.h | 5 +-
> > fs/erofs/super.c | 145 +++++++++++++++++++++++++++++++++++++++-
> > 4 files changed, 161 insertions(+), 7 deletions(-)
> >
> > diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
> > index 97538ff24a19..27aa6a99b371 100644
> > --- a/fs/erofs/decompressor.c
> > +++ b/fs/erofs/decompressor.c
> > @@ -41,7 +41,7 @@ int z_erofs_load_lz4_config(struct super_block *sb,
> > }
> > distance = le16_to_cpu(lz4->max_distance);
> > } else {
> > - distance = le16_to_cpu(dsb->lz4_max_distance);
> > + distance = le16_to_cpu(dsb->u1.lz4_max_distance);
> > }
> > EROFS_SB(sb)->lz4.max_distance_pages = distance ?
> > diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
> > index 1322ae63944b..ef3f8a99aa5f 100644
> > --- a/fs/erofs/erofs_fs.h
> > +++ b/fs/erofs/erofs_fs.h
> > @@ -18,15 +18,18 @@
> > * be incompatible with this kernel version.
> > */
> > #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
> > -#define EROFS_ALL_FEATURE_INCOMPAT EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> > +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
> > +#define EROFS_ALL_FEATURE_INCOMPAT \
> > + (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
> > + EROFS_FEATURE_INCOMPAT_COMPR_CFGS)
> > -/* 128-byte erofs on-disk super block */
> > +/* erofs on-disk super block (currently 128 bytes) */
> > struct erofs_super_block {
> > __le32 magic; /* file system magic number */
> > __le32 checksum; /* crc32c(super_block) */
> > __le32 feature_compat;
> > __u8 blkszbits; /* support block_size == PAGE_SIZE only */
> > - __u8 reserved;
> > + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */
> > __le16 root_nid; /* nid of root directory */
> > __le64 inos; /* total valid ino # (== f_files - f_favail) */
> > @@ -39,7 +42,11 @@ struct erofs_super_block {
> > __u8 uuid[16]; /* 128-bit uuid for volume */
> > __u8 volume_name[16]; /* volume name */
> > __le32 feature_incompat;
> > - __le16 lz4_max_distance;
> > + union {
> > + /* bitmap for available compression algorithms */
> > + __le16 available_compr_algs;
> > + __le16 lz4_max_distance;
> > + } __packed u1;
> > __u8 reserved2[42];
> > };
> > @@ -195,6 +202,7 @@ enum {
> > Z_EROFS_COMPRESSION_LZ4 = 0,
> > Z_EROFS_COMPRESSION_MAX
> > };
> > +#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1))
> > /* 14 bytes (+ length field = 16 bytes) */
> > struct z_erofs_lz4_cfgs {
> > diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
> > index 46b977f348eb..f3fa895d809f 100644
> > --- a/fs/erofs/internal.h
> > +++ b/fs/erofs/internal.h
> > @@ -75,6 +75,7 @@ struct erofs_sb_info {
> > struct xarray managed_pslots;
> > unsigned int shrinker_run_no;
> > + u16 available_compr_algs;
> > /* pseudo inode to manage cached pages */
> > struct inode *managed_cache;
> > @@ -90,6 +91,7 @@ struct erofs_sb_info {
> > /* inode slot unit size in bit shift */
> > unsigned char islotbits;
> > + u32 sb_size; /* total superblock size */
> > u32 build_time_nsec;
> > u64 build_time;
> > @@ -233,6 +235,7 @@ static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
> > }
> > EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
> > +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
> > EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
> > /* atomic flag definitions */
> > @@ -454,7 +457,7 @@ static inline int z_erofs_load_lz4_config(struct super_block *sb,
> > struct erofs_super_block *dsb,
> > struct z_erofs_lz4_cfgs *lz4, int len)
> > {
> > - if (lz4 || dsb->lz4_max_distance) {
> > + if (lz4 || dsb->u1.lz4_max_distance) {
> > erofs_err(sb, "lz4 algorithm isn't enabled");
> > return -EINVAL;
> > }
> > diff --git a/fs/erofs/super.c b/fs/erofs/super.c
> > index 1ca8da3f2125..c5e3039f51bf 100644
> > --- a/fs/erofs/super.c
> > +++ b/fs/erofs/super.c
> > @@ -122,6 +122,138 @@ static bool check_layout_compatibility(struct super_block *sb,
> > return true;
> > }
> > +#ifdef CONFIG_EROFS_FS_ZIP
> > +/* read variable-sized metadata, offset will be aligned by 4-byte */
> > +static void *erofs_read_metadata(struct super_block *sb, struct page **pagep,
> > + erofs_off_t *offset, int *lengthp)
> > +{
> > + struct page *page = *pagep;
> > + u8 *buffer, *ptr;
> > + int len, i, cnt;
> > + erofs_blk_t blk;
> > +
> > + *offset = round_up(*offset, 4);
> > + blk = erofs_blknr(*offset);
> > +
> > + if (!page || page->index != blk) {
> > + if (page) {
> > + unlock_page(page);
> > + put_page(page);
> > + }
> > + page = erofs_get_meta_page(sb, blk);
> > + if (IS_ERR(page))
> > + goto err_nullpage;
> > + }
> > +
> > + ptr = kmap(page);
> > + len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]);
> > + if (!len)
> > + len = U16_MAX + 1;
> > + buffer = kmalloc(len, GFP_KERNEL);
> > + if (!buffer) {
> > + buffer = ERR_PTR(-ENOMEM);
>
> Caller expects valid page w/o kmapped or a NULL page, right? it needs
> to call kunmap() here? or out label can be relocated above kunmap()?
Yeah, I misplaced it by mistake, thanks for pointing out!
Thanks,
Gao Xiang
>
> Thanks,
>
^ permalink raw reply [flat|nested] 11+ messages in thread