From: Gao Xiang <hsiangkao@linux.alibaba.com>
To: linux-erofs@lists.ozlabs.org
Cc: Gao Xiang <hsiangkao@linux.alibaba.com>,
Yue Hu <huyue2@coolpad.com>,
Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
Subject: [PATCH 7/8] erofs-utils: fuse: introduce partial-referenced pclusters
Date: Mon, 26 Sep 2022 23:25:10 +0800 [thread overview]
Message-ID: <20220926152511.94832-8-hsiangkao@linux.alibaba.com> (raw)
In-Reply-To: <20220926152511.94832-1-hsiangkao@linux.alibaba.com>
Due to deduplication for compressed data, pclusters can be partially
referenced with their prefixes.
Decompression algorithms should know that in advance, otherwise they
will fail out unexpectedly.
Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/internal.h | 4 ++++
include/erofs_fs.h | 7 ++++++-
lib/data.c | 3 ++-
lib/zmap.c | 6 ++++++
4 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 6fc58f9..3bce92f 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -136,6 +136,7 @@ EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
+EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
#define EROFS_I_EA_INITED (1 << 0)
@@ -286,6 +287,7 @@ enum {
BH_Encoded,
BH_FullMapped,
BH_Fragment,
+ BH_Partialref,
};
/* Has a disk mapping */
@@ -298,6 +300,8 @@ enum {
#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
/* Located in the special packed inode */
#define EROFS_MAP_FRAGMENT (1 << BH_Fragment)
+/* the extent refers to partial compressed data */
+#define EROFS_MAP_PARTIAL_REF (1 << BH_Partialref)
struct erofs_map_blocks {
char mpage[EROFS_BLKSIZ];
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index e492ad9..48ad5b5 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -26,6 +26,7 @@
#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008
#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010
#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020
+#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
@@ -33,7 +34,8 @@
EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \
EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \
- EROFS_FEATURE_INCOMPAT_FRAGMENTS)
+ EROFS_FEATURE_INCOMPAT_FRAGMENTS | \
+ EROFS_FEATURE_INCOMPAT_DEDUPE)
#define EROFS_SB_EXTSLOT_SIZE 16
@@ -371,6 +373,9 @@ enum {
#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2
#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0
+/* (noncompact, HEAD) This pcluster refers to compressed data partially */
+#define Z_EROFS_VLE_DI_PARTIAL_REF (1 << 15)
+
/*
* D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the
* compressed block count of a compressed extent (in logical clusters, aka.
diff --git a/lib/data.c b/lib/data.c
index bcb0f7e..76a6677 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -258,7 +258,8 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
} else {
DBG_BUGON(end != map.m_la + map.m_llen);
length = map.m_llen;
- partial = !(map.m_flags & EROFS_MAP_FULL_MAPPED);
+ partial = !(map.m_flags & EROFS_MAP_FULL_MAPPED) ||
+ (map.m_flags & EROFS_MAP_PARTIAL_REF);
}
if (map.m_la < offset) {
diff --git a/lib/zmap.c b/lib/zmap.c
index 2c2ba01..11228af 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -121,6 +121,7 @@ struct z_erofs_maprecorder {
u16 delta[2];
erofs_blk_t pblk, compressedlcs;
erofs_off_t nextpackoff;
+ bool partialref;
};
static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
@@ -183,6 +184,9 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
break;
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
+ if (advise & Z_EROFS_VLE_DI_PARTIAL_REF)
+ m->partialref = true;
+
m->clusterofs = le16_to_cpu(di->di_clusterofs);
m->pblk = le32_to_cpu(di->di_u.blkaddr);
break;
@@ -625,6 +629,8 @@ static int z_erofs_do_map_blocks(struct erofs_inode *vi,
goto out;
}
+ if (m.partialref)
+ map->m_flags |= EROFS_MAP_PARTIAL_REF;
map->m_llen = end - map->m_la;
if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
vi->z_tailextent_headlcn = m.lcn;
--
2.24.4
next prev parent reply other threads:[~2022-09-26 15:25 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-26 15:25 [PATCH 0/8] erofs-utils: collection for fragments and dedupe Gao Xiang
2022-09-26 15:25 ` [PATCH 1/8] erofs-utils: fuse: support interlaced uncompressed pcluster Gao Xiang
2022-09-26 15:25 ` [PATCH 2/8] erofs-utils: lib: support fragments data decompression Gao Xiang
2022-09-30 13:08 ` [PATCH] " Yue Hu
2022-09-26 15:25 ` [PATCH 3/8] erofs-utils: introduce z_erofs_inmem_extent Gao Xiang
2022-09-26 15:25 ` [PATCH 4/8] erofs-utils: mkfs: support interlaced uncompressed data layout Gao Xiang
2022-09-26 15:25 ` [PATCH 5/8] erofs-utils: mkfs: support fragments Gao Xiang
2022-09-26 15:25 ` [PATCH 6/8] erofs-utils: lib: add rb-tree implementation Gao Xiang
2022-09-26 15:25 ` Gao Xiang [this message]
2022-09-26 15:25 ` [PATCH 8/8] erofs-utils: mkfs: introduce global compressed data deduplication Gao Xiang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220926152511.94832-8-hsiangkao@linux.alibaba.com \
--to=hsiangkao@linux.alibaba.com \
--cc=ZiyangZhang@linux.alibaba.com \
--cc=huyue2@coolpad.com \
--cc=linux-erofs@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).