linux-erofs.lists.ozlabs.org archive mirror
 help / color / mirror / Atom feed
* [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data
       [not found] <cover.1637140430.git.huyue2@yulong.com>
@ 2021-11-17  9:22 ` Yue Hu
  2021-12-10  3:52   ` Gao Xiang
  2021-11-17  9:22 ` [RFC PATCH v3 2/2] erofs-utils: fuse: " Yue Hu via Linux-erofs
  1 sibling, 1 reply; 6+ messages in thread
From: Yue Hu @ 2021-11-17  9:22 UTC (permalink / raw)
  To: linux-erofs; +Cc: zhangwen, Yue Hu, geshifei, shaojunjun

Currently, we only support tail-end inline data for uncompressed
files, let's support it as well for compressed files.

The idea is from Xiang.

Signed-off-by: Yue Hu <huyue2@yulong.com>
---
v3:
- based on commit 9fe440d0ac03 ("erofs-utils: mkfs: introduce --quiet option").
- move h_idata_size ahead of h_advise for compatibility.
- rename feature/update messages which i think they are more exact.

v2:
- add 2 bytes to record compressed size of tail-pcluster suggested from
  Xiang and update related code.

 include/erofs/internal.h |  1 +
 include/erofs_fs.h       | 10 ++++-
 lib/compress.c           | 83 ++++++++++++++++++++++++++++++----------
 lib/compressor.c         |  9 +++--
 lib/decompress.c         |  4 ++
 lib/inode.c              | 42 ++++++++++----------
 mkfs/main.c              |  7 ++++
 7 files changed, 108 insertions(+), 48 deletions(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 8b154ed..54e5939 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -110,6 +110,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
 EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
 EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
 EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
+EROFS_FEATURE_FUNCS(tail_packing, incompat, INCOMPAT_TAIL_PACKING)
 EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
 
 #define EROFS_I_EA_INITED	(1 << 0)
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 66a68e3..0ebcd5b 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -22,11 +22,13 @@
 #define EROFS_FEATURE_INCOMPAT_COMPR_CFGS	0x00000002
 #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER	0x00000002
 #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE	0x00000004
+#define EROFS_FEATURE_INCOMPAT_TAIL_PACKING	0x00000010
 #define EROFS_ALL_FEATURE_INCOMPAT		\
 	(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
 	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
 	 EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
-	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
+	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
+	 EROFS_FEATURE_INCOMPAT_TAIL_PACKING)
 
 #define EROFS_SB_EXTSLOT_SIZE	16
 
@@ -266,13 +268,17 @@ struct z_erofs_lz4_cfgs {
  *                                  (4B) + 2B + (4B) if compacted 2B is on.
  * bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
  * bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
+ * bit 3 : tail-packing inline (un)compressed data
  */
 #define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
+#define Z_EROFS_ADVISE_INLINE_DATA		0x0008
 
 struct z_erofs_map_header {
-	__le32	h_reserved1;
+	__le16	h_reserved1;
+	/* record the (un)compressed size of tail-packing pcluster */
+	__le16	h_idata_size;
 	__le16	h_advise;
 	/*
 	 * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
diff --git a/lib/compress.c b/lib/compress.c
index 6ca5bed..d7d60b9 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -70,11 +70,10 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
 
 	di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
 
-	/* whether the tail-end uncompressed block or not */
+	/* whether the tail-end (un)compressed block or not */
 	if (!d1) {
-		/* TODO: tail-packing inline compressed data */
-		DBG_BUGON(!raw);
-		type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
+		type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :
+			Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
 		advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
 
 		di.di_advise = advise;
@@ -162,6 +161,17 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
 	return cfg.c_pclusterblks_def;
 }
 
+static int z_erofs_fill_inline_data(struct erofs_inode *inode, char *data,
+				    unsigned int len)
+{
+	inode->idata_size = len;
+	inode->idata = malloc(inode->idata_size);
+	if (!inode->idata)
+		return -ENOMEM;
+	memcpy(inode->idata, data, inode->idata_size);
+	return 0;
+}
+
 static int vle_compress_one(struct erofs_inode *inode,
 			    struct z_erofs_vle_compress_ctx *ctx,
 			    bool final)
@@ -172,15 +182,20 @@ static int vle_compress_one(struct erofs_inode *inode,
 	int ret;
 	static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ];
 	char *const dst = dstbuf + EROFS_BLKSIZ;
+	bool tail_pcluster = false;
 
 	while (len) {
-		const unsigned int pclustersize =
+		unsigned int pclustersize =
 			z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
 		bool raw;
 
-		if (len <= pclustersize) {
+		if (!tail_pcluster && len <= pclustersize) {
 			if (final) {
-				if (len <= EROFS_BLKSIZ)
+				/* TODO: compress with 2 pclusters */
+				if (erofs_sb_has_tail_packing()) {
+					tail_pcluster = true;
+					pclustersize = EROFS_BLKSIZ;
+				} else if (len <= EROFS_BLKSIZ)
 					goto nocompression;
 			} else {
 				break;
@@ -196,6 +211,16 @@ static int vle_compress_one(struct erofs_inode *inode,
 					  inode->i_srcpath,
 					  erofs_strerror(ret));
 			}
+			if (tail_pcluster && len < EROFS_BLKSIZ) {
+				ret = z_erofs_fill_inline_data(inode,
+					(char *)(ctx->queue + ctx->head), len);
+				if (ret)
+					return ret;
+				count = len;
+				raw = true;
+				ctx->compressedblks = 1;
+				goto add_head;
+			}
 nocompression:
 			ret = write_uncompressed_extent(ctx, &len, dst);
 			if (ret < 0)
@@ -204,6 +229,15 @@ nocompression:
 			ctx->compressedblks = 1;
 			raw = true;
 		} else {
+			if (tail_pcluster && ret < EROFS_BLKSIZ &&
+			    !(len - count)) {
+				ret = z_erofs_fill_inline_data(inode, dst, ret);
+				if (ret)
+					return ret;
+				raw = false;
+				ctx->compressedblks = 1;
+				goto add_head;
+			}
 			const unsigned int tailused = ret & (EROFS_BLKSIZ - 1);
 			const unsigned int padding =
 				erofs_sb_has_lz4_0padding() && tailused ?
@@ -228,11 +262,13 @@ nocompression:
 			raw = false;
 		}
 
+add_head:
 		ctx->head += count;
 		/* write compression indexes for this pcluster */
 		vle_write_indexes(ctx, count, raw);
 
-		ctx->blkaddr += ctx->compressedblks;
+		if (!inode->idata_size)
+			ctx->blkaddr += ctx->compressedblks;
 		len -= count;
 
 		if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
@@ -449,6 +485,7 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode,
 {
 	struct z_erofs_map_header h = {
 		.h_advise = cpu_to_le16(inode->z_advise),
+		.h_idata_size = cpu_to_le16(inode->idata_size),
 		.h_algorithmtype = inode->z_algorithmtype[1] << 4 |
 				   inode->z_algorithmtype[0],
 		/* lclustersize */
@@ -476,7 +513,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
 	if (fd < 0) {
 		ret = -errno;
-		goto err_free;
+		goto err_free_meta;
 	}
 
 	/* allocate main data buffer */
@@ -504,8 +541,6 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	inode->z_algorithmtype[1] = algorithmtype[1];
 	inode->z_logical_clusterbits = LOG_BLOCK_SIZE;
 
-	z_erofs_write_mapheader(inode, compressmeta);
-
 	blkaddr = erofs_mapbh(bh->block);	/* start_blkaddr */
 	ctx.blkaddr = blkaddr;
 	ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE;
@@ -531,19 +566,23 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 			goto err_bdrop;
 	}
 
+	inode->idata_size = 0;
+
 	/* do the final round */
 	ret = vle_compress_one(inode, &ctx, true);
 	if (ret)
-		goto err_bdrop;
+		goto err_free_id;
 
 	/* fall back to no compression mode */
 	compressed_blocks = ctx.blkaddr - blkaddr;
-	if (compressed_blocks >= BLK_ROUND_UP(inode->i_size)) {
+	if (compressed_blocks >= BLK_ROUND_UP(inode->i_size) -
+	    (inode->idata_size ? 1 : 0)) {
 		ret = -ENOSPC;
-		goto err_bdrop;
+		goto err_free_id;
 	}
 
 	vle_write_indexes_final(&ctx);
+	z_erofs_write_mapheader(inode, compressmeta);
 
 	close(fd);
 	DBG_BUGON(!compressed_blocks);
@@ -554,12 +593,11 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 		   inode->i_srcpath, (unsigned long long)inode->i_size,
 		   compressed_blocks);
 
-	/*
-	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
-	 *       when both mkfs & kernel support compression inline.
-	 */
-	erofs_bdrop(bh, false);
-	inode->idata_size = 0;
+	if (inode->idata_size)
+		inode->bh_data = bh;
+	else
+		erofs_bdrop(bh, false);
+
 	inode->u.i_blocks = compressed_blocks;
 
 	legacymetasize = ctx.metacur - compressmeta;
@@ -575,11 +613,14 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
 	return 0;
 
+err_free_id:
+	if (inode->idata)
+		free(inode->idata);
 err_bdrop:
 	erofs_bdrop(bh, true);	/* revoke buffer */
 err_close:
 	close(fd);
-err_free:
+err_free_meta:
 	free(compressmeta);
 	return ret;
 }
diff --git a/lib/compressor.c b/lib/compressor.c
index 89c1be1..9b9d2ff 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -23,7 +23,7 @@ int erofs_compress_destsize(struct erofs_compress *c,
 			    void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize)
 {
-	unsigned int uncompressed_size;
+	unsigned int compressed_size;
 	int ret;
 
 	DBG_BUGON(!c->alg);
@@ -35,9 +35,10 @@ int erofs_compress_destsize(struct erofs_compress *c,
 		return ret;
 
 	/* check if there is enough gains to compress */
-	uncompressed_size = *srcsize;
-	if (roundup(ret, EROFS_BLKSIZ) >= uncompressed_size *
-	    c->compress_threshold / 100)
+	compressed_size = *srcsize <= EROFS_BLKSIZ ? ret :
+			  roundup(ret, EROFS_BLKSIZ);
+
+	if (*srcsize <= compressed_size * c->compress_threshold / 100)
 		return -EAGAIN;
 	return ret;
 }
diff --git a/lib/decompress.c b/lib/decompress.c
index 2ee1439..6f4ecc2 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -7,6 +7,7 @@
 
 #include "erofs/decompress.h"
 #include "erofs/err.h"
+#include "erofs/print.h"
 
 #ifdef LZ4_ENABLED
 #include <lz4.h>
@@ -48,6 +49,9 @@ static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq)
 					  rq->decodedlength);
 
 	if (ret != (int)rq->decodedlength) {
+		erofs_err("failed to %s decompress %d in[%u, %u] out[%u]",
+			  rq->partial_decoding ? "partial" : "full",
+			  ret, rq->inputsize, inputmargin, rq->decodedlength);
 		ret = -EIO;
 		goto out;
 	}
diff --git a/lib/inode.c b/lib/inode.c
index 6597a26..b9dcb08 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -549,9 +549,6 @@ static int erofs_prepare_tail_block(struct erofs_inode *inode)
 	struct erofs_buffer_head *bh;
 	int ret;
 
-	if (!inode->idata_size)
-		return 0;
-
 	bh = inode->bh_data;
 	if (!bh) {
 		bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
@@ -585,8 +582,9 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 		inodesize = Z_EROFS_VLE_EXTENT_ALIGN(inodesize) +
 			    inode->extent_isize;
 
-	if (is_inode_layout_compression(inode))
+	if (!inode->idata_size)
 		goto noinline;
+
 	if (inode->datalayout == EROFS_INODE_CHUNK_BASED)
 		goto noinline;
 
@@ -595,23 +593,18 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 		goto noinline;
 	}
 
-	/*
-	 * if the file size is block-aligned for uncompressed files,
-	 * should use EROFS_INODE_FLAT_PLAIN data mapping mode.
-	 */
-	if (!inode->idata_size)
-		inode->datalayout = EROFS_INODE_FLAT_PLAIN;
-
 	bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size);
 	if (bh == ERR_PTR(-ENOSPC)) {
 		int ret;
 
-		inode->datalayout = EROFS_INODE_FLAT_PLAIN;
-noinline:
 		/* expend an extra block for tail-end data */
 		ret = erofs_prepare_tail_block(inode);
 		if (ret)
 			return ret;
+noinline:
+		if (!is_inode_layout_compression(inode))
+			inode->datalayout = EROFS_INODE_FLAT_PLAIN;
+
 		bh = erofs_balloc(INODE, inodesize, 0, 0);
 		if (IS_ERR(bh))
 			return PTR_ERR(bh);
@@ -619,7 +612,15 @@ noinline:
 	} else if (IS_ERR(bh)) {
 		return PTR_ERR(bh);
 	} else if (inode->idata_size) {
-		inode->datalayout = EROFS_INODE_FLAT_INLINE;
+		if (is_inode_layout_compression(inode)) {
+			struct z_erofs_map_header *h = inode->compressmeta;
+
+			h->h_advise |= Z_EROFS_ADVISE_INLINE_DATA;
+			erofs_dbg("%s: inline data (%u bytes)",
+				  inode->i_srcpath, inode->idata_size);
+		} else {
+			inode->datalayout = EROFS_INODE_FLAT_INLINE;
+		}
 
 		/* allocate inline buffer */
 		ibh = erofs_battach(bh, META, inode->idata_size);
@@ -684,13 +685,12 @@ static int erofs_write_tail_end(struct erofs_inode *inode)
 		ret = dev_write(inode->idata, pos, inode->idata_size);
 		if (ret)
 			return ret;
-		if (inode->idata_size < EROFS_BLKSIZ) {
-			ret = dev_fillzero(pos + inode->idata_size,
-					   EROFS_BLKSIZ - inode->idata_size,
-					   false);
-			if (ret)
-				return ret;
-		}
+		ret = dev_fillzero(pos + inode->idata_size,
+				   EROFS_BLKSIZ - inode->idata_size,
+				   false);
+		if (ret)
+			return ret;
+
 		inode->idata_size = 0;
 		free(inode->idata);
 		inode->idata = NULL;
diff --git a/mkfs/main.c b/mkfs/main.c
index 055d077..7505ff6 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -47,6 +47,7 @@ static struct option long_options[] = {
 	{"compress-hints", required_argument, NULL, 10},
 	{"chunksize", required_argument, NULL, 11},
 	{"quiet", no_argument, 0, 12},
+	{"inline-data", no_argument, NULL, 13},
 #ifdef WITH_ANDROID
 	{"mount-point", required_argument, NULL, 512},
 	{"product-out", required_argument, NULL, 513},
@@ -95,6 +96,7 @@ static void usage(void)
 	      " --help                display this help and exit\n"
 	      " --max-extent-bytes=#  set maximum decompressed extent size # in bytes\n"
 	      " --quiet               quiet execution (do not write anything to standard output.)\n"
+	      " --inline-data         tail-packing inline compressed data\n"
 #ifndef NDEBUG
 	      " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
 #endif
@@ -349,6 +351,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 12:
 			quiet = true;
 			break;
+		case 13:
+			erofs_sb_set_tail_packing();
+			break;
 		case 1:
 			usage();
 			exit(0);
@@ -607,6 +612,8 @@ int main(int argc, char **argv)
 	erofs_show_config();
 	if (erofs_sb_has_chunked_file())
 		erofs_warn("EXPERIMENTAL chunked file feature in use. Use at your own risk!");
+	if (erofs_sb_has_tail_packing())
+		erofs_warn("EXPERIMENTAL compression inline data feature in use. Use at your own risk!");
 	erofs_set_fs_root(cfg.c_src_path);
 #ifndef NDEBUG
 	if (cfg.c_random_pclusterblks)
-- 
2.17.1




^ permalink raw reply related	[flat|nested] 6+ messages in thread

* [RFC PATCH v3 2/2] erofs-utils: fuse: support tail-packing inline compressed data
       [not found] <cover.1637140430.git.huyue2@yulong.com>
  2021-11-17  9:22 ` [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data Yue Hu
@ 2021-11-17  9:22 ` Yue Hu via Linux-erofs
  2021-12-06  0:39   ` Gao Xiang
  1 sibling, 1 reply; 6+ messages in thread
From: Yue Hu via Linux-erofs @ 2021-11-17  9:22 UTC (permalink / raw)
  To: linux-erofs; +Cc: zhangwen, Yue Hu, geshifei, shaojunjun

Add tail-packing inline compressed data support for erofsfuse.

Signed-off-by: Yue Hu <huyue2@yulong.com>
---
v3:
- remove z_idata_addr, add z_idata_headlcn instead of m_taillcn.
- add bug_on for legacy if enable inline and disable big pcluster.
- extract z_erofs_do_map_blocks() instead of added
  z_erofs_map_tail_data_blocks() with similar logic.

v2:
- add tail-packing information to inode and get it on first read.
- update tail-packing checking logic.

 include/erofs/internal.h |   2 +
 lib/decompress.c         |   3 -
 lib/zmap.c               | 136 +++++++++++++++++++++++++++++++++------
 3 files changed, 120 insertions(+), 21 deletions(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 54e5939..5d1a44c 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -172,6 +172,8 @@ struct erofs_inode {
 			uint8_t  z_algorithmtype[2];
 			uint8_t  z_logical_clusterbits;
 			uint8_t  z_physical_clusterblks;
+			uint16_t z_idata_size;
+			uint32_t z_idata_headlcn;
 		};
 	};
 #ifdef WITH_ANDROID
diff --git a/lib/decompress.c b/lib/decompress.c
index 6f4ecc2..806ac91 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -71,9 +71,6 @@ out:
 int z_erofs_decompress(struct z_erofs_decompress_req *rq)
 {
 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
-		if (rq->inputsize != EROFS_BLKSIZ)
-			return -EFSCORRUPTED;
-
 		DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ);
 		DBG_BUGON(rq->decodedlength < rq->decodedskip);
 
diff --git a/lib/zmap.c b/lib/zmap.c
index 458030b..42783e5 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -10,6 +10,9 @@
 #include "erofs/io.h"
 #include "erofs/print.h"
 
+static int z_erofs_do_map_blocks(struct erofs_inode *vi,
+				 struct erofs_map_blocks *map);
+
 int z_erofs_fill_inode(struct erofs_inode *vi)
 {
 	if (!erofs_sb_has_big_pcluster() &&
@@ -18,12 +21,69 @@ int z_erofs_fill_inode(struct erofs_inode *vi)
 		vi->z_algorithmtype[0] = 0;
 		vi->z_algorithmtype[1] = 0;
 		vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
+		vi->z_idata_size = 0;
 
 		vi->flags |= EROFS_I_Z_INITED;
+		DBG_BUGON(erofs_sb_has_tail_packing());
 	}
 	return 0;
 }
 
+static erofs_off_t compacted_inline_data_addr(struct erofs_inode *vi,
+					      unsigned int totalidx)
+{
+	const erofs_off_t ebase = round_up(iloc(vi->nid) + vi->inode_isize +
+				  vi->xattr_isize, 8) +
+				  sizeof(struct z_erofs_map_header);
+	unsigned int compacted_4b_initial, compacted_4b_end;
+	unsigned int compacted_2b;
+	erofs_off_t addr;
+
+	compacted_4b_initial = (32 - ebase % 32) / 4;
+	if (compacted_4b_initial == 32 / 4)
+		compacted_4b_initial = 0;
+
+	if (compacted_4b_initial > totalidx) {
+		compacted_4b_initial = 0;
+		compacted_2b = 0;
+	} else if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) {
+		compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
+	} else
+		compacted_2b = 0;
+
+	compacted_4b_end = totalidx - compacted_4b_initial - compacted_2b;
+
+	addr = ebase + compacted_4b_initial * 4 + compacted_2b * 2;
+	if (compacted_4b_end > 1)
+		addr += (compacted_4b_end/2) * 8;
+	if (compacted_4b_end % 2)
+		addr += 8;
+
+	return addr;
+}
+
+static erofs_off_t legacy_inline_data_addr(struct erofs_inode *vi,
+					   unsigned int totalidx)
+{
+	return Z_EROFS_VLE_LEGACY_INDEX_ALIGN(iloc(vi->nid) + vi->inode_isize +
+					      vi->xattr_isize) +
+		totalidx * sizeof(struct z_erofs_vle_decompressed_index);
+}
+
+static erofs_off_t z_erofs_inline_data_addr(struct erofs_inode *vi)
+{
+	const unsigned int datamode = vi->datalayout;
+	const unsigned int totalidx = DIV_ROUND_UP(vi->i_size, EROFS_BLKSIZ);
+
+	if (datamode == EROFS_INODE_FLAT_COMPRESSION)
+		return compacted_inline_data_addr(vi, totalidx);
+
+	if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
+		return legacy_inline_data_addr(vi, totalidx);
+
+	return -EINVAL;
+}
+
 static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
 {
 	int ret;
@@ -44,6 +104,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
 
 	h = (struct z_erofs_map_header *)buf;
 	vi->z_advise = le16_to_cpu(h->h_advise);
+	vi->z_idata_size = le16_to_cpu(h->h_idata_size);
 	vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
 	vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
 
@@ -61,6 +122,16 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
 			  vi->nid * 1ULL);
 		return -EFSCORRUPTED;
 	}
+
+	if (vi->z_idata_size) {
+		struct erofs_map_blocks map = { .m_la = vi->i_size - 1 };
+
+		ret = z_erofs_do_map_blocks(vi, &map);
+		if (ret)
+			return ret;
+		vi->z_idata_headlcn = map.m_la >> vi->z_logical_clusterbits;
+	}
+
 	vi->flags |= EROFS_I_Z_INITED;
 	return 0;
 }
@@ -374,7 +445,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
 }
 
 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
-					    unsigned int initial_lcn)
+					    unsigned int initial_lcn,
+					    bool idatamap)
 {
 	struct erofs_inode *const vi = m->inode;
 	struct erofs_map_blocks *const map = m->map;
@@ -384,6 +456,12 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
 
 	DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
 		  m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
+
+	if (idatamap) {
+		map->m_plen = vi->z_idata_size;
+		return 0;
+	}
+
 	if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
 	    !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
 		map->m_plen = 1 << lclusterbits;
@@ -440,8 +518,8 @@ err_bonus_cblkcnt:
 	return -EFSCORRUPTED;
 }
 
-int z_erofs_map_blocks_iter(struct erofs_inode *vi,
-			    struct erofs_map_blocks *map)
+static int z_erofs_do_map_blocks(struct erofs_inode *vi,
+				 struct erofs_map_blocks *map)
 {
 	struct z_erofs_maprecorder m = {
 		.inode = vi,
@@ -452,18 +530,7 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
 	unsigned int lclusterbits, endoff;
 	unsigned long initial_lcn;
 	unsigned long long ofs, end;
-
-	/* when trying to read beyond EOF, leave it unmapped */
-	if (map->m_la >= vi->i_size) {
-		map->m_llen = map->m_la + 1 - vi->i_size;
-		map->m_la = vi->i_size;
-		map->m_flags = 0;
-		goto out;
-	}
-
-	err = z_erofs_fill_inode_lazy(vi);
-	if (err)
-		goto out;
+	bool idatamap;
 
 	lclusterbits = vi->z_logical_clusterbits;
 	ofs = map->m_la;
@@ -510,19 +577,52 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
 		goto out;
 	}
 
+	/* check if mapping tail-packing data */
+	idatamap = vi->z_idata_size && (ofs == vi->i_size - 1 ||
+		   m.lcn == vi->z_idata_headlcn);
+
+	/* need to trim tail-packing data if beyond file size */
 	map->m_llen = end - map->m_la;
-	map->m_pa = blknr_to_addr(m.pblk);
+	if (idatamap && end > vi->i_size)
+		map->m_llen -= end - vi->i_size;
 
-	err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
+	if (idatamap && (vi->z_advise & Z_EROFS_ADVISE_INLINE_DATA)) {
+		map->m_pa = z_erofs_inline_data_addr(vi);
+		map->m_flags |= EROFS_MAP_META;
+	} else {
+		map->m_pa = blknr_to_addr(m.pblk);
+	}
+
+	err = z_erofs_get_extent_compressedlen(&m, initial_lcn, idatamap);
 	if (err)
 		goto out;
 	map->m_flags |= EROFS_MAP_MAPPED;
-
 out:
 	erofs_dbg("m_la %" PRIu64 " m_pa %" PRIu64 " m_llen %" PRIu64 " m_plen %" PRIu64 " m_flags 0%o",
 		  map->m_la, map->m_pa,
 		  map->m_llen, map->m_plen, map->m_flags);
+	return err;
+}
+
+int z_erofs_map_blocks_iter(struct erofs_inode *vi,
+			    struct erofs_map_blocks *map)
+{
+	int err;
 
+	/* when trying to read beyond EOF, leave it unmapped */
+	if (map->m_la >= vi->i_size) {
+		map->m_llen = map->m_la + 1 - vi->i_size;
+		map->m_la = vi->i_size;
+		map->m_flags = 0;
+		return 0;
+	}
+
+	err = z_erofs_fill_inode_lazy(vi);
+	if (err)
+		goto out;
+
+	err = z_erofs_do_map_blocks(vi, map);
+out:
 	DBG_BUGON(err < 0 && err != -ENOMEM);
 	return err;
 }
-- 
2.17.1




^ permalink raw reply related	[flat|nested] 6+ messages in thread

* Re: [RFC PATCH v3 2/2] erofs-utils: fuse: support tail-packing inline compressed data
  2021-11-17  9:22 ` [RFC PATCH v3 2/2] erofs-utils: fuse: " Yue Hu via Linux-erofs
@ 2021-12-06  0:39   ` Gao Xiang
  2021-12-06  3:40     ` Yue Hu
  0 siblings, 1 reply; 6+ messages in thread
From: Gao Xiang @ 2021-12-06  0:39 UTC (permalink / raw)
  To: Yue Hu; +Cc: geshifei, zhangwen, linux-erofs, shaojunjun

Hi Yue,

On Wed, Nov 17, 2021 at 05:22:01PM +0800, Yue Hu via Linux-erofs wrote:
> Add tail-packing inline compressed data support for erofsfuse.
> 
> Signed-off-by: Yue Hu <huyue2@yulong.com>

Sorry about long delay, I'm very busy in container use cases now.
I've checked your patch, some inlined comments below.

> ---
> v3:
> - remove z_idata_addr, add z_idata_headlcn instead of m_taillcn.
> - add bug_on for legacy if enable inline and disable big pcluster.
> - extract z_erofs_do_map_blocks() instead of added
>   z_erofs_map_tail_data_blocks() with similar logic.
> 
> v2:
> - add tail-packing information to inode and get it on first read.
> - update tail-packing checking logic.
> 
>  include/erofs/internal.h |   2 +
>  lib/decompress.c         |   3 -
>  lib/zmap.c               | 136 +++++++++++++++++++++++++++++++++------
>  3 files changed, 120 insertions(+), 21 deletions(-)
> 
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 54e5939..5d1a44c 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -172,6 +172,8 @@ struct erofs_inode {
>  			uint8_t  z_algorithmtype[2];
>  			uint8_t  z_logical_clusterbits;
>  			uint8_t  z_physical_clusterblks;
> +			uint16_t z_idata_size;
> +			uint32_t z_idata_headlcn;
>  		};
>  	};
>  #ifdef WITH_ANDROID
> diff --git a/lib/decompress.c b/lib/decompress.c
> index 6f4ecc2..806ac91 100644
> --- a/lib/decompress.c
> +++ b/lib/decompress.c
> @@ -71,9 +71,6 @@ out:
>  int z_erofs_decompress(struct z_erofs_decompress_req *rq)
>  {
>  	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
> -		if (rq->inputsize != EROFS_BLKSIZ)
> -			return -EFSCORRUPTED;
> -

		if (rq->inputsize > EROFS_BLKSIZ)
			return -EFSCORRUPTED;

>  		DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ);
>  		DBG_BUGON(rq->decodedlength < rq->decodedskip);
>  
> diff --git a/lib/zmap.c b/lib/zmap.c
> index 458030b..42783e5 100644
> --- a/lib/zmap.c
> +++ b/lib/zmap.c
> @@ -10,6 +10,9 @@
>  #include "erofs/io.h"
>  #include "erofs/print.h"
>  
> +static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> +				 struct erofs_map_blocks *map);
> +
>  int z_erofs_fill_inode(struct erofs_inode *vi)
>  {
>  	if (!erofs_sb_has_big_pcluster() &&
> @@ -18,12 +21,69 @@ int z_erofs_fill_inode(struct erofs_inode *vi)
>  		vi->z_algorithmtype[0] = 0;
>  		vi->z_algorithmtype[1] = 0;
>  		vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
> +		vi->z_idata_size = 0;
>  
>  		vi->flags |= EROFS_I_Z_INITED;
> +		DBG_BUGON(erofs_sb_has_tail_packing());
>  	}
>  	return 0;
>  }
>  
> +static erofs_off_t compacted_inline_data_addr(struct erofs_inode *vi,
> +					      unsigned int totalidx)
> +{
> +	const erofs_off_t ebase = round_up(iloc(vi->nid) + vi->inode_isize +
> +				  vi->xattr_isize, 8) +
> +				  sizeof(struct z_erofs_map_header);
> +	unsigned int compacted_4b_initial, compacted_4b_end;
> +	unsigned int compacted_2b;
> +	erofs_off_t addr;
> +
> +	compacted_4b_initial = (32 - ebase % 32) / 4;
> +	if (compacted_4b_initial == 32 / 4)
> +		compacted_4b_initial = 0;
> +
> +	if (compacted_4b_initial > totalidx) {
> +		compacted_4b_initial = 0;
> +		compacted_2b = 0;
> +	} else if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) {
> +		compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
> +	} else
> +		compacted_2b = 0;
> +
> +	compacted_4b_end = totalidx - compacted_4b_initial - compacted_2b;
> +
> +	addr = ebase + compacted_4b_initial * 4 + compacted_2b * 2;
> +	if (compacted_4b_end > 1)
> +		addr += (compacted_4b_end/2) * 8;
> +	if (compacted_4b_end % 2)
> +		addr += 8;
> +
> +	return addr;
> +}
> +
> +static erofs_off_t legacy_inline_data_addr(struct erofs_inode *vi,
> +					   unsigned int totalidx)
> +{
> +	return Z_EROFS_VLE_LEGACY_INDEX_ALIGN(iloc(vi->nid) + vi->inode_isize +
> +					      vi->xattr_isize) +
> +		totalidx * sizeof(struct z_erofs_vle_decompressed_index);
> +}
> +
> +static erofs_off_t z_erofs_inline_data_addr(struct erofs_inode *vi)
> +{
> +	const unsigned int datamode = vi->datalayout;
> +	const unsigned int totalidx = DIV_ROUND_UP(vi->i_size, EROFS_BLKSIZ);
> +
> +	if (datamode == EROFS_INODE_FLAT_COMPRESSION)
> +		return compacted_inline_data_addr(vi, totalidx);
> +
> +	if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
> +		return legacy_inline_data_addr(vi, totalidx);
> +
> +	return -EINVAL;
> +}

no need these three new functions if introducing
EROFS_GET_BLOCKS_FINDTAIL, see below..

> +
>  static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
>  {
>  	int ret;
> @@ -44,6 +104,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
>  
>  	h = (struct z_erofs_map_header *)buf;
>  	vi->z_advise = le16_to_cpu(h->h_advise);
> +	vi->z_idata_size = le16_to_cpu(h->h_idata_size);
>  	vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
>  	vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
>  
> @@ -61,6 +122,16 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
>  			  vi->nid * 1ULL);
>  		return -EFSCORRUPTED;
>  	}
> +
> +	if (vi->z_idata_size) {
> +		struct erofs_map_blocks map = { .m_la = vi->i_size - 1 };
> +
> +		ret = z_erofs_do_map_blocks(vi, &map);
> +		if (ret)
> +			return ret;

How about introducing another mode called
EROFS_GET_BLOCKS_FINDTAIL

which implys .m_la = vi->i_size - 1 internally, so we won't need to
handle .m_la here.

> +		vi->z_idata_headlcn = map.m_la >> vi->z_logical_clusterbits;

Also updaing vi->z_idata_headlcn when EROFS_GET_BLOCKS_FINDTAIL.

> +	}
> +
>  	vi->flags |= EROFS_I_Z_INITED;
>  	return 0;
>  }
> @@ -374,7 +445,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
>  }
>  
>  static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
> -					    unsigned int initial_lcn)
> +					    unsigned int initial_lcn,
> +					    bool idatamap)
>  {
>  	struct erofs_inode *const vi = m->inode;
>  	struct erofs_map_blocks *const map = m->map;
> @@ -384,6 +456,12 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
>  
>  	DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
>  		  m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
> +
> +	if (idatamap) {
> +		map->m_plen = vi->z_idata_size;
> +		return 0;
> +	}
> +
>  	if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
>  	    !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
>  		map->m_plen = 1 << lclusterbits;
> @@ -440,8 +518,8 @@ err_bonus_cblkcnt:
>  	return -EFSCORRUPTED;
>  }
>  
> -int z_erofs_map_blocks_iter(struct erofs_inode *vi,
> -			    struct erofs_map_blocks *map)
> +static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> +				 struct erofs_map_blocks *map)
>  {
>  	struct z_erofs_maprecorder m = {
>  		.inode = vi,
> @@ -452,18 +530,7 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
>  	unsigned int lclusterbits, endoff;
>  	unsigned long initial_lcn;
>  	unsigned long long ofs, end;
> -
> -	/* when trying to read beyond EOF, leave it unmapped */
> -	if (map->m_la >= vi->i_size) {
> -		map->m_llen = map->m_la + 1 - vi->i_size;
> -		map->m_la = vi->i_size;
> -		map->m_flags = 0;
> -		goto out;
> -	}
> -
> -	err = z_erofs_fill_inode_lazy(vi);
> -	if (err)
> -		goto out;
> +	bool idatamap;
>  
>  	lclusterbits = vi->z_logical_clusterbits;
>  	ofs = map->m_la;
> @@ -510,19 +577,52 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
>  		goto out;
>  	}
>  
> +	/* check if mapping tail-packing data */
> +	idatamap = vi->z_idata_size && (ofs == vi->i_size - 1 ||
> +		   m.lcn == vi->z_idata_headlcn);

better namin as `is_idata'...

I think no need to handle ofs == vi->i_size - 1 specially here
if EROFS_GET_BLOCKS_FINDTAIL is introduced...

> +
> +	/* need to trim tail-packing data if beyond file size */
>  	map->m_llen = end - map->m_la;
> -	map->m_pa = blknr_to_addr(m.pblk);
> +	if (idatamap && end > vi->i_size)
> +		map->m_llen -= end - vi->i_size;

No need as well?

>  
> -	err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
> +	if (idatamap && (vi->z_advise & Z_EROFS_ADVISE_INLINE_DATA)) {
> +		map->m_pa = z_erofs_inline_data_addr(vi);

How about setting another u32 z_idataoff when EROFS_GET_BLOCKS_FINDTAIL
so we won't need to introduce another calculate methods instead?

Thanks,
Gao Xiang

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [RFC PATCH v3 2/2] erofs-utils: fuse: support tail-packing inline compressed data
  2021-12-06  0:39   ` Gao Xiang
@ 2021-12-06  3:40     ` Yue Hu
  0 siblings, 0 replies; 6+ messages in thread
From: Yue Hu @ 2021-12-06  3:40 UTC (permalink / raw)
  To: Gao Xiang; +Cc: linux-erofs, Yue Hu, geshifei, zhangwen, shaojunjun

Hi Xiang,

On Mon, 6 Dec 2021 08:39:04 +0800
Gao Xiang <hsiangkao@linux.alibaba.com> wrote:

> Hi Yue,
> 
> On Wed, Nov 17, 2021 at 05:22:01PM +0800, Yue Hu via Linux-erofs wrote:
> > Add tail-packing inline compressed data support for erofsfuse.
> > 
> > Signed-off-by: Yue Hu <huyue2@yulong.com>  
> 
> Sorry about long delay, I'm very busy in container use cases now.
> I've checked your patch, some inlined comments below.
> 
> > ---
> > v3:
> > - remove z_idata_addr, add z_idata_headlcn instead of m_taillcn.
> > - add bug_on for legacy if enable inline and disable big pcluster.
> > - extract z_erofs_do_map_blocks() instead of added
> >   z_erofs_map_tail_data_blocks() with similar logic.
> > 
> > v2:
> > - add tail-packing information to inode and get it on first read.
> > - update tail-packing checking logic.
> > 
> >  include/erofs/internal.h |   2 +
> >  lib/decompress.c         |   3 -
> >  lib/zmap.c               | 136 +++++++++++++++++++++++++++++++++------
> >  3 files changed, 120 insertions(+), 21 deletions(-)
> > 
> > diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> > index 54e5939..5d1a44c 100644
> > --- a/include/erofs/internal.h
> > +++ b/include/erofs/internal.h
> > @@ -172,6 +172,8 @@ struct erofs_inode {
> >  			uint8_t  z_algorithmtype[2];
> >  			uint8_t  z_logical_clusterbits;
> >  			uint8_t  z_physical_clusterblks;
> > +			uint16_t z_idata_size;
> > +			uint32_t z_idata_headlcn;
> >  		};
> >  	};
> >  #ifdef WITH_ANDROID
> > diff --git a/lib/decompress.c b/lib/decompress.c
> > index 6f4ecc2..806ac91 100644
> > --- a/lib/decompress.c
> > +++ b/lib/decompress.c
> > @@ -71,9 +71,6 @@ out:
> >  int z_erofs_decompress(struct z_erofs_decompress_req *rq)
> >  {
> >  	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
> > -		if (rq->inputsize != EROFS_BLKSIZ)
> > -			return -EFSCORRUPTED;
> > -  
> 
> 		if (rq->inputsize > EROFS_BLKSIZ)
> 			return -EFSCORRUPTED;
> 
> >  		DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ);
> >  		DBG_BUGON(rq->decodedlength < rq->decodedskip);
> >  
> > diff --git a/lib/zmap.c b/lib/zmap.c
> > index 458030b..42783e5 100644
> > --- a/lib/zmap.c
> > +++ b/lib/zmap.c
> > @@ -10,6 +10,9 @@
> >  #include "erofs/io.h"
> >  #include "erofs/print.h"
> >  
> > +static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> > +				 struct erofs_map_blocks *map);
> > +
> >  int z_erofs_fill_inode(struct erofs_inode *vi)
> >  {
> >  	if (!erofs_sb_has_big_pcluster() &&
> > @@ -18,12 +21,69 @@ int z_erofs_fill_inode(struct erofs_inode *vi)
> >  		vi->z_algorithmtype[0] = 0;
> >  		vi->z_algorithmtype[1] = 0;
> >  		vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
> > +		vi->z_idata_size = 0;
> >  
> >  		vi->flags |= EROFS_I_Z_INITED;
> > +		DBG_BUGON(erofs_sb_has_tail_packing());
> >  	}
> >  	return 0;
> >  }
> >  
> > +static erofs_off_t compacted_inline_data_addr(struct erofs_inode *vi,
> > +					      unsigned int totalidx)
> > +{
> > +	const erofs_off_t ebase = round_up(iloc(vi->nid) + vi->inode_isize +
> > +				  vi->xattr_isize, 8) +
> > +				  sizeof(struct z_erofs_map_header);
> > +	unsigned int compacted_4b_initial, compacted_4b_end;
> > +	unsigned int compacted_2b;
> > +	erofs_off_t addr;
> > +
> > +	compacted_4b_initial = (32 - ebase % 32) / 4;
> > +	if (compacted_4b_initial == 32 / 4)
> > +		compacted_4b_initial = 0;
> > +
> > +	if (compacted_4b_initial > totalidx) {
> > +		compacted_4b_initial = 0;
> > +		compacted_2b = 0;
> > +	} else if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) {
> > +		compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
> > +	} else
> > +		compacted_2b = 0;
> > +
> > +	compacted_4b_end = totalidx - compacted_4b_initial - compacted_2b;
> > +
> > +	addr = ebase + compacted_4b_initial * 4 + compacted_2b * 2;
> > +	if (compacted_4b_end > 1)
> > +		addr += (compacted_4b_end/2) * 8;
> > +	if (compacted_4b_end % 2)
> > +		addr += 8;
> > +
> > +	return addr;
> > +}
> > +
> > +static erofs_off_t legacy_inline_data_addr(struct erofs_inode *vi,
> > +					   unsigned int totalidx)
> > +{
> > +	return Z_EROFS_VLE_LEGACY_INDEX_ALIGN(iloc(vi->nid) + vi->inode_isize +
> > +					      vi->xattr_isize) +
> > +		totalidx * sizeof(struct z_erofs_vle_decompressed_index);
> > +}
> > +
> > +static erofs_off_t z_erofs_inline_data_addr(struct erofs_inode *vi)
> > +{
> > +	const unsigned int datamode = vi->datalayout;
> > +	const unsigned int totalidx = DIV_ROUND_UP(vi->i_size, EROFS_BLKSIZ);
> > +
> > +	if (datamode == EROFS_INODE_FLAT_COMPRESSION)
> > +		return compacted_inline_data_addr(vi, totalidx);
> > +
> > +	if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
> > +		return legacy_inline_data_addr(vi, totalidx);
> > +
> > +	return -EINVAL;
> > +}  
> 
> no need these three new functions if introducing
> EROFS_GET_BLOCKS_FINDTAIL, see below..

Let me optimize the logic.

> 
> > +
> >  static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
> >  {
> >  	int ret;
> > @@ -44,6 +104,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
> >  
> >  	h = (struct z_erofs_map_header *)buf;
> >  	vi->z_advise = le16_to_cpu(h->h_advise);
> > +	vi->z_idata_size = le16_to_cpu(h->h_idata_size);
> >  	vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
> >  	vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
> >  
> > @@ -61,6 +122,16 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
> >  			  vi->nid * 1ULL);
> >  		return -EFSCORRUPTED;
> >  	}
> > +
> > +	if (vi->z_idata_size) {
> > +		struct erofs_map_blocks map = { .m_la = vi->i_size - 1 };
> > +
> > +		ret = z_erofs_do_map_blocks(vi, &map);
> > +		if (ret)
> > +			return ret;  
> 
> How about introducing another mode called
> EROFS_GET_BLOCKS_FINDTAIL

Aha, nice name. That flag have leaped into my mind :)

> 
> which implys .m_la = vi->i_size - 1 internally, so we won't need to
> handle .m_la here.
> 
> > +		vi->z_idata_headlcn = map.m_la >> vi->z_logical_clusterbits;  
> 
> Also updaing vi->z_idata_headlcn when EROFS_GET_BLOCKS_FINDTAIL.

Right.

> 
> > +	}
> > +
> >  	vi->flags |= EROFS_I_Z_INITED;
> >  	return 0;
> >  }
> > @@ -374,7 +445,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
> >  }
> >  
> >  static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
> > -					    unsigned int initial_lcn)
> > +					    unsigned int initial_lcn,
> > +					    bool idatamap)
> >  {
> >  	struct erofs_inode *const vi = m->inode;
> >  	struct erofs_map_blocks *const map = m->map;
> > @@ -384,6 +456,12 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
> >  
> >  	DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
> >  		  m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
> > +
> > +	if (idatamap) {
> > +		map->m_plen = vi->z_idata_size;
> > +		return 0;
> > +	}
> > +
> >  	if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
> >  	    !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
> >  		map->m_plen = 1 << lclusterbits;
> > @@ -440,8 +518,8 @@ err_bonus_cblkcnt:
> >  	return -EFSCORRUPTED;
> >  }
> >  
> > -int z_erofs_map_blocks_iter(struct erofs_inode *vi,
> > -			    struct erofs_map_blocks *map)
> > +static int z_erofs_do_map_blocks(struct erofs_inode *vi,
> > +				 struct erofs_map_blocks *map)
> >  {
> >  	struct z_erofs_maprecorder m = {
> >  		.inode = vi,
> > @@ -452,18 +530,7 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
> >  	unsigned int lclusterbits, endoff;
> >  	unsigned long initial_lcn;
> >  	unsigned long long ofs, end;
> > -
> > -	/* when trying to read beyond EOF, leave it unmapped */
> > -	if (map->m_la >= vi->i_size) {
> > -		map->m_llen = map->m_la + 1 - vi->i_size;
> > -		map->m_la = vi->i_size;
> > -		map->m_flags = 0;
> > -		goto out;
> > -	}
> > -
> > -	err = z_erofs_fill_inode_lazy(vi);
> > -	if (err)
> > -		goto out;
> > +	bool idatamap;
> >  
> >  	lclusterbits = vi->z_logical_clusterbits;
> >  	ofs = map->m_la;
> > @@ -510,19 +577,52 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
> >  		goto out;
> >  	}
> >  
> > +	/* check if mapping tail-packing data */
> > +	idatamap = vi->z_idata_size && (ofs == vi->i_size - 1 ||
> > +		   m.lcn == vi->z_idata_headlcn);  
> 
> better namin as `is_idata'...

Okay.

> 
> I think no need to handle ofs == vi->i_size - 1 specially here
> if EROFS_GET_BLOCKS_FINDTAIL is introduced...

Right.

> 
> > +
> > +	/* need to trim tail-packing data if beyond file size */
> >  	map->m_llen = end - map->m_la;
> > -	map->m_pa = blknr_to_addr(m.pblk);
> > +	if (idatamap && end > vi->i_size)
> > +		map->m_llen -= end - vi->i_size;  
> 
> No need as well?

Let me confirm.

> 
> >  
> > -	err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
> > +	if (idatamap && (vi->z_advise & Z_EROFS_ADVISE_INLINE_DATA)) {
> > +		map->m_pa = z_erofs_inline_data_addr(vi);  
> 
> How about setting another u32 z_idataoff when EROFS_GET_BLOCKS_FINDTAIL
> so we won't need to introduce another calculate methods instead?

Agree, need to improve it.

> 
> Thanks,
> Gao Xiang


^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data
  2021-11-17  9:22 ` [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data Yue Hu
@ 2021-12-10  3:52   ` Gao Xiang
  2021-12-10  4:01     ` Yue Hu
  0 siblings, 1 reply; 6+ messages in thread
From: Gao Xiang @ 2021-12-10  3:52 UTC (permalink / raw)
  To: Yue Hu; +Cc: geshifei, zhangwen, linux-erofs, shaojunjun

Hi Yue,

On Wed, Nov 17, 2021 at 05:22:00PM +0800, Yue Hu wrote:
> Currently, we only support tail-end inline data for uncompressed
> files, let's support it as well for compressed files.
> 
> The idea is from Xiang.
> 
> Signed-off-by: Yue Hu <huyue2@yulong.com>

Sorry about delay. Finally I seeked some time to look into this...

> ---
> v3:
> - based on commit 9fe440d0ac03 ("erofs-utils: mkfs: introduce --quiet option").
> - move h_idata_size ahead of h_advise for compatibility.
> - rename feature/update messages which i think they are more exact.
> 
> v2:
> - add 2 bytes to record compressed size of tail-pcluster suggested from
>   Xiang and update related code.
> 
>  include/erofs/internal.h |  1 +
>  include/erofs_fs.h       | 10 ++++-
>  lib/compress.c           | 83 ++++++++++++++++++++++++++++++----------
>  lib/compressor.c         |  9 +++--
>  lib/decompress.c         |  4 ++
>  lib/inode.c              | 42 ++++++++++----------
>  mkfs/main.c              |  7 ++++
>  7 files changed, 108 insertions(+), 48 deletions(-)
> 
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 8b154ed..54e5939 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -110,6 +110,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
>  EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
>  EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
>  EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
> +EROFS_FEATURE_FUNCS(tail_packing, incompat, INCOMPAT_TAIL_PACKING)

How about moving fuse support as [PATCH 1/2] and introducing these?

Also the on-disk feature name is somewhat confusing, maybe
INCOMPAT_ZTAILPACKING

is better?

>  EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
>  
>  #define EROFS_I_EA_INITED	(1 << 0)
> diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> index 66a68e3..0ebcd5b 100644
> --- a/include/erofs_fs.h
> +++ b/include/erofs_fs.h
> @@ -22,11 +22,13 @@
>  #define EROFS_FEATURE_INCOMPAT_COMPR_CFGS	0x00000002
>  #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER	0x00000002
>  #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE	0x00000004
> +#define EROFS_FEATURE_INCOMPAT_TAIL_PACKING	0x00000010
>  #define EROFS_ALL_FEATURE_INCOMPAT		\
>  	(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
>  	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
>  	 EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
> -	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
> +	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
> +	 EROFS_FEATURE_INCOMPAT_TAIL_PACKING)
>  
>  #define EROFS_SB_EXTSLOT_SIZE	16
>  
> @@ -266,13 +268,17 @@ struct z_erofs_lz4_cfgs {
>   *                                  (4B) + 2B + (4B) if compacted 2B is on.
>   * bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
>   * bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
> + * bit 3 : tail-packing inline (un)compressed data
>   */
>  #define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
>  #define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
>  #define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
> +#define Z_EROFS_ADVISE_INLINE_DATA		0x0008
>  
>  struct z_erofs_map_header {
> -	__le32	h_reserved1;
> +	__le16	h_reserved1;
> +	/* record the (un)compressed size of tail-packing pcluster */
> +	__le16	h_idata_size;
>  	__le16	h_advise;
>  	/*
>  	 * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
> diff --git a/lib/compress.c b/lib/compress.c
> index 6ca5bed..d7d60b9 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -70,11 +70,10 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
>  
>  	di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
>  
> -	/* whether the tail-end uncompressed block or not */
> +	/* whether the tail-end (un)compressed block or not */
>  	if (!d1) {
> -		/* TODO: tail-packing inline compressed data */
> -		DBG_BUGON(!raw);
> -		type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
> +		type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :
> +			Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
>  		advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
>  
>  		di.di_advise = advise;
> @@ -162,6 +161,17 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
>  	return cfg.c_pclusterblks_def;
>  }
>  
> +static int z_erofs_fill_inline_data(struct erofs_inode *inode, char *data,
> +				    unsigned int len)
> +{
> +	inode->idata_size = len;
> +	inode->idata = malloc(inode->idata_size);
> +	if (!inode->idata)
> +		return -ENOMEM;
> +	memcpy(inode->idata, data, inode->idata_size);
> +	return 0;
> +}
> +
>  static int vle_compress_one(struct erofs_inode *inode,
>  			    struct z_erofs_vle_compress_ctx *ctx,
>  			    bool final)
> @@ -172,15 +182,20 @@ static int vle_compress_one(struct erofs_inode *inode,
>  	int ret;
>  	static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ];
>  	char *const dst = dstbuf + EROFS_BLKSIZ;
> +	bool tail_pcluster = false;
>  
>  	while (len) {
> -		const unsigned int pclustersize =
> +		unsigned int pclustersize =
>  			z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
>  		bool raw;
>  
> -		if (len <= pclustersize) {
> +		if (!tail_pcluster && len <= pclustersize) {
>  			if (final) {
> -				if (len <= EROFS_BLKSIZ)
> +				/* TODO: compress with 2 pclusters */
> +				if (erofs_sb_has_tail_packing()) {
> +					tail_pcluster = true;
> +					pclustersize = EROFS_BLKSIZ;
> +				} else if (len <= EROFS_BLKSIZ)
>  					goto nocompression;
>  			} else {

It seems somewhat messy...
Just a rough thought, how about the following code (not even tested...)

--- a/lib/compress.c
+++ b/lib/compress.c
@@ -182,21 +182,22 @@ static int vle_compress_one(struct erofs_inode *inode,
        int ret;
        static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ];
        char *const dst = dstbuf + EROFS_BLKSIZ;
-       bool tail_pcluster = false;
+       bool trailing = false;

        while (len) {
                unsigned int pclustersize =
                        z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
                bool raw;

-               if (!tail_pcluster && len <= pclustersize) {
+               if (len <= pclustersize) {
                        if (final) {
                                /* TODO: compress with 2 pclusters */
                                if (erofs_sb_has_tail_packing()) {
-                                       tail_pcluster = true;
+                                       trailing = true;
                                        pclustersize = EROFS_BLKSIZ;
-                               } else if (len <= EROFS_BLKSIZ)
+                               } else if (len <= EROFS_BLKSIZ) {
                                        goto nocompression;
+                               }
                        } else {
                                break;
                        }
@@ -211,23 +212,29 @@ static int vle_compress_one(struct erofs_inode *inode,
                                          inode->i_srcpath,
                                          erofs_strerror(ret));
                        }
-                       if (tail_pcluster && len < EROFS_BLKSIZ) {
+                       if (trailing && len < EROFS_BLKSIZ) {
                                ret = z_erofs_fill_inline_data(inode,
                                        (char *)(ctx->queue + ctx->head), len);
                                if (ret)
                                        return ret;
                                count = len;
                                raw = true;
+                               ctx->compressedblks = 0;
+                       } else {
+nocompression:
+                               ret = write_uncompressed_extent(ctx, &len, dst);
+                               if (ret < 0)
+                                       return ret;
+                               count = ret;
                                ctx->compressedblks = 1;
-                               goto add_head;
+                               raw = true;
                        }
-nocompression:
-                       ret = write_uncompressed_extent(ctx, &len, dst);
-                       if (ret < 0)
+               } else if (trailing && ret < EROFS_BLKSIZ && len == count) {
+                       ret = z_erofs_fill_inline_data(inode, dst, ret);
+                       if (ret)
                                return ret;
-                       count = ret;
-                       ctx->compressedblks = 1;
-                       raw = true;
+                       raw = false;
+                       ctx->compressedblks = 0;
                } else {
                        if (tail_pcluster && ret < EROFS_BLKSIZ &&
                            !(len - count)) {
@@ -262,7 +269,6 @@ nocompression:
                        raw = false;
                }

-add_head:
                ctx->head += count;
                /* write compression indexes for this pcluster */
                vle_write_indexes(ctx, count, raw);

Thanks,
Gao Xiang

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data
  2021-12-10  3:52   ` Gao Xiang
@ 2021-12-10  4:01     ` Yue Hu
  0 siblings, 0 replies; 6+ messages in thread
From: Yue Hu @ 2021-12-10  4:01 UTC (permalink / raw)
  To: Gao Xiang; +Cc: linux-erofs, Yue Hu, geshifei, zhangwen, shaojunjun

Hi Xiang,

On Fri, 10 Dec 2021 11:52:08 +0800
Gao Xiang <hsiangkao@linux.alibaba.com> wrote:

> Hi Yue,
> 
> On Wed, Nov 17, 2021 at 05:22:00PM +0800, Yue Hu wrote:
> > Currently, we only support tail-end inline data for uncompressed
> > files, let's support it as well for compressed files.
> > 
> > The idea is from Xiang.
> > 
> > Signed-off-by: Yue Hu <huyue2@yulong.com>  
> 
> Sorry about delay. Finally I seeked some time to look into this...
> 
> > ---
> > v3:
> > - based on commit 9fe440d0ac03 ("erofs-utils: mkfs: introduce --quiet option").
> > - move h_idata_size ahead of h_advise for compatibility.
> > - rename feature/update messages which i think they are more exact.
> > 
> > v2:
> > - add 2 bytes to record compressed size of tail-pcluster suggested from
> >   Xiang and update related code.
> > 
> >  include/erofs/internal.h |  1 +
> >  include/erofs_fs.h       | 10 ++++-
> >  lib/compress.c           | 83 ++++++++++++++++++++++++++++++----------
> >  lib/compressor.c         |  9 +++--
> >  lib/decompress.c         |  4 ++
> >  lib/inode.c              | 42 ++++++++++----------
> >  mkfs/main.c              |  7 ++++
> >  7 files changed, 108 insertions(+), 48 deletions(-)
> > 
> > diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> > index 8b154ed..54e5939 100644
> > --- a/include/erofs/internal.h
> > +++ b/include/erofs/internal.h
> > @@ -110,6 +110,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
> >  EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
> >  EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
> >  EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
> > +EROFS_FEATURE_FUNCS(tail_packing, incompat, INCOMPAT_TAIL_PACKING)  
> 
> How about moving fuse support as [PATCH 1/2] and introducing these?
> 
> Also the on-disk feature name is somewhat confusing, maybe
> INCOMPAT_ZTAILPACKING
> 
> is better?

ok, i will do that.

> 
> >  EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
> >  
> >  #define EROFS_I_EA_INITED	(1 << 0)
> > diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> > index 66a68e3..0ebcd5b 100644
> > --- a/include/erofs_fs.h
> > +++ b/include/erofs_fs.h
> > @@ -22,11 +22,13 @@
> >  #define EROFS_FEATURE_INCOMPAT_COMPR_CFGS	0x00000002
> >  #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER	0x00000002
> >  #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE	0x00000004
> > +#define EROFS_FEATURE_INCOMPAT_TAIL_PACKING	0x00000010
> >  #define EROFS_ALL_FEATURE_INCOMPAT		\
> >  	(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
> >  	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
> >  	 EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
> > -	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
> > +	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
> > +	 EROFS_FEATURE_INCOMPAT_TAIL_PACKING)
> >  
> >  #define EROFS_SB_EXTSLOT_SIZE	16
> >  
> > @@ -266,13 +268,17 @@ struct z_erofs_lz4_cfgs {
> >   *                                  (4B) + 2B + (4B) if compacted 2B is on.
> >   * bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
> >   * bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
> > + * bit 3 : tail-packing inline (un)compressed data
> >   */
> >  #define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
> >  #define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
> >  #define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
> > +#define Z_EROFS_ADVISE_INLINE_DATA		0x0008
> >  
> >  struct z_erofs_map_header {
> > -	__le32	h_reserved1;
> > +	__le16	h_reserved1;
> > +	/* record the (un)compressed size of tail-packing pcluster */
> > +	__le16	h_idata_size;
> >  	__le16	h_advise;
> >  	/*
> >  	 * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
> > diff --git a/lib/compress.c b/lib/compress.c
> > index 6ca5bed..d7d60b9 100644
> > --- a/lib/compress.c
> > +++ b/lib/compress.c
> > @@ -70,11 +70,10 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
> >  
> >  	di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
> >  
> > -	/* whether the tail-end uncompressed block or not */
> > +	/* whether the tail-end (un)compressed block or not */
> >  	if (!d1) {
> > -		/* TODO: tail-packing inline compressed data */
> > -		DBG_BUGON(!raw);
> > -		type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
> > +		type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :
> > +			Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
> >  		advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
> >  
> >  		di.di_advise = advise;
> > @@ -162,6 +161,17 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode)
> >  	return cfg.c_pclusterblks_def;
> >  }
> >  
> > +static int z_erofs_fill_inline_data(struct erofs_inode *inode, char *data,
> > +				    unsigned int len)
> > +{
> > +	inode->idata_size = len;
> > +	inode->idata = malloc(inode->idata_size);
> > +	if (!inode->idata)
> > +		return -ENOMEM;
> > +	memcpy(inode->idata, data, inode->idata_size);
> > +	return 0;
> > +}
> > +
> >  static int vle_compress_one(struct erofs_inode *inode,
> >  			    struct z_erofs_vle_compress_ctx *ctx,
> >  			    bool final)
> > @@ -172,15 +182,20 @@ static int vle_compress_one(struct erofs_inode *inode,
> >  	int ret;
> >  	static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ];
> >  	char *const dst = dstbuf + EROFS_BLKSIZ;
> > +	bool tail_pcluster = false;
> >  
> >  	while (len) {
> > -		const unsigned int pclustersize =
> > +		unsigned int pclustersize =
> >  			z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
> >  		bool raw;
> >  
> > -		if (len <= pclustersize) {
> > +		if (!tail_pcluster && len <= pclustersize) {
> >  			if (final) {
> > -				if (len <= EROFS_BLKSIZ)
> > +				/* TODO: compress with 2 pclusters */
> > +				if (erofs_sb_has_tail_packing()) {
> > +					tail_pcluster = true;
> > +					pclustersize = EROFS_BLKSIZ;
> > +				} else if (len <= EROFS_BLKSIZ)
> >  					goto nocompression;
> >  			} else {  
> 
> It seems somewhat messy...
> Just a rough thought, how about the following code (not even tested...)
> 
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -182,21 +182,22 @@ static int vle_compress_one(struct erofs_inode *inode,
>         int ret;
>         static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ];
>         char *const dst = dstbuf + EROFS_BLKSIZ;
> -       bool tail_pcluster = false;
> +       bool trailing = false;
> 
>         while (len) {
>                 unsigned int pclustersize =
>                         z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ;
>                 bool raw;
> 
> -               if (!tail_pcluster && len <= pclustersize) {
> +               if (len <= pclustersize) {
>                         if (final) {
>                                 /* TODO: compress with 2 pclusters */
>                                 if (erofs_sb_has_tail_packing()) {
> -                                       tail_pcluster = true;
> +                                       trailing = true;
>                                         pclustersize = EROFS_BLKSIZ;
> -                               } else if (len <= EROFS_BLKSIZ)
> +                               } else if (len <= EROFS_BLKSIZ) {
>                                         goto nocompression;
> +                               }
>                         } else {
>                                 break;
>                         }
> @@ -211,23 +212,29 @@ static int vle_compress_one(struct erofs_inode *inode,
>                                           inode->i_srcpath,
>                                           erofs_strerror(ret));
>                         }
> -                       if (tail_pcluster && len < EROFS_BLKSIZ) {
> +                       if (trailing && len < EROFS_BLKSIZ) {
>                                 ret = z_erofs_fill_inline_data(inode,
>                                         (char *)(ctx->queue + ctx->head), len);
>                                 if (ret)
>                                         return ret;
>                                 count = len;
>                                 raw = true;
> +                               ctx->compressedblks = 0;
> +                       } else {
> +nocompression:
> +                               ret = write_uncompressed_extent(ctx, &len, dst);
> +                               if (ret < 0)
> +                                       return ret;
> +                               count = ret;
>                                 ctx->compressedblks = 1;
> -                               goto add_head;
> +                               raw = true;
>                         }
> -nocompression:
> -                       ret = write_uncompressed_extent(ctx, &len, dst);
> -                       if (ret < 0)
> +               } else if (trailing && ret < EROFS_BLKSIZ && len == count) {
> +                       ret = z_erofs_fill_inline_data(inode, dst, ret);
> +                       if (ret)
>                                 return ret;
> -                       count = ret;
> -                       ctx->compressedblks = 1;
> -                       raw = true;
> +                       raw = false;
> +                       ctx->compressedblks = 0;
>                 } else {
>                         if (tail_pcluster && ret < EROFS_BLKSIZ &&
>                             !(len - count)) {
> @@ -262,7 +269,6 @@ nocompression:
>                         raw = false;
>                 }
> 
> -add_head:
>                 ctx->head += count;
>                 /* write compression indexes for this pcluster */
>                 vle_write_indexes(ctx, count, raw);
> 
> Thanks,
> Gao Xiang

Let me think above.

Thanks.


^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2021-12-10  4:03 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
     [not found] <cover.1637140430.git.huyue2@yulong.com>
2021-11-17  9:22 ` [RFC PATCH v3 1/2] erofs-utils: support tail-packing inline compressed data Yue Hu
2021-12-10  3:52   ` Gao Xiang
2021-12-10  4:01     ` Yue Hu
2021-11-17  9:22 ` [RFC PATCH v3 2/2] erofs-utils: fuse: " Yue Hu via Linux-erofs
2021-12-06  0:39   ` Gao Xiang
2021-12-06  3:40     ` Yue Hu

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).