[RFC PATCH v0 0/4] erofs-utils: add LZMA fixed-sized output compression support

[RFC PATCH v0 0/4] erofs-utils: add LZMA fixed-sized output compression support

[Gao Xiang via Linux-erofs]

From: Gao Xiang <hsiangkao@aol.com>

Hi folks,

Thanks to Lasse, liblzma finally has a preliminary usable fixed-sized
output compression support. I've made a quick integration and it seems
work. The preliminary C/R result of this is:
 459382784  enwik9_4k.squashfs.xz.img
 401649664  enwik9_8k.squashfs.xz.img
 371617792  enwik9_4k.lzma.erofs.img
 360972288  enwik9_16k.squashfs.xz.img

* Also -zlzma,1,
 389115904  enwik9_4k.lzma.erofs.img

So it's worthwhile to take the further step and get an entire solution.

TODOs:
 - make a formal configure.ac / Makefile.am rather than such hack;
 - save DICT_SIZE to super block rather than using a fixed value;
 - support threaded compression to boost up build time (including
   segment compression);
 - look on sorting out kernel side support.

Thanks,
Gao Xiang

Cc: Lasse Collin <lasse.collin@tukaani.org>

Gao Xiang (4):
  erofs-utils: clevel set up as an individual function
  erofs-utils: add liblzma dependency
  erofs-utils: mkfs: add LZMA algorithm support
  erofs-utils: fuse: add LZMA algorithm support

 fuse/Makefile.am         |  2 +-
 include/erofs_fs.h       |  3 +-
 lib/Makefile.am          |  2 +
 lib/compress.c           | 16 ++++---
 lib/compressor.c         | 23 +++++++---
 lib/compressor.h         |  7 ++-
 lib/compressor_liblzma.c | 93 ++++++++++++++++++++++++++++++++++++++++
 lib/compressor_lz4.c     |  1 -
 lib/compressor_lz4hc.c   | 21 ++++++---
 lib/data.c               |  7 +--
 lib/decompress.c         | 62 +++++++++++++++++++++++++++
 mkfs/Makefile.am         |  2 +-
 12 files changed, 211 insertions(+), 28 deletions(-)
 create mode 100644 lib/compressor_liblzma.c

-- 
2.24.0

[RFC PATCH v0 1/4] erofs-utils: clevel set up as an individual function

[Gao Xiang via Linux-erofs]

From: Gao Xiang <hsiangkao@aol.com>

Compression level passed in can be verified then. Also, in order for
preparation of upcoming LZMA fixed-sized output compression.

Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
 lib/compress.c         | 11 ++++++-----
 lib/compressor.c       | 22 +++++++++++++++-------
 lib/compressor.h       |  6 ++++--
 lib/compressor_lz4.c   |  1 -
 lib/compressor_lz4hc.c | 21 +++++++++++++++------
 5 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/compress.c b/lib/compress.c
index 86db940b6edd..2e8deaf81907 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -20,8 +20,6 @@
 #include "compressor.h"
 
 static struct erofs_compress compresshandle;
-static int compressionlevel;
-
 static struct z_erofs_map_header mapheader;
 
 struct z_erofs_vle_compress_ctx {
@@ -165,8 +163,7 @@ static int vle_compress_one(struct erofs_inode *inode,
 		}
 
 		count = len;
-		ret = erofs_compress_destsize(h, compressionlevel,
-					      ctx->queue + ctx->head,
+		ret = erofs_compress_destsize(h, ctx->queue + ctx->head,
 					      &count, dst, EROFS_BLKSIZ);
 		if (ret <= 0) {
 			if (ret != -EAGAIN) {
@@ -520,7 +517,11 @@ int z_erofs_compress_init(void)
 	if (!cfg.c_compr_alg_master)
 		return 0;
 
-	compressionlevel = cfg.c_compr_level_master < 0 ?
+	ret = erofs_compressor_setlevel(&compresshandle, cfg.c_compr_level_master);
+	if (ret)
+		return ret;
+
+	compresshandle.compression_level = cfg.c_compr_level_master < 0 ?
 		compresshandle.alg->default_level :
 		cfg.c_compr_level_master;
 
diff --git a/lib/compressor.c b/lib/compressor.c
index b2434e0e5418..e28aa8f934c0 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -22,11 +22,8 @@ static struct erofs_compressor *compressors[] = {
 };
 
 int erofs_compress_destsize(struct erofs_compress *c,
-			    int compression_level,
-			    void *src,
-			    unsigned int *srcsize,
-			    void *dst,
-			    unsigned int dstsize)
+			    void *src, unsigned int *srcsize,
+			    void *dst, unsigned int dstsize)
 {
 	int ret;
 
@@ -34,8 +31,7 @@ int erofs_compress_destsize(struct erofs_compress *c,
 	if (!c->alg->compress_destsize)
 		return -ENOTSUP;
 
-	ret = c->alg->compress_destsize(c, compression_level,
-					src, srcsize, dst, dstsize);
+	ret = c->alg->compress_destsize(c, src, srcsize, dst, dstsize);
 	if (ret < 0)
 		return ret;
 
@@ -50,6 +46,18 @@ const char *z_erofs_list_available_compressors(unsigned int i)
 	return i >= ARRAY_SIZE(compressors) ? NULL : compressors[i]->name;
 }
 
+int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
+{
+	DBG_BUGON(!c->alg);
+	if (c->alg->setlevel)
+		return c->alg->setlevel(c, compression_level);
+
+	if (compression_level >= 0)
+		return -EINVAL;
+	c->compression_level = 0;
+	return 0;
+}
+
 int erofs_compressor_init(struct erofs_compress *c, char *alg_name)
 {
 	int ret, i;
diff --git a/lib/compressor.h b/lib/compressor.h
index b2471c41ca4e..8c4e72cfa8b9 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -21,9 +21,9 @@ struct erofs_compressor {
 
 	int (*init)(struct erofs_compress *c);
 	int (*exit)(struct erofs_compress *c);
+	int (*setlevel)(struct erofs_compress *c, int compression_level);
 
 	int (*compress_destsize)(struct erofs_compress *c,
-				 int compress_level,
 				 void *src, unsigned int *srcsize,
 				 void *dst, unsigned int dstsize);
 };
@@ -32,6 +32,7 @@ struct erofs_compress {
 	struct erofs_compressor *alg;
 
 	unsigned int compress_threshold;
+	unsigned int compression_level;
 
 	/* *_destsize specific */
 	unsigned int destsize_alignsize;
@@ -45,10 +46,11 @@ struct erofs_compress {
 extern struct erofs_compressor erofs_compressor_lz4;
 extern struct erofs_compressor erofs_compressor_lz4hc;
 
-int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
+int erofs_compress_destsize(struct erofs_compress *c,
 			    void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize);
 
+int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
 int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
 int erofs_compressor_exit(struct erofs_compress *c);
 
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index 8540a0d01cbb..c2141f39e95b 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -11,7 +11,6 @@
 #include "compressor.h"
 
 static int lz4_compress_destsize(struct erofs_compress *c,
-				 int compression_level,
 				 void *src, unsigned int *srcsize,
 				 void *dst, unsigned int dstsize)
 {
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index 6680563986c3..5188a7cf6ea4 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -12,16 +12,13 @@
 #include "compressor.h"
 
 static int lz4hc_compress_destsize(struct erofs_compress *c,
-				   int compression_level,
-				   void *src,
-				   unsigned int *srcsize,
-				   void *dst,
-				   unsigned int dstsize)
+				   void *src, unsigned int *srcsize,
+				   void *dst, unsigned int dstsize)
 {
 	int srcSize = (int)*srcsize;
 	int rc = LZ4_compress_HC_destSize(c->private_data, src, dst,
 					  &srcSize, (int)dstsize,
-					  compression_level);
+					  c->compression_level);
 	if (!rc)
 		return -EFAULT;
 	*srcsize = srcSize;
@@ -47,12 +44,24 @@ static int compressor_lz4hc_init(struct erofs_compress *c)
 	return 0;
 }
 
+static int compressor_lz4hc_setlevel(struct erofs_compress *c,
+				     int compression_level)
+{
+	if (compression_level > LZ4HC_CLEVEL_MAX)
+		return -EINVAL;
+
+	c->compression_level = compression_level < 0 ?
+		LZ4HC_CLEVEL_DEFAULT : compression_level;
+	return 0;
+}
+
 struct erofs_compressor erofs_compressor_lz4hc = {
 	.name = "lz4hc",
 	.default_level = LZ4HC_CLEVEL_DEFAULT,
 	.best_level = LZ4HC_CLEVEL_MAX,
 	.init = compressor_lz4hc_init,
 	.exit = compressor_lz4hc_exit,
+	.setlevel = compressor_lz4hc_setlevel,
 	.compress_destsize = lz4hc_compress_destsize,
 };
 
-- 
2.24.0

[RFC PATCH v0 2/4] erofs-utils: add liblzma dependency

[Gao Xiang via Linux-erofs]

From: Gao Xiang <hsiangkao@aol.com>

liblzma [1] has an experimental fixed-sized output LZMA compression
support now. Let's use it to have a try!

TODO:
	complete configure.ac / Makefile.am

Cc: Lasse Collin <lasse.collin@tukaani.org>
Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
 fuse/Makefile.am | 2 +-
 lib/Makefile.am  | 1 +
 mkfs/Makefile.am | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/fuse/Makefile.am b/fuse/Makefile.am
index e7757bc7d047..e471cb810c8c 100644
--- a/fuse/Makefile.am
+++ b/fuse/Makefile.am
@@ -7,4 +7,4 @@ erofsfuse_SOURCES = dir.c main.c
 erofsfuse_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 erofsfuse_CFLAGS += -DFUSE_USE_VERSION=26 ${libfuse_CFLAGS} ${libselinux_CFLAGS}
 erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse_LIBS} ${liblz4_LIBS} ${libselinux_LIBS}
-
+erofsfuse_LDADD += -L/tmp/xz-test/lib -llzma
diff --git a/lib/Makefile.am b/lib/Makefile.am
index f21dc35eda51..e048a16d73f2 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -13,3 +13,4 @@ liberofs_la_SOURCES += compressor_lz4hc.c
 endif
 endif
 
+liberofs_la_CFLAGS += -I/tmp/xz-test/include
diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am
index 8b8e05132bc0..7993cb76ee23 100644
--- a/mkfs/Makefile.am
+++ b/mkfs/Makefile.am
@@ -7,4 +7,4 @@ AM_CPPFLAGS = ${libuuid_CFLAGS} ${libselinux_CFLAGS}
 mkfs_erofs_SOURCES = main.c
 mkfs_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 mkfs_erofs_LDADD = ${libuuid_LIBS} $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} ${liblz4_LIBS}
-
+mkfs_erofs_LDADD += -L/tmp/xz-test/lib -llzma
-- 
2.24.0

[RFC PATCH v0 3/4] erofs-utils: mkfs: add LZMA algorithm support

[Gao Xiang via Linux-erofs]

From: Gao Xiang <hsiangkao@aol.com>

This patch adds LZMA compression algorithms to erofs-utils
compression framework by using upstream liblzma.

Cc: Lasse Collin <lasse.collin@tukaani.org>
Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
 include/erofs_fs.h       |  3 +-
 lib/Makefile.am          |  1 +
 lib/compress.c           |  5 ++-
 lib/compressor.c         |  1 +
 lib/compressor.h         |  1 +
 lib/compressor_liblzma.c | 93 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 102 insertions(+), 2 deletions(-)
 create mode 100644 lib/compressor_liblzma.c

diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index a69f179a51a5..24772043cb8b 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -193,7 +193,8 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
 
 /* available compression algorithm types (for h_algorithmtype) */
 enum {
-	Z_EROFS_COMPRESSION_LZ4	= 0,
+	Z_EROFS_COMPRESSION_LZ4		= 0,
+	Z_EROFS_COMPRESSION_LZMA	= 1,
 	Z_EROFS_COMPRESSION_MAX
 };
 
diff --git a/lib/Makefile.am b/lib/Makefile.am
index e048a16d73f2..e9b3255a2653 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -14,3 +14,4 @@ endif
 endif
 
 liberofs_la_CFLAGS += -I/tmp/xz-test/include
+liberofs_la_SOURCES += compressor_liblzma.c
diff --git a/lib/compress.c b/lib/compress.c
index 2e8deaf81907..d77998fb6857 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -493,6 +493,8 @@ static int erofs_get_compress_algorithm_id(const char *name)
 {
 	if (!strcmp(name, "lz4") || !strcmp(name, "lz4hc"))
 		return Z_EROFS_COMPRESSION_LZ4;
+	if (!strcmp(name, "lzma"))
+		return Z_EROFS_COMPRESSION_LZMA;
 	return -ENOTSUP;
 }
 
@@ -511,7 +513,8 @@ int z_erofs_compress_init(void)
 	 * clear LZ4_0PADDING feature for old kernel compatibility.
 	 */
 	if (!cfg.c_compr_alg_master ||
-	    strncmp(cfg.c_compr_alg_master, "lz4", 3))
+	    (strncmp(cfg.c_compr_alg_master, "lz4", 3) &&
+	     strcmp(cfg.c_compr_alg_master, "lzma")))
 		erofs_sb_clear_lz4_0padding();
 
 	if (!cfg.c_compr_alg_master)
diff --git a/lib/compressor.c b/lib/compressor.c
index e28aa8f934c0..cc39fc7a685a 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -19,6 +19,7 @@ static struct erofs_compressor *compressors[] = {
 #endif
 		&erofs_compressor_lz4,
 #endif
+		&erofs_compressor_lzma,
 };
 
 int erofs_compress_destsize(struct erofs_compress *c,
diff --git a/lib/compressor.h b/lib/compressor.h
index 8c4e72cfa8b9..e783ca1de509 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -45,6 +45,7 @@ struct erofs_compress {
 /* list of compression algorithms */
 extern struct erofs_compressor erofs_compressor_lz4;
 extern struct erofs_compressor erofs_compressor_lz4hc;
+extern struct erofs_compressor erofs_compressor_lzma;
 
 int erofs_compress_destsize(struct erofs_compress *c,
 			    void *src, unsigned int *srcsize,
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
new file mode 100644
index 000000000000..af135c2cd32b
--- /dev/null
+++ b/lib/compressor_liblzma.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor_liblzma.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25@huawei.com>
+ */
+#include <stdlib.h>
+#include <lzma.h>
+#include "erofs/internal.h"
+#include "compressor.h"
+
+struct erofs_liblzma_context {
+	lzma_options_lzma opt;
+	lzma_stream strm;
+};
+
+static int erofs_liblzma_compress_destsize(struct erofs_compress *c,
+					   void *src, unsigned int *srcsize,
+					   void *dst, unsigned int dstsize)
+{
+	struct erofs_liblzma_context *ctx = c->private_data;
+	lzma_stream *strm = &ctx->strm;
+
+	lzma_ret ret = lzma_erofs_encoder(strm, &ctx->opt);
+	if (ret != LZMA_OK)
+		return -EFAULT;
+
+	strm->next_in = src;
+	strm->avail_in = *srcsize;
+	strm->next_out = dst;
+	strm->avail_out = dstsize;
+
+	ret = lzma_code(strm, LZMA_FINISH);
+	if (ret != LZMA_STREAM_END)
+		return -EBADMSG;
+
+	*srcsize = strm->total_in;
+	return strm->total_out;
+}
+
+static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
+{
+	struct erofs_liblzma_context *ctx = c->private_data;
+
+	if (!ctx)
+		return -EINVAL;
+
+	lzma_end(&ctx->strm);
+	free(ctx);
+	return 0;
+}
+
+static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
+					     int compression_level)
+{
+	struct erofs_liblzma_context *ctx = c->private_data;
+
+	if (compression_level < 0)
+		compression_level = LZMA_PRESET_DEFAULT;
+
+	if (lzma_lzma_preset(&ctx->opt, compression_level))
+		return -EINVAL;
+
+	ctx->opt.dict_size = 64U << 10;
+	c->compression_level = compression_level;
+	return 0;
+}
+
+static int erofs_compressor_liblzma_init(struct erofs_compress *c)
+{
+	struct erofs_liblzma_context *ctx;
+
+	c->alg = &erofs_compressor_lzma;
+	ctx = malloc(sizeof(*ctx));
+	if (!ctx)
+		return -ENOMEM;
+	ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
+	c->private_data = ctx;
+	return 0;
+}
+
+struct erofs_compressor erofs_compressor_lzma = {
+	.name = "lzma",
+	.default_level = LZMA_PRESET_DEFAULT,
+	.best_level = LZMA_PRESET_EXTREME,
+	.init = erofs_compressor_liblzma_init,
+	.exit = erofs_compressor_liblzma_exit,
+	.setlevel = erofs_compressor_liblzma_setlevel,
+	.compress_destsize = erofs_liblzma_compress_destsize,
+};
+
-- 
2.24.0

[RFC PATCH v0 4/4] erofs-utils: fuse: add LZMA algorithm support

[Gao Xiang via Linux-erofs]

From: Gao Xiang <hsiangkao@aol.com>

This patch adds LZMA compression algorithms to erofsfuse to test
if LZMA fixed-sized output algorithm works as expected.

Cc: Lasse Collin <lasse.collin@tukaani.org>
Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
 lib/data.c       |  7 +++---
 lib/decompress.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 3 deletions(-)

diff --git a/lib/data.c b/lib/data.c
index 3781846743aa..4247dd3a33aa 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -146,9 +146,10 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
 		if (ret < 0)
 			return -EIO;
 
-		algorithmformat = map.m_flags & EROFS_MAP_ZIPPED ?
-						Z_EROFS_COMPRESSION_LZ4 :
-						Z_EROFS_COMPRESSION_SHIFTED;
+		if (map.m_flags & EROFS_MAP_ZIPPED)
+			algorithmformat = inode->z_algorithmtype[0];
+		else
+			algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
 
 		/*
 		 * trim to the needed size if the returned extent is quite
diff --git a/lib/decompress.c b/lib/decompress.c
index 490c4bc771da..fd9f24bcbeb6 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -10,6 +10,66 @@
 #include "erofs/decompress.h"
 #include "erofs/err.h"
 
+#include <lzma.h>
+
+/* FIXME! need to record it into superblock */
+#define DICT_SIZE	(64U << 10)
+
+static int z_erofs_decompress_lzma(struct z_erofs_decompress_req *rq)
+{
+	int ret = 0;
+	u8 *dest = (u8 *)rq->out;
+	u8 *src = (u8 *)rq->in;
+	u8 *buff = NULL;
+	unsigned int inputmargin = 0;
+	lzma_stream strm;
+	lzma_ret ret2;
+
+	if (!erofs_sb_has_lz4_0padding())
+		return -EFSCORRUPTED;
+
+	while (!src[inputmargin & ~PAGE_MASK])
+		if (!(++inputmargin & ~PAGE_MASK))
+			break;
+
+	if (inputmargin >= rq->inputsize)
+		return -EFSCORRUPTED;
+
+	if (rq->decodedskip) {
+		buff = malloc(rq->decodedlength);
+		if (!buff)
+			return -ENOMEM;
+		dest = buff;
+	}
+
+	strm = (lzma_stream)LZMA_STREAM_INIT;
+	strm.next_in = src + inputmargin;
+	strm.avail_in = rq->inputsize - inputmargin;
+	strm.next_out = dest;
+	strm.avail_out = rq->decodedlength;
+
+	ret2 = lzma_erofs_decoder(&strm, strm.avail_in, rq->decodedlength,
+				  !rq->partial_decoding, DICT_SIZE);
+	if (ret2 != LZMA_OK) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret2 = lzma_code(&strm, LZMA_FINISH);
+	if (ret2 != LZMA_STREAM_END) {
+		ret = -EFSCORRUPTED;
+		goto out;
+	}
+
+	if (rq->decodedskip)
+		memcpy(rq->out, dest + rq->decodedskip,
+		       rq->decodedlength - rq->decodedskip);
+out:
+	if (buff)
+		free(buff);
+	return ret;
+}
+
 #ifdef LZ4_ENABLED
 #include <lz4.h>
 
@@ -84,5 +144,7 @@ int z_erofs_decompress(struct z_erofs_decompress_req *rq)
 	if (rq->alg == Z_EROFS_COMPRESSION_LZ4)
 		return z_erofs_decompress_lz4(rq);
 #endif
+	if (rq->alg == Z_EROFS_COMPRESSION_LZMA)
+		return z_erofs_decompress_lzma(rq);
 	return -EOPNOTSUPP;
 }
-- 
2.24.0