* [RFC PATCH v0 1/4] erofs-utils: clevel set up as an individual function
2021-01-17 17:45 ` [RFC PATCH v0 0/4] erofs-utils: add LZMA fixed-sized output compression support Gao Xiang via Linux-erofs
@ 2021-01-17 17:46 ` Gao Xiang via Linux-erofs
2021-01-17 17:46 ` [RFC PATCH v0 2/4] erofs-utils: add liblzma dependency Gao Xiang via Linux-erofs
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-01-17 17:46 UTC (permalink / raw)
To: linux-erofs
From: Gao Xiang <hsiangkao@aol.com>
Compression level passed in can be verified then. Also, in order for
preparation of upcoming LZMA fixed-sized output compression.
Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
lib/compress.c | 11 ++++++-----
lib/compressor.c | 22 +++++++++++++++-------
lib/compressor.h | 6 ++++--
lib/compressor_lz4.c | 1 -
lib/compressor_lz4hc.c | 21 +++++++++++++++------
5 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/lib/compress.c b/lib/compress.c
index 86db940b6edd..2e8deaf81907 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -20,8 +20,6 @@
#include "compressor.h"
static struct erofs_compress compresshandle;
-static int compressionlevel;
-
static struct z_erofs_map_header mapheader;
struct z_erofs_vle_compress_ctx {
@@ -165,8 +163,7 @@ static int vle_compress_one(struct erofs_inode *inode,
}
count = len;
- ret = erofs_compress_destsize(h, compressionlevel,
- ctx->queue + ctx->head,
+ ret = erofs_compress_destsize(h, ctx->queue + ctx->head,
&count, dst, EROFS_BLKSIZ);
if (ret <= 0) {
if (ret != -EAGAIN) {
@@ -520,7 +517,11 @@ int z_erofs_compress_init(void)
if (!cfg.c_compr_alg_master)
return 0;
- compressionlevel = cfg.c_compr_level_master < 0 ?
+ ret = erofs_compressor_setlevel(&compresshandle, cfg.c_compr_level_master);
+ if (ret)
+ return ret;
+
+ compresshandle.compression_level = cfg.c_compr_level_master < 0 ?
compresshandle.alg->default_level :
cfg.c_compr_level_master;
diff --git a/lib/compressor.c b/lib/compressor.c
index b2434e0e5418..e28aa8f934c0 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -22,11 +22,8 @@ static struct erofs_compressor *compressors[] = {
};
int erofs_compress_destsize(struct erofs_compress *c,
- int compression_level,
- void *src,
- unsigned int *srcsize,
- void *dst,
- unsigned int dstsize)
+ void *src, unsigned int *srcsize,
+ void *dst, unsigned int dstsize)
{
int ret;
@@ -34,8 +31,7 @@ int erofs_compress_destsize(struct erofs_compress *c,
if (!c->alg->compress_destsize)
return -ENOTSUP;
- ret = c->alg->compress_destsize(c, compression_level,
- src, srcsize, dst, dstsize);
+ ret = c->alg->compress_destsize(c, src, srcsize, dst, dstsize);
if (ret < 0)
return ret;
@@ -50,6 +46,18 @@ const char *z_erofs_list_available_compressors(unsigned int i)
return i >= ARRAY_SIZE(compressors) ? NULL : compressors[i]->name;
}
+int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level)
+{
+ DBG_BUGON(!c->alg);
+ if (c->alg->setlevel)
+ return c->alg->setlevel(c, compression_level);
+
+ if (compression_level >= 0)
+ return -EINVAL;
+ c->compression_level = 0;
+ return 0;
+}
+
int erofs_compressor_init(struct erofs_compress *c, char *alg_name)
{
int ret, i;
diff --git a/lib/compressor.h b/lib/compressor.h
index b2471c41ca4e..8c4e72cfa8b9 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -21,9 +21,9 @@ struct erofs_compressor {
int (*init)(struct erofs_compress *c);
int (*exit)(struct erofs_compress *c);
+ int (*setlevel)(struct erofs_compress *c, int compression_level);
int (*compress_destsize)(struct erofs_compress *c,
- int compress_level,
void *src, unsigned int *srcsize,
void *dst, unsigned int dstsize);
};
@@ -32,6 +32,7 @@ struct erofs_compress {
struct erofs_compressor *alg;
unsigned int compress_threshold;
+ unsigned int compression_level;
/* *_destsize specific */
unsigned int destsize_alignsize;
@@ -45,10 +46,11 @@ struct erofs_compress {
extern struct erofs_compressor erofs_compressor_lz4;
extern struct erofs_compressor erofs_compressor_lz4hc;
-int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
+int erofs_compress_destsize(struct erofs_compress *c,
void *src, unsigned int *srcsize,
void *dst, unsigned int dstsize);
+int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level);
int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
int erofs_compressor_exit(struct erofs_compress *c);
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index 8540a0d01cbb..c2141f39e95b 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -11,7 +11,6 @@
#include "compressor.h"
static int lz4_compress_destsize(struct erofs_compress *c,
- int compression_level,
void *src, unsigned int *srcsize,
void *dst, unsigned int dstsize)
{
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index 6680563986c3..5188a7cf6ea4 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -12,16 +12,13 @@
#include "compressor.h"
static int lz4hc_compress_destsize(struct erofs_compress *c,
- int compression_level,
- void *src,
- unsigned int *srcsize,
- void *dst,
- unsigned int dstsize)
+ void *src, unsigned int *srcsize,
+ void *dst, unsigned int dstsize)
{
int srcSize = (int)*srcsize;
int rc = LZ4_compress_HC_destSize(c->private_data, src, dst,
&srcSize, (int)dstsize,
- compression_level);
+ c->compression_level);
if (!rc)
return -EFAULT;
*srcsize = srcSize;
@@ -47,12 +44,24 @@ static int compressor_lz4hc_init(struct erofs_compress *c)
return 0;
}
+static int compressor_lz4hc_setlevel(struct erofs_compress *c,
+ int compression_level)
+{
+ if (compression_level > LZ4HC_CLEVEL_MAX)
+ return -EINVAL;
+
+ c->compression_level = compression_level < 0 ?
+ LZ4HC_CLEVEL_DEFAULT : compression_level;
+ return 0;
+}
+
struct erofs_compressor erofs_compressor_lz4hc = {
.name = "lz4hc",
.default_level = LZ4HC_CLEVEL_DEFAULT,
.best_level = LZ4HC_CLEVEL_MAX,
.init = compressor_lz4hc_init,
.exit = compressor_lz4hc_exit,
+ .setlevel = compressor_lz4hc_setlevel,
.compress_destsize = lz4hc_compress_destsize,
};
--
2.24.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [RFC PATCH v0 3/4] erofs-utils: mkfs: add LZMA algorithm support
2021-01-17 17:45 ` [RFC PATCH v0 0/4] erofs-utils: add LZMA fixed-sized output compression support Gao Xiang via Linux-erofs
2021-01-17 17:46 ` [RFC PATCH v0 1/4] erofs-utils: clevel set up as an individual function Gao Xiang via Linux-erofs
2021-01-17 17:46 ` [RFC PATCH v0 2/4] erofs-utils: add liblzma dependency Gao Xiang via Linux-erofs
@ 2021-01-17 17:46 ` Gao Xiang via Linux-erofs
2021-01-17 17:46 ` [RFC PATCH v0 4/4] erofs-utils: fuse: " Gao Xiang via Linux-erofs
3 siblings, 0 replies; 5+ messages in thread
From: Gao Xiang via Linux-erofs @ 2021-01-17 17:46 UTC (permalink / raw)
To: linux-erofs; +Cc: Lasse Collin
From: Gao Xiang <hsiangkao@aol.com>
This patch adds LZMA compression algorithms to erofs-utils
compression framework by using upstream liblzma.
Cc: Lasse Collin <lasse.collin@tukaani.org>
Signed-off-by: Gao Xiang <hsiangkao@aol.com>
---
include/erofs_fs.h | 3 +-
lib/Makefile.am | 1 +
lib/compress.c | 5 ++-
lib/compressor.c | 1 +
lib/compressor.h | 1 +
lib/compressor_liblzma.c | 93 ++++++++++++++++++++++++++++++++++++++++
6 files changed, 102 insertions(+), 2 deletions(-)
create mode 100644 lib/compressor_liblzma.c
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index a69f179a51a5..24772043cb8b 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -193,7 +193,8 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
/* available compression algorithm types (for h_algorithmtype) */
enum {
- Z_EROFS_COMPRESSION_LZ4 = 0,
+ Z_EROFS_COMPRESSION_LZ4 = 0,
+ Z_EROFS_COMPRESSION_LZMA = 1,
Z_EROFS_COMPRESSION_MAX
};
diff --git a/lib/Makefile.am b/lib/Makefile.am
index e048a16d73f2..e9b3255a2653 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -14,3 +14,4 @@ endif
endif
liberofs_la_CFLAGS += -I/tmp/xz-test/include
+liberofs_la_SOURCES += compressor_liblzma.c
diff --git a/lib/compress.c b/lib/compress.c
index 2e8deaf81907..d77998fb6857 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -493,6 +493,8 @@ static int erofs_get_compress_algorithm_id(const char *name)
{
if (!strcmp(name, "lz4") || !strcmp(name, "lz4hc"))
return Z_EROFS_COMPRESSION_LZ4;
+ if (!strcmp(name, "lzma"))
+ return Z_EROFS_COMPRESSION_LZMA;
return -ENOTSUP;
}
@@ -511,7 +513,8 @@ int z_erofs_compress_init(void)
* clear LZ4_0PADDING feature for old kernel compatibility.
*/
if (!cfg.c_compr_alg_master ||
- strncmp(cfg.c_compr_alg_master, "lz4", 3))
+ (strncmp(cfg.c_compr_alg_master, "lz4", 3) &&
+ strcmp(cfg.c_compr_alg_master, "lzma")))
erofs_sb_clear_lz4_0padding();
if (!cfg.c_compr_alg_master)
diff --git a/lib/compressor.c b/lib/compressor.c
index e28aa8f934c0..cc39fc7a685a 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -19,6 +19,7 @@ static struct erofs_compressor *compressors[] = {
#endif
&erofs_compressor_lz4,
#endif
+ &erofs_compressor_lzma,
};
int erofs_compress_destsize(struct erofs_compress *c,
diff --git a/lib/compressor.h b/lib/compressor.h
index 8c4e72cfa8b9..e783ca1de509 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -45,6 +45,7 @@ struct erofs_compress {
/* list of compression algorithms */
extern struct erofs_compressor erofs_compressor_lz4;
extern struct erofs_compressor erofs_compressor_lz4hc;
+extern struct erofs_compressor erofs_compressor_lzma;
int erofs_compress_destsize(struct erofs_compress *c,
void *src, unsigned int *srcsize,
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
new file mode 100644
index 000000000000..af135c2cd32b
--- /dev/null
+++ b/lib/compressor_liblzma.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor_liblzma.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25@huawei.com>
+ */
+#include <stdlib.h>
+#include <lzma.h>
+#include "erofs/internal.h"
+#include "compressor.h"
+
+struct erofs_liblzma_context {
+ lzma_options_lzma opt;
+ lzma_stream strm;
+};
+
+static int erofs_liblzma_compress_destsize(struct erofs_compress *c,
+ void *src, unsigned int *srcsize,
+ void *dst, unsigned int dstsize)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+ lzma_stream *strm = &ctx->strm;
+
+ lzma_ret ret = lzma_erofs_encoder(strm, &ctx->opt);
+ if (ret != LZMA_OK)
+ return -EFAULT;
+
+ strm->next_in = src;
+ strm->avail_in = *srcsize;
+ strm->next_out = dst;
+ strm->avail_out = dstsize;
+
+ ret = lzma_code(strm, LZMA_FINISH);
+ if (ret != LZMA_STREAM_END)
+ return -EBADMSG;
+
+ *srcsize = strm->total_in;
+ return strm->total_out;
+}
+
+static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (!ctx)
+ return -EINVAL;
+
+ lzma_end(&ctx->strm);
+ free(ctx);
+ return 0;
+}
+
+static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
+ int compression_level)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (compression_level < 0)
+ compression_level = LZMA_PRESET_DEFAULT;
+
+ if (lzma_lzma_preset(&ctx->opt, compression_level))
+ return -EINVAL;
+
+ ctx->opt.dict_size = 64U << 10;
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_liblzma_init(struct erofs_compress *c)
+{
+ struct erofs_liblzma_context *ctx;
+
+ c->alg = &erofs_compressor_lzma;
+ ctx = malloc(sizeof(*ctx));
+ if (!ctx)
+ return -ENOMEM;
+ ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
+ c->private_data = ctx;
+ return 0;
+}
+
+struct erofs_compressor erofs_compressor_lzma = {
+ .name = "lzma",
+ .default_level = LZMA_PRESET_DEFAULT,
+ .best_level = LZMA_PRESET_EXTREME,
+ .init = erofs_compressor_liblzma_init,
+ .exit = erofs_compressor_liblzma_exit,
+ .setlevel = erofs_compressor_liblzma_setlevel,
+ .compress_destsize = erofs_liblzma_compress_destsize,
+};
+
--
2.24.0
^ permalink raw reply related [flat|nested] 5+ messages in thread