[PATCH v4 0/5] fs/squashfs: Add new decompression algorithms

[PATCH v4 0/5] fs/squashfs: Add new decompression algorithms

[Joao Marcos Costa]

Hello,

Following the SquashFS support, this series adds support for LZO and ZSTD
algorithms.

Changes in v4:
	- Replace sqfs_decompress() 1st parameter by a reference to
	squashfs_ctxt.
	- Undo changes in squashfs_ctxt instance at sqfs.c
Changes in v3:
	- Reorganize patches order.
	- Remove memory leak from sqfs_zstd_decompress().
Changes in v2:
	- Change tests architecture.
	- Add tests for LZO and ZSTD.
	- Skip compression algorithms unsupported by the host's squashfs-tools
	version.
	- Add sqfs_decompress_init() and sqfs_decompress_cleanup() to SquashFS
	support.

Joao Marcos Costa (5):
  fs/squashfs: Add init and clean-up functions to decompression
  fs/squashfs: replace sqfs_decompress() parameter
  fs/squashfs: add support for ZSTD decompression
  fs/squashfs: add support for LZO decompression
  test/py: Add tests for LZO and ZSTD

 fs/squashfs/sqfs.c                            |  40 +++----
 fs/squashfs/sqfs_decompressor.c               | 104 +++++++++++++++++-
 fs/squashfs/sqfs_decompressor.h               |   7 +-
 fs/squashfs/sqfs_filesystem.h                 |  12 +-
 .../test_fs/test_squashfs/sqfs_common.py      |  76 +++++++++----
 .../test_fs/test_squashfs/test_sqfs_load.py   |  40 ++++---
 .../test_fs/test_squashfs/test_sqfs_ls.py     |  33 ++++--
 7 files changed, 236 insertions(+), 76 deletions(-)

-- 
2.17.1

[PATCH v4 1/5] fs/squashfs: Add init and clean-up functions to decompression

[Joao Marcos Costa]

Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
functions are called respectively in sqfs_probe() and sqfs_close(). For
now, only ZSTD requires an initialization logic. ZSTD support will be
added in a follow-up commit.

Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
it can no longer be local to sqfs.c.

Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
---
Changes in v4:
	- Undo changes from v3 in squashfs_ctxt instance scope.
Changes in v3:
	- Remove memory leak from sqfs_zstd_decompress()
	- Rename structure in sqfs.c and remove the 'static' so it can be
	exported to sqfs_decompressor.c
Changes in v2:
	- This patch was not present in the previous version.
 fs/squashfs/sqfs.c              | 15 +++++++++------
 fs/squashfs/sqfs_decompressor.c | 34 ++++++++++++++++++++++++++++++---
 fs/squashfs/sqfs_decompressor.h |  3 +++
 fs/squashfs/sqfs_filesystem.h   |  9 ++++++++-
 4 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 340e5ebdb9..598b42cd34 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -23,12 +23,6 @@
 #include "sqfs_filesystem.h"
 #include "sqfs_utils.h"
 
-struct squashfs_ctxt {
-	struct disk_partition cur_part_info;
-	struct blk_desc *cur_dev;
-	struct squashfs_super_block *sblk;
-};
-
 static struct squashfs_ctxt ctxt;
 
 static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf)
@@ -1023,6 +1017,14 @@ int sqfs_probe(struct blk_desc *fs_dev_desc, struct disk_partition *fs_partition
 
 	ctxt.sblk = sblk;
 
+	ret = sqfs_decompressor_init(&ctxt);
+
+	if (ret) {
+		ctxt.cur_dev = NULL;
+		free(ctxt.sblk);
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -1525,6 +1527,7 @@ void sqfs_close(void)
 {
 	free(ctxt.sblk);
 	ctxt.cur_dev = NULL;
+	sqfs_decompressor_cleanup(&ctxt);
 }
 
 void sqfs_closedir(struct fs_dir_stream *dirs)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 09ca6cf6d0..633a8a1818 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -14,9 +14,37 @@
 #endif
 
 #include "sqfs_decompressor.h"
-#include "sqfs_filesystem.h"
 #include "sqfs_utils.h"
 
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
+{
+	u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression);
+
+	switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+	case SQFS_COMP_ZLIB:
+		break;
+#endif
+	default:
+		printf("Error: unknown compression type.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
+{
+	u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression);
+
+	switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+	case SQFS_COMP_ZLIB:
+		break;
+#endif
+	}
+}
+
 #if IS_ENABLED(CONFIG_ZLIB)
 static void zlib_decompression_status(int ret)
 {
@@ -35,14 +63,14 @@ static void zlib_decompression_status(int ret)
 #endif
 
 int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
-		    void *source, u32 lenp)
+		    void *source, u32 src_len)
 {
 	int ret = 0;
 
 	switch (comp_type) {
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
-		ret = uncompress(dest, dest_len, source, lenp);
+		ret = uncompress(dest, dest_len, source, src_len);
 		if (ret) {
 			zlib_decompression_status(ret);
 			return -EINVAL;
diff --git a/fs/squashfs/sqfs_decompressor.h b/fs/squashfs/sqfs_decompressor.h
index 378965dda8..450257e5ce 100644
--- a/fs/squashfs/sqfs_decompressor.h
+++ b/fs/squashfs/sqfs_decompressor.h
@@ -9,6 +9,7 @@
 #define SQFS_DECOMPRESSOR_H
 
 #include <stdint.h>
+#include "sqfs_filesystem.h"
 
 #define SQFS_COMP_ZLIB 1
 #define SQFS_COMP_LZMA 2
@@ -54,5 +55,7 @@ union squashfs_compression_opts {
 
 int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
 		    void *source, u32 lenp);
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt);
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt);
 
 #endif /* SQFS_DECOMPRESSOR_H */
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index d63e3a41ad..ff2b0b1d34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -9,8 +9,9 @@
 #define SQFS_FILESYSTEM_H
 
 #include <asm/unaligned.h>
-#include <stdint.h>
 #include <fs.h>
+#include <part.h>
+#include <stdint.h>
 
 #define SQFS_UNCOMPRESSED_DATA 0x0002
 #define SQFS_MAGIC_NUMBER 0x73717368
@@ -72,6 +73,12 @@ struct squashfs_super_block {
 	__le64 export_table_start;
 };
 
+struct squashfs_ctxt {
+	struct disk_partition cur_part_info;
+	struct blk_desc *cur_dev;
+	struct squashfs_super_block *sblk;
+};
+
 struct squashfs_directory_index {
 	u32 index;
 	u32 start;
-- 
2.17.1

[PATCH v4 2/5] fs/squashfs: replace sqfs_decompress() parameter

[Joao Marcos Costa]

Replace 'u16 comp_type' by a reference to squashfs_ctxt structure.

Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
---
Changes in v4:
	- This patch was not present in the previous version.
 fs/squashfs/sqfs.c              | 25 +++++++++----------------
 fs/squashfs/sqfs_decompressor.c |  5 +++--
 fs/squashfs/sqfs_decompressor.h |  4 ++--
 3 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 598b42cd34..9bd7b98d88 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -104,9 +104,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
 	struct squashfs_super_block *sblk = ctxt.sblk;
 	unsigned long dest_len;
 	int block, offset, ret;
-	u16 header, comp_type;
-
-	comp_type = get_unaligned_le16(&sblk->compression);
+	u16 header;
 
 	if (inode_fragment_index >= get_unaligned_le32(&sblk->fragments))
 		return -EINVAL;
@@ -165,7 +163,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
 	if (SQFS_COMPRESSED_METADATA(header)) {
 		src_len = SQFS_METADATA_SIZE(header);
 		dest_len = SQFS_METADATA_BLOCK_SIZE;
-		ret = sqfs_decompress(comp_type, entries, &dest_len, metadata,
+		ret = sqfs_decompress(&ctxt, entries, &dest_len, metadata,
 				      src_len);
 		if (ret) {
 			ret = -EINVAL;
@@ -629,13 +627,12 @@ static int sqfs_read_inode_table(unsigned char **inode_table)
 {
 	struct squashfs_super_block *sblk = ctxt.sblk;
 	u64 start, n_blks, table_offset, table_size;
-	int j, ret = 0, metablks_count, comp_type;
+	int j, ret = 0, metablks_count;
 	unsigned char *src_table, *itb;
 	u32 src_len, dest_offset = 0;
 	unsigned long dest_len;
 	bool compressed;
 
-	comp_type = get_unaligned_le16(&sblk->compression);
 	table_size = get_unaligned_le64(&sblk->directory_table_start) -
 		get_unaligned_le64(&sblk->inode_table_start);
 	start = get_unaligned_le64(&sblk->inode_table_start) /
@@ -680,7 +677,7 @@ static int sqfs_read_inode_table(unsigned char **inode_table)
 		sqfs_read_metablock(itb, table_offset, &compressed, &src_len);
 		if (compressed) {
 			dest_len = SQFS_METADATA_BLOCK_SIZE;
-			ret = sqfs_decompress(comp_type, *inode_table +
+			ret = sqfs_decompress(&ctxt, *inode_table +
 					      dest_offset, &dest_len,
 					      src_table, src_len);
 			if (ret) {
@@ -711,15 +708,13 @@ free_itb:
 static int sqfs_read_directory_table(unsigned char **dir_table, u32 **pos_list)
 {
 	u64 start, n_blks, table_offset, table_size;
-	int j, ret = 0, metablks_count = -1, comp_type;
 	struct squashfs_super_block *sblk = ctxt.sblk;
+	int j, ret = 0, metablks_count = -1;
 	unsigned char *src_table, *dtb;
 	u32 src_len, dest_offset = 0;
 	unsigned long dest_len;
 	bool compressed;
 
-	comp_type = get_unaligned_le16(&sblk->compression);
-
 	/* DIRECTORY TABLE */
 	table_size = get_unaligned_le64(&sblk->fragment_table_start) -
 		get_unaligned_le64(&sblk->directory_table_start);
@@ -773,7 +768,7 @@ static int sqfs_read_directory_table(unsigned char **dir_table, u32 **pos_list)
 		sqfs_read_metablock(dtb, table_offset, &compressed, &src_len);
 		if (compressed) {
 			dest_len = SQFS_METADATA_BLOCK_SIZE;
-			ret = sqfs_decompress(comp_type, *dir_table +
+			ret = sqfs_decompress(&ctxt, *dir_table +
 					      (j * SQFS_METADATA_BLOCK_SIZE),
 					      &dest_len, src_table, src_len);
 			if (ret) {
@@ -1197,7 +1192,7 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len,
 	char *dir, *fragment_block, *datablock = NULL, *data_buffer = NULL;
 	char *fragment, *file, *resolved, *data;
 	u64 start, n_blks, table_size, data_offset, table_offset;
-	int ret, j, i_number, comp_type, datablk_count = 0;
+	int ret, j, i_number, datablk_count = 0;
 	struct squashfs_super_block *sblk = ctxt.sblk;
 	struct squashfs_fragment_block_entry frag_entry;
 	struct squashfs_file_info finfo = {0};
@@ -1213,8 +1208,6 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len,
 
 	*actread = 0;
 
-	comp_type = get_unaligned_le16(&sblk->compression);
-
 	/*
 	 * sqfs_opendir will uncompress inode and directory tables, and will
 	 * return a pointer to the directory that contains the requested file.
@@ -1346,7 +1339,7 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len,
 		/* Load the data */
 		if (SQFS_COMPRESSED_BLOCK(finfo.blk_sizes[j])) {
 			dest_len = get_unaligned_le32(&sblk->block_size);
-			ret = sqfs_decompress(comp_type, datablock, &dest_len,
+			ret = sqfs_decompress(&ctxt, datablock, &dest_len,
 					      data, table_size);
 			if (ret)
 				goto free_buffer;
@@ -1396,7 +1389,7 @@ int sqfs_read(const char *filename, void *buf, loff_t offset, loff_t len,
 			goto free_fragment;
 		}
 
-		ret = sqfs_decompress(comp_type, fragment_block, &dest_len,
+		ret = sqfs_decompress(&ctxt, fragment_block, &dest_len,
 				      (void *)fragment  + table_offset,
 				      frag_entry.size);
 		if (ret) {
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 633a8a1818..cf388ace58 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -62,9 +62,10 @@ static void zlib_decompression_status(int ret)
 }
 #endif
 
-int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
-		    void *source, u32 src_len)
+int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
+		    unsigned long *dest_len, void *source, u32 src_len)
 {
+	u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression);
 	int ret = 0;
 
 	switch (comp_type) {
diff --git a/fs/squashfs/sqfs_decompressor.h b/fs/squashfs/sqfs_decompressor.h
index 450257e5ce..892cfb6974 100644
--- a/fs/squashfs/sqfs_decompressor.h
+++ b/fs/squashfs/sqfs_decompressor.h
@@ -53,8 +53,8 @@ union squashfs_compression_opts {
 	struct squashfs_lzo_opts *lzo;
 };
 
-int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
-		    void *source, u32 lenp);
+int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
+		    unsigned long *dest_len, void *source, u32 src_len);
 int sqfs_decompressor_init(struct squashfs_ctxt *ctxt);
 void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt);
 
-- 
2.17.1

[PATCH v4 3/5] fs/squashfs: add support for ZSTD decompression

[Joao Marcos Costa]

Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
can upper bound the decompressed size, which will be the SquashFS data
block (or metadata block) size, so there is no need to use streaming
API. Add ZSTD's worskpace to squashfs_ctxt structure.

Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
---
Changes in v4:
	- Add parameter to sqfs_zstd_decompress().
Changes in v3:
	- Remove memory leak from sqfs_zstd_decompress().
Changes in v2:
	- No changes since last version.
 fs/squashfs/sqfs_decompressor.c | 42 +++++++++++++++++++++++++++++++++
 fs/squashfs/sqfs_filesystem.h   |  3 +++
 2 files changed, 45 insertions(+)

diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index cf388ace58..9457ee5f0f 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -13,6 +13,10 @@
 #include <u-boot/zlib.h>
 #endif
 
+#if IS_ENABLED(CONFIG_ZSTD)
+#include <linux/zstd.h>
+#endif
+
 #include "sqfs_decompressor.h"
 #include "sqfs_utils.h"
 
@@ -24,6 +28,13 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
 		break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+	case SQFS_COMP_ZSTD:
+		ctxt->zstd_workspace = malloc(ZSTD_DCtxWorkspaceBound());
+		if (!ctxt->zstd_workspace)
+			return -ENOMEM;
+		break;
 #endif
 	default:
 		printf("Error: unknown compression type.\n");
@@ -41,6 +52,11 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
 		break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+	case SQFS_COMP_ZSTD:
+		free(ctxt->zstd_workspace);
+		break;
 #endif
 	}
 }
@@ -62,6 +78,22 @@ static void zlib_decompression_status(int ret)
 }
 #endif
 
+#if IS_ENABLED(CONFIG_ZSTD)
+static int sqfs_zstd_decompress(struct squashfs_ctxt *ctxt, void *dest,
+				unsigned long dest_len, void *source, u32 src_len)
+{
+	ZSTD_DCtx *ctx;
+	size_t wsize;
+	int ret;
+
+	wsize = ZSTD_DCtxWorkspaceBound();
+	ctx = ZSTD_initDCtx(ctxt->zstd_workspace, wsize);
+	ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
+
+	return ZSTD_isError(ret);
+}
+#endif /* CONFIG_ZSTD */
+
 int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
 		    unsigned long *dest_len, void *source, u32 src_len)
 {
@@ -79,6 +111,16 @@ int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
 
 		break;
 #endif
+#if IS_ENABLED(CONFIG_ZSTD)
+	case SQFS_COMP_ZSTD:
+		ret = sqfs_zstd_decompress(ctxt, dest, *dest_len, source, src_len);
+		if (ret) {
+			printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret));
+			return -EINVAL;
+		}
+
+		break;
+#endif
 	default:
 		printf("Error: unknown compression type.\n");
 		return -EINVAL;
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index ff2b0b1d34..856cd15e34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -77,6 +77,9 @@ struct squashfs_ctxt {
 	struct disk_partition cur_part_info;
 	struct blk_desc *cur_dev;
 	struct squashfs_super_block *sblk;
+#if IS_ENABLED(CONFIG_ZSTD)
+	void *zstd_workspace;
+#endif
 };
 
 struct squashfs_directory_index {
-- 
2.17.1

[PATCH v4 4/5] fs/squashfs: add support for LZO decompression

[Joao Marcos Costa]

Add call to lzo's lzo1x_decompress_safe() into sqfs_decompress().

U-Boot's LZO sources may still have some unsolved issues that could make the
decompression crash when dealing with fragmented files, so those should be
avoided. The "-no-fragments" option can be passed to mksquashfs.

Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
---
Changes in v4:
	- No changes since the last version.
Changes in v3:
	- Change commit message.
Changes in v2:
	- Change commit message.
 fs/squashfs/sqfs_decompressor.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 9457ee5f0f..d69ddb24a4 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,6 +9,11 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+
+#if IS_ENABLED(CONFIG_LZO)
+#include <linux/lzo.h>
+#endif
+
 #if IS_ENABLED(CONFIG_ZLIB)
 #include <u-boot/zlib.h>
 #endif
@@ -25,6 +30,10 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
 	u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression);
 
 	switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+	case SQFS_COMP_LZO:
+		break;
+#endif
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
 		break;
@@ -49,6 +58,10 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
 	u16 comp_type = get_unaligned_le16(&ctxt->sblk->compression);
 
 	switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+	case SQFS_COMP_LZO:
+		break;
+#endif
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
 		break;
@@ -101,6 +114,18 @@ int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
 	int ret = 0;
 
 	switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+	case SQFS_COMP_LZO: {
+		size_t lzo_dest_len = *dest_len;
+		ret = lzo1x_decompress_safe(source, src_len, dest, &lzo_dest_len);
+		if (ret) {
+			printf("LZO decompression failed. Error code: %d\n", ret);
+			return -EINVAL;
+		}
+
+		break;
+	}
+#endif
 #if IS_ENABLED(CONFIG_ZLIB)
 	case SQFS_COMP_ZLIB:
 		ret = uncompress(dest, dest_len, source, src_len);
-- 
2.17.1

[PATCH v4 5/5] test/py: Add tests for LZO and ZSTD

[Joao Marcos Costa]

Improve SquashFS tests architecture. Add 'Compression' class. LZO
algorithm may crash if the file is fragmented, so the fragments are
disabled when testing LZO.

Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>
---
Changes in v4:
	- No changes since the last version.
Changes in v3:
	- No changes since the last version.
Changes in v2:
	- This patch was not present in the previous version.
 .../test_fs/test_squashfs/sqfs_common.py      | 76 ++++++++++++++-----
 .../test_fs/test_squashfs/test_sqfs_load.py   | 40 ++++++----
 .../test_fs/test_squashfs/test_sqfs_ls.py     | 33 +++++---
 3 files changed, 102 insertions(+), 47 deletions(-)

diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index 2dc344d1b2..c96f92c1d8 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -5,6 +5,7 @@
 import os
 import random
 import string
+import subprocess
 
 def sqfs_get_random_letters(size):
     letters = []
@@ -19,24 +20,57 @@ def sqfs_generate_file(path, size):
     file.write(content)
     file.close()
 
-# generate image with three files and a symbolic link
-def sqfs_generate_image(cons):
-    src = os.path.join(cons.config.build_dir, "sqfs_src/")
-    dest = os.path.join(cons.config.build_dir, "sqfs")
-    os.mkdir(src)
-    sqfs_generate_file(src + "frag_only", 100)
-    sqfs_generate_file(src + "blks_frag", 5100)
-    sqfs_generate_file(src + "blks_only", 4096)
-    os.symlink("frag_only", src + "sym")
-    os.system("mksquashfs " + src + " " + dest + " -b 4096 -always-use-fragments")
-
-# removes all files created by sqfs_generate_image()
-def sqfs_clean(cons):
-    src = os.path.join(cons.config.build_dir, "sqfs_src/")
-    dest = os.path.join(cons.config.build_dir, "sqfs")
-    os.remove(src + "frag_only")
-    os.remove(src + "blks_frag")
-    os.remove(src + "blks_only")
-    os.remove(src + "sym")
-    os.rmdir(src)
-    os.remove(dest)
+class Compression:
+    def __init__(self, name, files, sizes, block_size = 4096):
+        self.name = name
+        self.files = files
+        self.sizes = sizes
+        self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
+
+    def add_opt(self, opt):
+        self.mksquashfs_opts += " " + opt
+
+    def gen_image(self, build_dir):
+        src = os.path.join(build_dir, "sqfs_src/")
+        os.mkdir(src)
+        for (f, s) in zip(self.files, self.sizes):
+            sqfs_generate_file(src + f, s)
+
+        # the symbolic link always targets the first file
+        os.symlink(self.files[0], src + "sym")
+
+        sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+        i_o = src + " " + sqfs_img
+        opts = self.mksquashfs_opts
+        try:
+            subprocess.run(["mksquashfs " + i_o + opts], shell = True, check = True)
+        except:
+            print("mksquashfs error. Compression type: " + self.name)
+            raise RuntimeError
+
+    def clean_source(self, build_dir):
+        src = os.path.join(build_dir, "sqfs_src/")
+        for f in self.files:
+            os.remove(src + f)
+        os.remove(src + "sym")
+        os.rmdir(src)
+
+    def cleanup(self, build_dir):
+        self.clean_source(build_dir)
+        sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+        os.remove(sqfs_img)
+
+files = ["blks_only", "blks_frag", "frag_only"]
+sizes = [4096, 5100, 100]
+gzip = Compression("gzip", files, sizes)
+zstd = Compression("zstd", files, sizes)
+lzo = Compression("lzo", files, sizes)
+
+# use fragment blocks for files larger than block_size
+gzip.add_opt("-always-use-fragments")
+zstd.add_opt("-always-use-fragments")
+
+# avoid fragments if lzo is used
+lzo.add_opt("-no-fragments")
+
+comp_opts = [gzip, zstd, lzo]
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index eb1baae5c5..9e90062384 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -12,23 +12,35 @@ from sqfs_common import *
 @pytest.mark.buildconfigspec('fs_squashfs')
 @pytest.mark.requiredtool('mksquashfs')
 def test_sqfs_load(u_boot_console):
-    cons = u_boot_console
-    sqfs_generate_image(cons)
+    build_dir = u_boot_console.config.build_dir
     command = "sqfsload host 0 $kernel_addr_r "
-    path = os.path.join(cons.config.build_dir, "sqfs")
 
-    try:
+    for opt in comp_opts:
+        # generate and load the squashfs image
+        try:
+            opt.gen_image(build_dir)
+        except RuntimeError:
+            opt.clean_source(build_dir)
+            # skip unsupported compression types
+            continue
+
+        path = os.path.join(build_dir, "sqfs-" + opt.name)
         output = u_boot_console.run_command("host bind 0 " + path)
+
         output = u_boot_console.run_command(command + "xxx")
         assert "File not found." in output
-        output = u_boot_console.run_command(command + "frag_only")
-        assert "100 bytes read in" in output
-        output = u_boot_console.run_command(command + "blks_frag")
-        assert "5100 bytes read in" in output
-        output = u_boot_console.run_command(command + "blks_only")
-        assert "4096 bytes read in" in output
+
+        for (f, s) in zip(opt.files, opt.sizes):
+            try:
+                output = u_boot_console.run_command(command + f)
+                assert str(s) in output
+            except:
+                assert False
+                opt.cleanup(build_dir)
+
+        # test symbolic link
         output = u_boot_console.run_command(command + "sym")
-        assert "100 bytes read in" in output
-    except:
-        sqfs_clean(cons)
-    sqfs_clean(cons)
+        assert str(opt.sizes[0]) in output
+
+        # remove generated files
+        opt.cleanup(build_dir)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
index 3a7b75c778..a0dca2e2fc 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
@@ -12,16 +12,25 @@ from sqfs_common import *
 @pytest.mark.buildconfigspec('fs_squashfs')
 @pytest.mark.requiredtool('mksquashfs')
 def test_sqfs_ls(u_boot_console):
-    cons = u_boot_console
-    sqfs_generate_image(cons)
-    path = os.path.join(cons.config.build_dir, "sqfs")
-    try:
+    build_dir = u_boot_console.config.build_dir
+    for opt in comp_opts:
+        try:
+            opt.gen_image(build_dir)
+        except RuntimeError:
+            opt.clean_source(build_dir)
+            # skip unsupported compression types
+            continue
+        path = os.path.join(build_dir, "sqfs-" + opt.name)
         output = u_boot_console.run_command("host bind 0 " + path)
-        output = u_boot_console.run_command("sqfsls host 0")
-        assert "4 file(s), 0 dir(s)" in output
-        assert "<SYM>   sym" in output
-        output = u_boot_console.run_command("sqfsls host 0 xxx")
-        assert "** Cannot find directory. **" in output
-    except:
-        sqfs_clean(cons)
-    sqfs_clean(cons)
+
+        try:
+            # list files in root directory
+            output = u_boot_console.run_command("sqfsls host 0")
+            assert str(len(opt.files) + 1) + " file(s), 0 dir(s)" in output
+            assert "<SYM>   sym" in output
+            output = u_boot_console.run_command("sqfsls host 0 xxx")
+            assert "** Cannot find directory. **" in output
+        except:
+            opt.cleanup(build_dir)
+            assert False
+        opt.cleanup(build_dir)
-- 
2.17.1

[PATCH v4 1/5] fs/squashfs: Add init and clean-up functions to decompression

[Thomas Petazzoni]

Hello,

On Tue, 18 Aug 2020 17:17:21 +0200
Joao Marcos Costa <joaomarcos.costa@bootlin.com> wrote:

> Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
> functions are called respectively in sqfs_probe() and sqfs_close(). For
> now, only ZSTD requires an initialization logic. ZSTD support will be
> added in a follow-up commit.
> 
> Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
> passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
> it can no longer be local to sqfs.c.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Thanks, looks better. One small nit, though.

>  #if IS_ENABLED(CONFIG_ZLIB)
>  static void zlib_decompression_status(int ret)
>  {
> @@ -35,14 +63,14 @@ static void zlib_decompression_status(int ret)
>  #endif
>  
>  int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
> -		    void *source, u32 lenp)
> +		    void *source, u32 src_len)
>  {
>  	int ret = 0;
>  
>  	switch (comp_type) {
>  #if IS_ENABLED(CONFIG_ZLIB)
>  	case SQFS_COMP_ZLIB:
> -		ret = uncompress(dest, dest_len, source, lenp);
> +		ret = uncompress(dest, dest_len, source, src_len);

These two changes are not related to this patch.

Thomas
-- 
Thomas Petazzoni, CTO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

[PATCH v4 1/5] fs/squashfs: Add init and clean-up functions to decompression

[Tom Rini]

On Tue, Aug 18, 2020 at 05:17:21PM +0200, Joao Marcos Costa wrote:

> Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
> functions are called respectively in sqfs_probe() and sqfs_close(). For
> now, only ZSTD requires an initialization logic. ZSTD support will be
> added in a follow-up commit.
> 
> Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
> passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
> it can no longer be local to sqfs.c.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Applied to u-boot/master, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200825/01b7df3f/attachment.sig>

[PATCH v4 2/5] fs/squashfs: replace sqfs_decompress() parameter

[Tom Rini]

On Tue, Aug 18, 2020 at 05:17:22PM +0200, Joao Marcos Costa wrote:

> Replace 'u16 comp_type' by a reference to squashfs_ctxt structure.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Applied to u-boot/master, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200825/cd7ed944/attachment.sig>

[PATCH v4 3/5] fs/squashfs: add support for ZSTD decompression

[Tom Rini]

On Tue, Aug 18, 2020 at 05:17:23PM +0200, Joao Marcos Costa wrote:

> Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
> can upper bound the decompressed size, which will be the SquashFS data
> block (or metadata block) size, so there is no need to use streaming
> API. Add ZSTD's worskpace to squashfs_ctxt structure.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Applied to u-boot/master, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200825/e925726a/attachment.sig>

[PATCH v4 4/5] fs/squashfs: add support for LZO decompression

[Tom Rini]

On Tue, Aug 18, 2020 at 05:17:24PM +0200, Joao Marcos Costa wrote:

> Add call to lzo's lzo1x_decompress_safe() into sqfs_decompress().
> 
> U-Boot's LZO sources may still have some unsolved issues that could make the
> decompression crash when dealing with fragmented files, so those should be
> avoided. The "-no-fragments" option can be passed to mksquashfs.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Applied to u-boot/master, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200825/6ed01d1c/attachment.sig>

[PATCH v4 5/5] test/py: Add tests for LZO and ZSTD

[Tom Rini]

On Tue, Aug 18, 2020 at 05:17:25PM +0200, Joao Marcos Costa wrote:

> Improve SquashFS tests architecture. Add 'Compression' class. LZO
> algorithm may crash if the file is fragmented, so the fragments are
> disabled when testing LZO.
> 
> Signed-off-by: Joao Marcos Costa <joaomarcos.costa@bootlin.com>

Applied to u-boot/master, thanks!

-- 
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200825/844419c7/attachment.sig>