[PATCH v2 1/5] erofs-utils: introduce dump.erofs

[PATCH v2 1/5] erofs-utils: introduce dump.erofs

[Guo Xuenan]

From: Wang Qi <mpiglet@outlook.com>

Add dump-tool for erofs to facilitate users directly
analyzing the erofs image file.

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Wang Qi <mpiglet@outlook.com>
---
 Makefile.am        |  2 +-
 configure.ac       |  2 ++
 dump/Makefile.am   |  9 +++++
 dump/main.c        | 85 ++++++++++++++++++++++++++++++++++++++++++++++
 include/erofs/io.h |  3 ++
 lib/namei.c        |  5 ++-
 6 files changed, 102 insertions(+), 4 deletions(-)
 create mode 100644 dump/Makefile.am
 create mode 100644 dump/main.c

diff --git a/Makefile.am b/Makefile.am
index b804aa9..fedf7b5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3,7 +3,7 @@
 
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = man lib mkfs
+SUBDIRS = man lib mkfs dump
 if ENABLE_FUSE
 SUBDIRS += fuse
 endif
diff --git a/configure.ac b/configure.ac
index f626064..f4fe548 100644
--- a/configure.ac
+++ b/configure.ac
@@ -280,6 +280,8 @@ AC_CONFIG_FILES([Makefile
 		 man/Makefile
 		 lib/Makefile
 		 mkfs/Makefile
+		 dump/Makefile
 		 fuse/Makefile])
+
 AC_OUTPUT
 
diff --git a/dump/Makefile.am b/dump/Makefile.am
new file mode 100644
index 0000000..8e18c0f
--- /dev/null
+++ b/dump/Makefile.am
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+AUTOMAKE_OPTIONS = foreign
+bin_PROGRAMS     = dump.erofs
+dump_erofs_SOURCES = main.c
+dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}
+
diff --git a/dump/main.c b/dump/main.c
new file mode 100644
index 0000000..8f299ca
--- /dev/null
+++ b/dump/main.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2021-2022 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Wang Qi <mpiglet@outlook.com>
+ *            Guo Xuenan <guoxuenan@huawei.com>
+ */
+
+#include <stdlib.h>
+#include <getopt.h>
+#include <sys/sysmacros.h>
+#include <time.h>
+#include <lz4.h>
+
+#include "erofs/print.h"
+#include "erofs/io.h"
+
+static struct option long_options[] = {
+	{"help", no_argument, 0, 1},
+	{0, 0, 0, 0},
+};
+
+static void usage(void)
+{
+	fputs("usage: [options] erofs-image\n\n"
+		"Dump erofs layout from erofs-image, and [options] are:\n"
+		"--help  display this help and exit.\n"
+		"-V      print the version number of dump.erofs and exit.\n",
+		stderr);
+}
+static void dumpfs_print_version(void)
+{
+	fprintf(stderr, "dump.erofs %s\n", cfg.c_version);
+}
+
+static int dumpfs_parse_options_cfg(int argc, char **argv)
+{
+	int opt;
+
+	while ((opt = getopt_long(argc, argv, "V",
+					long_options, NULL)) != -1) {
+		switch (opt) {
+		case 'V':
+			dumpfs_print_version();
+			exit(0);
+		case 1:
+		    usage();
+		    exit(0);
+		default: /* '?' */
+			return -EINVAL;
+		}
+	}
+
+	if (optind >= argc)
+		return -EINVAL;
+
+	cfg.c_img_path = strdup(argv[optind++]);
+	if (!cfg.c_img_path)
+		return -ENOMEM;
+
+	if (optind < argc) {
+		erofs_err("unexpected argument: %s\n", argv[optind]);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	int err = 0;
+
+	erofs_init_configure();
+	err = dumpfs_parse_options_cfg(argc, argv);
+
+	if (cfg.c_img_path)
+		free(cfg.c_img_path);
+
+	if (err) {
+		if (err == -EINVAL)
+			usage();
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 5574245..00e5de8 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -10,6 +10,7 @@
 #define __EROFS_IO_H
 
 #include <unistd.h>
+#include <sys/types.h>
 #include "internal.h"
 
 #ifndef O_BINARY
@@ -25,6 +26,8 @@ int dev_fillzero(u64 offset, size_t len, bool padding);
 int dev_fsync(void);
 int dev_resize(erofs_blk_t nblocks);
 u64 dev_length(void);
+dev_t erofs_new_decode_dev(u32 dev);
+int erofs_read_inode_from_disk(struct erofs_inode *vi);
 
 static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
 			    u32 nblocks)
diff --git a/lib/namei.c b/lib/namei.c
index 4e06ba4..b45e9d8 100644
--- a/lib/namei.c
+++ b/lib/namei.c
@@ -5,7 +5,6 @@
  * Created by Li Guifu <blucerlee@gmail.com>
  */
 #include <linux/kdev_t.h>
-#include <sys/types.h>
 #include <unistd.h>
 #include <stdio.h>
 #include <errno.h>
@@ -15,7 +14,7 @@
 #include "erofs/print.h"
 #include "erofs/io.h"
 
-static dev_t erofs_new_decode_dev(u32 dev)
+dev_t erofs_new_decode_dev(u32 dev)
 {
 	const unsigned int major = (dev & 0xfff00) >> 8;
 	const unsigned int minor = (dev & 0xff) | ((dev >> 12) & 0xfff00);
@@ -23,7 +22,7 @@ static dev_t erofs_new_decode_dev(u32 dev)
 	return makedev(major, minor);
 }
 
-static int erofs_read_inode_from_disk(struct erofs_inode *vi)
+int erofs_read_inode_from_disk(struct erofs_inode *vi)
 {
 	int ret, ifmt;
 	char buf[sizeof(struct erofs_inode_extended)];
-- 
2.25.4

[PATCH v2 2/5] dump.erofs: add "-s" option to dump superblock information

[Guo Xuenan]

From: Wang Qi <mpiglet@outlook.com>

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Wang Qi <mpiglet@outlook.com>
---
 dump/Makefile.am |  3 +-
 dump/main.c      | 94 +++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 88 insertions(+), 9 deletions(-)

diff --git a/dump/Makefile.am b/dump/Makefile.am
index 8e18c0f..1ba58da 100644
--- a/dump/Makefile.am
+++ b/dump/Makefile.am
@@ -3,7 +3,8 @@
 
 AUTOMAKE_OPTIONS = foreign
 bin_PROGRAMS     = dump.erofs
+AM_CPPFLAGS = ${libuuid_CFLAGS}
 dump_erofs_SOURCES = main.c
 dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
-dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}
+dump_erofs_LDADD = ${libuuid_LIBS} $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}
 
diff --git a/dump/main.c b/dump/main.c
index 8f299ca..33521bf 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -6,25 +6,57 @@
  *            Guo Xuenan <guoxuenan@huawei.com>
  */
 
+#define _GNU_SOURCE
 #include <stdlib.h>
 #include <getopt.h>
 #include <sys/sysmacros.h>
 #include <time.h>
 #include <lz4.h>
-
 #include "erofs/print.h"
 #include "erofs/io.h"
 
+#ifdef HAVE_LIBUUID
+#include <uuid.h>
+#endif
+
+struct dumpcfg {
+	bool print_superblock;
+	bool print_version;
+};
+static struct dumpcfg dumpcfg;
+
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
 	{0, 0, 0, 0},
 };
 
+#define EROFS_FEATURE_COMPAT	0
+#define EROFS_FEATURE_INCOMPAT	1
+
+struct feature {
+	int compat;
+	unsigned int mask;
+	const char *name;
+};
+
+static struct feature feature_lists[] = {
+	{	EROFS_FEATURE_COMPAT, EROFS_FEATURE_COMPAT_SB_CHKSUM,
+		"superblock-checksum"	},
+
+	{	EROFS_FEATURE_INCOMPAT, EROFS_FEATURE_INCOMPAT_LZ4_0PADDING,
+		"lz4-0padding"	},
+	{	EROFS_FEATURE_INCOMPAT, EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER,
+		"big-pcluster"	},
+
+	{	0, 0, 0	},
+};
+
 static void usage(void)
 {
 	fputs("usage: [options] erofs-image\n\n"
 		"Dump erofs layout from erofs-image, and [options] are:\n"
 		"--help  display this help and exit.\n"
+		"-s          print information about superblock\n"
 		"-V      print the version number of dump.erofs and exit.\n",
 		stderr);
 }
@@ -37,9 +69,12 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
 {
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "V",
+	while ((opt = getopt_long(argc, argv, "sV",
 					long_options, NULL)) != -1) {
 		switch (opt) {
+		case 's':
+			dumpcfg.print_superblock = true;
+			break;
 		case 'V':
 			dumpfs_print_version();
 			exit(0);
@@ -65,21 +100,64 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
 	return 0;
 }
 
+static void dumpfs_print_superblock(void)
+{
+	time_t time = sbi.build_time;
+	unsigned int features[] = {sbi.feature_compat, sbi.feature_incompat};
+	char uuid_str[37] = "not available";
+	int i = 0;
+	int j = 0;
+
+	fprintf(stdout, "Filesystem magic number:			0x%04X\n", EROFS_SUPER_MAGIC_V1);
+	fprintf(stdout, "Filesystem blocks:				%lu\n", sbi.blocks);
+	fprintf(stdout, "Filesystem inode metadata start block:		%u\n", sbi.meta_blkaddr);
+	fprintf(stdout, "Filesystem shared xattr metadata start block:	%u\n", sbi.xattr_blkaddr);
+	fprintf(stdout, "Filesystem root nid:				%ld\n", sbi.root_nid);
+	fprintf(stdout, "Filesystem valid inode count:			%lu\n", sbi.inos);
+	fprintf(stdout, "Filesystem created:				%s", ctime(&time));
+	fprintf(stdout, "Filesystem features:				");
+	for (; i < ARRAY_SIZE(features); i++) {
+		for (; j < ARRAY_SIZE(feature_lists); j++) {
+			if (i == feature_lists[j].compat
+				&& (features[i] & feature_lists[j].mask))
+				fprintf(stdout, "%s ", feature_lists[j].name);
+		}
+	}
+	fprintf(stdout, "\n");
+#ifdef HAVE_LIBUUID
+	uuid_unparse_lower(sbi.uuid, uuid_str);
+#endif
+	fprintf(stdout, "Filesystem UUID:				%s\n", uuid_str);
+}
+
 int main(int argc, char **argv)
 {
 	int err = 0;
 
 	erofs_init_configure();
 	err = dumpfs_parse_options_cfg(argc, argv);
-
-	if (cfg.c_img_path)
-		free(cfg.c_img_path);
-
 	if (err) {
 		if (err == -EINVAL)
 			usage();
-		return -1;
+		goto out;
 	}
 
-	return 0;
+	err = dev_open_ro(cfg.c_img_path);
+	if (err) {
+		erofs_err("open image file failed");
+		goto out;
+	}
+
+	err = erofs_read_superblock();
+	if (err) {
+		erofs_err("read superblock failed");
+		goto out;
+	}
+
+	if (dumpcfg.print_superblock)
+		dumpfs_print_superblock();
+out:
+	if (cfg.c_img_path)
+		free(cfg.c_img_path);
+	return err;
 }
-- 
2.25.4

[PATCH v2 3/5] dump.erofs: add -S options for collecting statistics of the whole filesystem

[Guo Xuenan]

From: Wang Qi <mpiglet@outlook.com>

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Wang Qi <mpiglet@outlook.com>
---
 dump/main.c | 364 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 361 insertions(+), 3 deletions(-)

diff --git a/dump/main.c b/dump/main.c
index 33521bf..0778354 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -21,10 +21,63 @@
 
 struct dumpcfg {
 	bool print_superblock;
+	bool print_statistic;
 	bool print_version;
 };
 static struct dumpcfg dumpcfg;
 
+static const char chart_format[] = "%-16s	%-11d %8.2f%% |%-50s|\n";
+static const char header_format[] = "%-16s %11s %16s |%-50s|\n";
+static char *file_types[] = {
+	".so", ".png", ".jpg", ".xml", ".html", ".odex",
+	".vdex", ".apk", ".ttf", ".jar", ".json", ".ogg",
+	".oat", ".art", ".rc", ".otf", ".txt", "others",
+};
+#define OTHERFILETYPE	ARRAY_SIZE(file_types)
+/* (1 << FILE_MAX_SIZE_BITS)KB */
+#define	FILE_MAX_SIZE_BITS	16
+
+static const char * const file_category_types[] = {
+	[EROFS_FT_UNKNOWN] = "unknown type",
+	[EROFS_FT_REG_FILE] = "regular file",
+	[EROFS_FT_DIR] = "directory",
+	[EROFS_FT_CHRDEV] = "char dev",
+	[EROFS_FT_BLKDEV] = "block dev",
+	[EROFS_FT_FIFO] = "FIFO file",
+	[EROFS_FT_SOCK] = "SOCK file",
+	[EROFS_FT_SYMLINK] = "symlink file",
+};
+
+struct statistics {
+	unsigned long blocks;
+	unsigned long files;
+	unsigned long files_total_size;
+	unsigned long files_total_origin_size;
+	double compress_rate;
+	unsigned long compressed_files;
+	unsigned long uncompressed_files;
+
+	unsigned long regular_files;
+	unsigned long dir_files;
+	unsigned long chardev_files;
+	unsigned long blkdev_files;
+	unsigned long fifo_files;
+	unsigned long sock_files;
+	unsigned long symlink_files;
+
+	/* statistics the number of files based on inode_info->flags */
+	unsigned long file_category_stat[EROFS_FT_MAX];
+	/* statistics the number of files based on file name extensions */
+	unsigned int file_type_stat[OTHERFILETYPE];
+	/* statistics the number of files based on file orignial size */
+	unsigned int file_original_size[FILE_MAX_SIZE_BITS + 1];
+	/* statistics the number of files based on the compressed
+	 * size of file
+	 */
+	unsigned int file_comp_size[FILE_MAX_SIZE_BITS + 1];
+};
+static struct statistics stats;
+
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
 	{0, 0, 0, 0},
@@ -57,24 +110,29 @@ static void usage(void)
 		"Dump erofs layout from erofs-image, and [options] are:\n"
 		"--help  display this help and exit.\n"
 		"-s          print information about superblock\n"
+		"-S      print statistic information of the erofs-image\n"
 		"-V      print the version number of dump.erofs and exit.\n",
-		stderr);
+		stdout);
 }
+
 static void dumpfs_print_version(void)
 {
-	fprintf(stderr, "dump.erofs %s\n", cfg.c_version);
+	fprintf(stdout, "dump.erofs %s\n", cfg.c_version);
 }
 
 static int dumpfs_parse_options_cfg(int argc, char **argv)
 {
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "sV",
+	while ((opt = getopt_long(argc, argv, "sSV",
 					long_options, NULL)) != -1) {
 		switch (opt) {
 		case 's':
 			dumpcfg.print_superblock = true;
 			break;
+		case 'S':
+			dumpcfg.print_statistic = true;
+			break;
 		case 'V':
 			dumpfs_print_version();
 			exit(0);
@@ -100,6 +158,303 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
 	return 0;
 }
 
+static int get_file_compressed_size(struct erofs_inode *inode,
+		erofs_off_t *size)
+{
+	*size = 0;
+	switch (inode->datalayout) {
+	case EROFS_INODE_FLAT_INLINE:
+	case EROFS_INODE_FLAT_PLAIN:
+		stats.uncompressed_files++;
+		*size = inode->i_size;
+		break;
+	case EROFS_INODE_FLAT_COMPRESSION_LEGACY:
+	case EROFS_INODE_FLAT_COMPRESSION:
+		stats.compressed_files++;
+		*size = inode->u.i_blocks * EROFS_BLKSIZ;
+		break;
+	default:
+		erofs_err("unknown datalayout");
+		return -1;
+	}
+	return 0;
+}
+
+static int get_file_type(const char *filename)
+{
+	char *postfix = strrchr(filename, '.');
+	int type = 0;
+
+	if (postfix == NULL)
+		return OTHERFILETYPE - 1;
+	while (type < OTHERFILETYPE - 1) {
+		if (strcmp(postfix, file_types[type]) == 0)
+			break;
+		type++;
+	}
+	return type;
+}
+
+static void update_file_size_statatics(erofs_off_t occupied_size,
+		erofs_off_t original_size)
+{
+	int occupied_size_mark;
+	int original_size_mark;
+
+	original_size_mark = 0;
+	occupied_size_mark = 0;
+	occupied_size >>= 10;
+	original_size >>= 10;
+
+	while (occupied_size || original_size) {
+		if (occupied_size) {
+			occupied_size >>= 1;
+			occupied_size_mark++;
+		}
+		if (original_size) {
+			original_size >>= 1;
+			original_size_mark++;
+		}
+	}
+
+	if (original_size_mark >= FILE_MAX_SIZE_BITS)
+		stats.file_original_size[FILE_MAX_SIZE_BITS]++;
+	else
+		stats.file_original_size[original_size_mark]++;
+
+	if (occupied_size_mark >= FILE_MAX_SIZE_BITS)
+		stats.file_comp_size[FILE_MAX_SIZE_BITS]++;
+	else
+		stats.file_comp_size[occupied_size_mark]++;
+}
+
+static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid)
+{
+	struct erofs_inode vi = { .nid = nid};
+	int err;
+	char buf[EROFS_BLKSIZ];
+	char filename[PATH_MAX + 1];
+	erofs_off_t offset;
+
+	err = erofs_read_inode_from_disk(&vi);
+	if (err)
+		return err;
+
+	offset = 0;
+	while (offset < vi.i_size) {
+		erofs_off_t maxsize = min_t(erofs_off_t,
+			vi.i_size - offset, EROFS_BLKSIZ);
+		struct erofs_dirent *de = (void *)buf;
+		struct erofs_dirent *end;
+		unsigned int nameoff;
+
+		err = erofs_pread(&vi, buf, maxsize, offset);
+		if (err)
+			return err;
+
+		nameoff = le16_to_cpu(de->nameoff);
+
+		if (nameoff < sizeof(struct erofs_dirent) ||
+		    nameoff >= PAGE_SIZE) {
+			erofs_err("invalid de[0].nameoff %u @ nid %llu",
+				  nameoff, nid | 0ULL);
+			return -EFSCORRUPTED;
+		}
+		end = (void *)buf + nameoff;
+		while (de < end) {
+			const char *dname;
+			unsigned int dname_len;
+			struct erofs_inode inode = { .nid = de->nid };
+			erofs_off_t occupied_size = 0;
+
+			nameoff = le16_to_cpu(de->nameoff);
+			dname = (char *)buf + nameoff;
+
+			if (de + 1 >= end)
+				dname_len = strnlen(dname, maxsize - nameoff);
+			else
+				dname_len =
+					le16_to_cpu(de[1].nameoff) - nameoff;
+
+			/* a corrupted entry is found */
+			if (nameoff + dname_len > maxsize ||
+				dname_len > EROFS_NAME_LEN) {
+				erofs_err("bogus dirent @ nid %llu",
+						le64_to_cpu(de->nid) | 0ULL);
+				DBG_BUGON(1);
+				return -EFSCORRUPTED;
+			}
+			if (de->nid != nid && de->nid != parent_nid)
+				stats.files++;
+
+			memset(filename, 0, PATH_MAX + 1);
+			memcpy(filename, dname, dname_len);
+			if (de->file_type >= EROFS_FT_MAX) {
+				erofs_err("invalid file type %llu", de->nid);
+				continue;
+			}
+			if (de->file_type != EROFS_FT_DIR)
+				stats.file_category_stat[de->file_type]++;
+			switch (de->file_type) {
+			case EROFS_FT_REG_FILE:
+				err = erofs_read_inode_from_disk(&inode);
+				if (err) {
+					erofs_err("read file inode from disk failed!");
+					return err;
+				}
+				stats.files_total_origin_size += inode.i_size;
+				stats.file_type_stat[get_file_type(filename)]++;
+
+				err = get_file_compressed_size(&inode,
+						&occupied_size);
+				if (err) {
+					erofs_err("get file size failed\n");
+					return err;
+				}
+				stats.files_total_size += occupied_size;
+				update_file_size_statatics(occupied_size, inode.i_size);
+				break;
+
+			case EROFS_FT_DIR:
+				if (de->nid != nid && de->nid != parent_nid) {
+					stats.uncompressed_files++;
+					err = erofs_read_dir(de->nid, nid);
+					if (err) {
+						fprintf(stdout,
+								"parse dir nid %llu error occurred\n",
+								de->nid);
+						return err;
+					}
+					stats.file_category_stat[EROFS_FT_DIR]++;
+				}
+				break;
+			case EROFS_FT_UNKNOWN:
+			case EROFS_FT_CHRDEV:
+			case EROFS_FT_BLKDEV:
+			case EROFS_FT_FIFO:
+			case EROFS_FT_SOCK:
+			case EROFS_FT_SYMLINK:
+				stats.uncompressed_files++;
+				break;
+			default:
+				erofs_err("%d file type not exists", de->file_type);
+			}
+			++de;
+		}
+		offset += maxsize;
+	}
+	return 0;
+}
+
+static void dumpfs_print_statistic_of_filetype(void)
+{
+	fprintf(stdout, "Filesystem total file count:		%lu\n",
+			stats.files);
+	for (int i = 0; i < EROFS_FT_MAX; i++)
+		fprintf(stdout, "Filesystem %s count:		%lu\n",
+			file_category_types[i], stats.file_category_stat[i]);
+}
+
+static void dumpfs_print_chart_row(char *col1, unsigned int col2,
+		double col3, char *col4)
+{
+	char row[500] = {0};
+
+	sprintf(row, chart_format, col1, col2, col3, col4);
+	fprintf(stdout, row);
+}
+
+static void dumpfs_print_chart_of_file(unsigned int *file_counts,
+		unsigned int len)
+{
+	char col1[30];
+	unsigned int col2;
+	double col3;
+	char col4[400];
+	unsigned int lowerbound = 0;
+	unsigned int upperbound = 1;
+
+	fprintf(stdout, header_format, ">=(KB) .. <(KB) ", "count",
+			"ratio", "distribution");
+	for (int i = 0; i < len; i++) {
+		memset(col1, 0, sizeof(col1));
+		memset(col4, 0, sizeof(col4));
+		if (i == len - 1)
+			sprintf(col1, "%6d ..", lowerbound);
+		else if (i <= 6)
+			sprintf(col1, "%6d .. %-6d", lowerbound, upperbound);
+		else
+
+			sprintf(col1, "%6d .. %-6d", lowerbound, upperbound);
+		col2 = file_counts[i];
+		col3 = (double)(100 * col2) / (double)stats.file_category_stat[EROFS_FT_REG_FILE];
+		memset(col4, '#', col3 / 2);
+		dumpfs_print_chart_row(col1, col2, col3, col4);
+		lowerbound = upperbound;
+		upperbound <<= 1;
+	}
+}
+
+static void dumpfs_print_chart_of_file_type(char **file_types, unsigned int len)
+{
+	char col1[30];
+	unsigned int col2;
+	double col3;
+	char col4[401];
+
+	fprintf(stdout, header_format, "type", "count", "ratio",
+			"distribution");
+	for (int i = 0; i < len; i++) {
+		memset(col1, 0, sizeof(col1));
+		memset(col4, 0, sizeof(col4));
+		sprintf(col1, "%-17s", file_types[i]);
+		col2 = stats.file_type_stat[i];
+		col3 = (double)(100 * col2) / (double)stats.file_category_stat[EROFS_FT_REG_FILE];
+		memset(col4, '#', col3 / 2);
+		dumpfs_print_chart_row(col1, col2, col3, col4);
+	}
+}
+
+static void dumpfs_print_statistic_of_compression(void)
+{
+	stats.compress_rate = (double)(100 * stats.files_total_size) /
+		(double)(stats.files_total_origin_size);
+	fprintf(stdout, "Filesystem compressed files:            %lu\n",
+			stats.compressed_files);
+	fprintf(stdout, "Filesystem uncompressed files:          %lu\n",
+			stats.uncompressed_files);
+	fprintf(stdout, "Filesystem total original file size:    %lu Bytes\n",
+			stats.files_total_origin_size);
+	fprintf(stdout, "Filesystem total file size:             %lu Bytes\n",
+			stats.files_total_size);
+	fprintf(stdout, "Filesystem compress rate:               %.2f%%\n",
+			stats.compress_rate);
+}
+
+static void dumpfs_print_statistic(void)
+{
+	int err;
+
+	stats.blocks = sbi.blocks;
+	err = erofs_read_dir(sbi.root_nid, sbi.root_nid);
+	if (err) {
+		erofs_err("read dir failed");
+		return;
+	}
+
+	dumpfs_print_statistic_of_filetype();
+	dumpfs_print_statistic_of_compression();
+
+	fprintf(stdout, "\nOriginal file size distribution:\n");
+	dumpfs_print_chart_of_file(stats.file_original_size,
+			ARRAY_SIZE(stats.file_original_size));
+	fprintf(stdout, "\nOn-Disk file size distribution:\n");
+	dumpfs_print_chart_of_file(stats.file_comp_size,
+			ARRAY_SIZE(stats.file_comp_size));
+	fprintf(stdout, "\nFile type distribution:\n");
+	dumpfs_print_chart_of_file_type(file_types, OTHERFILETYPE);
+}
+
 static void dumpfs_print_superblock(void)
 {
 	time_t time = sbi.build_time;
@@ -156,6 +511,9 @@ int main(int argc, char **argv)
 
 	if (dumpcfg.print_superblock)
 		dumpfs_print_superblock();
+
+	if (dumpcfg.print_statistic)
+		dumpfs_print_statistic();
 out:
 	if (cfg.c_img_path)
 		free(cfg.c_img_path);
-- 
2.25.4

[PATCH v2 4/5] dump.erofs: add -i options to dump file information of specific inode number

[Guo Xuenan]

From: Wang Qi <mpiglet@outlook.com>

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Wang Qi <mpiglet@outlook.com>
---
 dump/main.c | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 159 insertions(+), 2 deletions(-)

diff --git a/dump/main.c b/dump/main.c
index 0778354..233de50 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -21,8 +21,10 @@
 
 struct dumpcfg {
 	bool print_superblock;
+	bool print_inode;
 	bool print_statistic;
 	bool print_version;
+	u64 ino;
 };
 static struct dumpcfg dumpcfg;
 
@@ -109,8 +111,9 @@ static void usage(void)
 	fputs("usage: [options] erofs-image\n\n"
 		"Dump erofs layout from erofs-image, and [options] are:\n"
 		"--help  display this help and exit.\n"
-		"-s          print information about superblock\n"
+		"-s      print information about superblock\n"
 		"-S      print statistic information of the erofs-image\n"
+		"-i #    print target # inode info\n"
 		"-V      print the version number of dump.erofs and exit.\n",
 		stdout);
 }
@@ -123,10 +126,16 @@ static void dumpfs_print_version(void)
 static int dumpfs_parse_options_cfg(int argc, char **argv)
 {
 	int opt;
+	u64 i;
 
-	while ((opt = getopt_long(argc, argv, "sSV",
+	while ((opt = getopt_long(argc, argv, "i:sSV",
 					long_options, NULL)) != -1) {
 		switch (opt) {
+		case 'i':
+			i = atoll(optarg);
+			dumpcfg.print_inode = true;
+			dumpcfg.ino = i;
+			break;
 		case 's':
 			dumpcfg.print_superblock = true;
 			break;
@@ -179,6 +188,151 @@ static int get_file_compressed_size(struct erofs_inode *inode,
 	}
 	return 0;
 }
+static int get_path_by_nid(erofs_nid_t nid, erofs_nid_t parent_nid,
+		erofs_nid_t target, char *path, unsigned int pos)
+{
+	int err;
+	struct erofs_inode inode = {.nid = nid};
+	erofs_off_t offset;
+	char buf[EROFS_BLKSIZ];
+
+	path[pos++] = '/';
+	if (target == sbi.root_nid)
+		return 0;
+
+	err = erofs_read_inode_from_disk(&inode);
+	if (err) {
+		erofs_err("read inode %lu failed", nid);
+		return err;
+	}
+
+	offset = 0;
+	while (offset < inode.i_size) {
+		erofs_off_t maxsize = min_t(erofs_off_t,
+					inode.i_size - offset, EROFS_BLKSIZ);
+		struct erofs_dirent *de = (void *)buf;
+		struct erofs_dirent *end;
+		unsigned int nameoff;
+
+		err = erofs_pread(&inode, buf, maxsize, offset);
+		if (err)
+			return err;
+
+		nameoff = le16_to_cpu(de->nameoff);
+		if (nameoff < sizeof(struct erofs_dirent) ||
+		    nameoff >= PAGE_SIZE) {
+			erofs_err("invalid de[0].nameoff %u @ nid %llu",
+				  nameoff, nid | 0ULL);
+			return -EFSCORRUPTED;
+		}
+
+		end = (void *)buf + nameoff;
+		while (de < end) {
+			const char *dname;
+			unsigned int dname_len;
+
+			nameoff = le16_to_cpu(de->nameoff);
+			dname = (char *)buf + nameoff;
+			if (de + 1 >= end)
+				dname_len = strnlen(dname, maxsize - nameoff);
+			else
+				dname_len = le16_to_cpu(de[1].nameoff)
+					- nameoff;
+
+			/* a corrupted entry is found */
+			if (nameoff + dname_len > maxsize ||
+			    dname_len > EROFS_NAME_LEN) {
+				erofs_err("bogus dirent @ nid %llu",
+						le64_to_cpu(de->nid) | 0ULL);
+				DBG_BUGON(1);
+				return -EFSCORRUPTED;
+			}
+
+			if (de->nid == target) {
+				memcpy(path + pos, dname, dname_len);
+				return 0;
+			}
+
+			if (de->file_type == EROFS_FT_DIR &&
+					de->nid != parent_nid &&
+					de->nid != nid) {
+				memcpy(path + pos, dname, dname_len);
+				err = get_path_by_nid(de->nid, nid,
+						target, path, pos + dname_len);
+				if (!err)
+					return 0;
+				memset(path + pos, 0, dname_len);
+			}
+			++de;
+		}
+		offset += maxsize;
+	}
+	return -1;
+}
+
+static void dumpfs_print_inode(void)
+{
+	int err;
+	erofs_off_t size;
+	u16 access_mode;
+	time_t t;
+	erofs_nid_t nid = dumpcfg.ino;
+	struct erofs_inode inode = {.nid = nid};
+	char path[PATH_MAX + 1] = {0};
+	char access_mode_str[] = "rwxrwxrwx";
+
+	err = erofs_read_inode_from_disk(&inode);
+	if (err) {
+		erofs_err("read inode %lu from disk failed", nid);
+		return;
+	}
+
+	err = get_file_compressed_size(&inode, &size);
+	if (err) {
+		erofs_err("get file size failed\n");
+		return;
+	}
+
+	fprintf(stdout, "Inode %lu info:\n", dumpcfg.ino);
+	err = get_path_by_nid(sbi.root_nid, sbi.root_nid, nid, path, 0);
+
+	fprintf(stdout, "File path:            %s\n",
+			!err ? path : "path not found");
+	fprintf(stdout, "File nid:             %lu\n", inode.nid);
+	fprintf(stdout, "File inode core size: %d\n", inode.inode_isize);
+	fprintf(stdout, "File original size:   %lu\n", inode.i_size);
+	fprintf(stdout,	"File on-disk size:    %lu\n", size);
+	fprintf(stdout, "File compress rate:   %.2f%%\n",
+			(double)(100 * size) / (double)(inode.i_size));
+	fprintf(stdout, "File extent size:     %u\n", inode.extent_isize);
+	fprintf(stdout,	"File xattr size:      %u\n", inode.xattr_isize);
+	fprintf(stdout, "File type:            ");
+	switch (inode.i_mode & S_IFMT) {
+	case S_IFBLK:  fprintf(stdout, "block device\n");     break;
+	case S_IFCHR:  fprintf(stdout, "character device\n"); break;
+	case S_IFDIR:  fprintf(stdout, "directory\n");        break;
+	case S_IFIFO:  fprintf(stdout, "FIFO/pipe\n");        break;
+	case S_IFLNK:  fprintf(stdout, "symlink\n");          break;
+	case S_IFREG:  fprintf(stdout, "regular file\n");     break;
+	case S_IFSOCK: fprintf(stdout, "socket\n");           break;
+	default:       fprintf(stdout, "unknown?\n");         break;
+	}
+
+	access_mode = inode.i_mode & 0777;
+	t = inode.i_ctime;
+	for (int i = 8; i >= 0; i--)
+		if (((access_mode >> i) & 1) == 0)
+			access_mode_str[8 - i] = '-';
+	fprintf(stdout, "File access:          %04o/%s\n",
+			access_mode, access_mode_str);
+	fprintf(stdout, "File uid:             %u\n", inode.i_uid);
+	fprintf(stdout, "File gid:             %u\n", inode.i_gid);
+	fprintf(stdout, "File datalayout:      %d\n", inode.datalayout);
+	fprintf(stdout,	"File nlink:           %u\n", inode.i_nlink);
+	fprintf(stdout, "File create time:     %s", ctime(&t));
+	fprintf(stdout, "File access time:     %s", ctime(&t));
+	fprintf(stdout, "File modify time:     %s", ctime(&t));
+}
 
 static int get_file_type(const char *filename)
 {
@@ -514,6 +668,9 @@ int main(int argc, char **argv)
 
 	if (dumpcfg.print_statistic)
 		dumpfs_print_statistic();
+
+	if (dumpcfg.print_inode)
+		dumpfs_print_inode();
 out:
 	if (cfg.c_img_path)
 		free(cfg.c_img_path);
-- 
2.25.4

[PATCH v2 5/5] dump.erofs: add -I options to dump the layout of a particular inode on disk

[Guo Xuenan]

From: Wang Qi <mpiglet@outlook.com>

Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Wang Qi <mpiglet@outlook.com>
---
 dump/main.c  | 90 +++++++++++++++++++++++++++++++++++++++++++++++-----
 lib/config.c |  3 ++
 2 files changed, 85 insertions(+), 8 deletions(-)

diff --git a/dump/main.c b/dump/main.c
index 233de50..5f20e84 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -23,8 +23,10 @@ struct dumpcfg {
 	bool print_superblock;
 	bool print_inode;
 	bool print_statistic;
+	bool print_inode_phy;
 	bool print_version;
 	u64 ino;
+	u64 ino_phy;
 };
 static struct dumpcfg dumpcfg;
 
@@ -111,9 +113,10 @@ static void usage(void)
 	fputs("usage: [options] erofs-image\n\n"
 		"Dump erofs layout from erofs-image, and [options] are:\n"
 		"--help  display this help and exit.\n"
+		"-i #    print target # inode info\n"
+		"-I #    print target # inode on-disk info\n"
 		"-s      print information about superblock\n"
 		"-S      print statistic information of the erofs-image\n"
-		"-i #    print target # inode info\n"
 		"-V      print the version number of dump.erofs and exit.\n",
 		stdout);
 }
@@ -128,7 +131,7 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
 	int opt;
 	u64 i;
 
-	while ((opt = getopt_long(argc, argv, "i:sSV",
+	while ((opt = getopt_long(argc, argv, "i:I:sSV",
 					long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'i':
@@ -136,6 +139,11 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
 			dumpcfg.print_inode = true;
 			dumpcfg.ino = i;
 			break;
+		case 'I':
+			i = atoll(optarg);
+			dumpcfg.print_inode_phy = true;
+			dumpcfg.ino_phy = i;
+			break;
 		case 's':
 			dumpcfg.print_superblock = true;
 			break;
@@ -188,6 +196,7 @@ static int get_file_compressed_size(struct erofs_inode *inode,
 	}
 	return 0;
 }
+
 static int get_path_by_nid(erofs_nid_t nid, erofs_nid_t parent_nid,
 		erofs_nid_t target, char *path, unsigned int pos)
 {
@@ -270,6 +279,69 @@ static int get_path_by_nid(erofs_nid_t nid, erofs_nid_t parent_nid,
 	return -1;
 }
 
+static void dumpfs_print_inode_phy(void)
+{
+	int err;
+	erofs_nid_t nid = dumpcfg.ino_phy;
+	struct erofs_inode inode = {.nid = nid};
+	char path[PATH_MAX + 1] = {0};
+
+	err = erofs_read_inode_from_disk(&inode);
+	if (err < 0) {
+		erofs_err("read inode %lu from disk failed", nid);
+		return;
+	}
+
+	const erofs_off_t ibase = iloc(inode.nid);
+	const erofs_off_t pos = Z_EROFS_VLE_LEGACY_INDEX_ALIGN(
+			ibase + inode.inode_isize + inode.xattr_isize);
+	erofs_blk_t blocks = inode.u.i_blocks;
+	erofs_blk_t start = 0;
+	erofs_blk_t end = 0;
+	struct erofs_map_blocks map = {
+		.index = UINT_MAX,
+		.m_la = 0,
+	};
+
+	fprintf(stdout, "Inode %lu on-disk info:\n", nid);
+	err = get_path_by_nid(sbi.root_nid, sbi.root_nid, nid, path, 0);
+	if (!err)
+		fprintf(stderr, "File Path:			%s\n",
+				path);
+	else
+		erofs_err("path not found");
+
+	switch (inode.datalayout) {
+	case EROFS_INODE_FLAT_INLINE:
+	case EROFS_INODE_FLAT_PLAIN:
+		if (inode.u.i_blkaddr == NULL_ADDR)
+			start = end = erofs_blknr(pos);
+		else {
+			start = inode.u.i_blkaddr;
+			end = start + BLK_ROUND_UP(inode.i_size) - 1;
+		}
+		fprintf(stdout, "File size:			%lu\n",
+				inode.i_size);
+		fprintf(stdout,
+				"Plain Block Address:		%u - %u\n",
+				start, end);
+		break;
+
+	case EROFS_INODE_FLAT_COMPRESSION_LEGACY:
+	case EROFS_INODE_FLAT_COMPRESSION:
+		err = z_erofs_map_blocks_iter(&inode, &map, 0);
+		if (err)
+			erofs_err("get file blocks range failed");
+
+		start = erofs_blknr(map.m_pa);
+		end = start - 1 + blocks;
+		fprintf(stderr,
+				"Compressed Block Address:	%u - %u\n",
+				start, end);
+		break;
+	}
+}
+
 static void dumpfs_print_inode(void)
 {
 	int err;
@@ -648,19 +720,19 @@ int main(int argc, char **argv)
 	if (err) {
 		if (err == -EINVAL)
 			usage();
-		goto out;
+		goto exit;
 	}
 
 	err = dev_open_ro(cfg.c_img_path);
 	if (err) {
 		erofs_err("open image file failed");
-		goto out;
+		goto exit;
 	}
 
 	err = erofs_read_superblock();
 	if (err) {
 		erofs_err("read superblock failed");
-		goto out;
+		goto exit;
 	}
 
 	if (dumpcfg.print_superblock)
@@ -671,8 +743,10 @@ int main(int argc, char **argv)
 
 	if (dumpcfg.print_inode)
 		dumpfs_print_inode();
-out:
-	if (cfg.c_img_path)
-		free(cfg.c_img_path);
+
+	if (dumpcfg.print_inode_phy)
+		dumpfs_print_inode_phy();
+exit:
+	erofs_exit_configure();
 	return err;
 }
diff --git a/lib/config.c b/lib/config.c
index 99fcf49..405fae6 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -7,6 +7,7 @@
  * Created by Li Guifu <bluce.liguifu@huawei.com>
  */
 #include <string.h>
+#include <stdlib.h>
 #include "erofs/print.h"
 #include "erofs/internal.h"
 
@@ -45,6 +46,8 @@ void erofs_exit_configure(void)
 	if (cfg.sehnd)
 		selabel_close(cfg.sehnd);
 #endif
+	if (cfg.c_img_path)
+		free(cfg.c_img_path);
 }
 
 static unsigned int fullpath_prefix;	/* root directory prefix length */
-- 
2.25.4

Re: [PATCH v2 1/5] erofs-utils: introduce dump.erofs

[Gao Xiang]

Hi Xuenan,

some nit below:

On Tue, Sep 14, 2021 at 03:44:20PM +0800, Guo Xuenan wrote:
> From: Wang Qi <mpiglet@outlook.com>
> 
> Add dump-tool for erofs to facilitate users directly
> analyzing the erofs image file.
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> Signed-off-by: Wang Qi <mpiglet@outlook.com>
> ---
>  Makefile.am        |  2 +-
>  configure.ac       |  2 ++
>  dump/Makefile.am   |  9 +++++
>  dump/main.c        | 85 ++++++++++++++++++++++++++++++++++++++++++++++
>  include/erofs/io.h |  3 ++
>  lib/namei.c        |  5 ++-
>  6 files changed, 102 insertions(+), 4 deletions(-)
>  create mode 100644 dump/Makefile.am
>  create mode 100644 dump/main.c
> 
> diff --git a/Makefile.am b/Makefile.am
> index b804aa9..fedf7b5 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -3,7 +3,7 @@
>  
>  ACLOCAL_AMFLAGS = -I m4
>  
> -SUBDIRS = man lib mkfs
> +SUBDIRS = man lib mkfs dump
>  if ENABLE_FUSE
>  SUBDIRS += fuse
>  endif
> diff --git a/configure.ac b/configure.ac
> index f626064..f4fe548 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -280,6 +280,8 @@ AC_CONFIG_FILES([Makefile
>  		 man/Makefile
>  		 lib/Makefile
>  		 mkfs/Makefile
> +		 dump/Makefile
>  		 fuse/Makefile])
> +
>  AC_OUTPUT
>  
> diff --git a/dump/Makefile.am b/dump/Makefile.am
> new file mode 100644
> index 0000000..8e18c0f
> --- /dev/null
> +++ b/dump/Makefile.am
> @@ -0,0 +1,9 @@
> +# SPDX-License-Identifier: GPL-2.0+
> +# Makefile.am
> +
> +AUTOMAKE_OPTIONS = foreign
> +bin_PROGRAMS     = dump.erofs
> +dump_erofs_SOURCES = main.c
> +dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
> +dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}

I don't see the lz4 library is used in this patchset.
How about adding this dependency in the related patch if needed?

> +
> diff --git a/dump/main.c b/dump/main.c
> new file mode 100644
> index 0000000..8f299ca
> --- /dev/null
> +++ b/dump/main.c
> @@ -0,0 +1,85 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * Copyright (C) 2021-2022 HUAWEI, Inc.
> + *             http://www.huawei.com/
> + * Created by Wang Qi <mpiglet@outlook.com>
> + *            Guo Xuenan <guoxuenan@huawei.com>
> + */
> +
> +#include <stdlib.h>
> +#include <getopt.h>
> +#include <sys/sysmacros.h>
> +#include <time.h>
> +#include <lz4.h>

Same here.

> +
> +#include "erofs/print.h"
> +#include "erofs/io.h"
> +
> +static struct option long_options[] = {
> +	{"help", no_argument, 0, 1},
> +	{0, 0, 0, 0},
> +};
> +
> +static void usage(void)
> +{
> +	fputs("usage: [options] erofs-image\n\n"
> +		"Dump erofs layout from erofs-image, and [options] are:\n"

usage: [options] IMAGE
Dump erofs layout from IMAGE, and [options] are:

> +		"--help  display this help and exit.\n"
> +		"-V      print the version number of dump.erofs and exit.\n",
> +		stderr);
> +}
> +static void dumpfs_print_version(void)
> +{
> +	fprintf(stderr, "dump.erofs %s\n", cfg.c_version);
> +}
> +
> +static int dumpfs_parse_options_cfg(int argc, char **argv)
> +{
> +	int opt;
> +
> +	while ((opt = getopt_long(argc, argv, "V",
> +					long_options, NULL)) != -1) {
> +		switch (opt) {
> +		case 'V':
> +			dumpfs_print_version();
> +			exit(0);
> +		case 1:
> +		    usage();
> +		    exit(0);
> +		default: /* '?' */
> +			return -EINVAL;
> +		}
> +	}
> +
> +	if (optind >= argc)
> +		return -EINVAL;
> +
> +	cfg.c_img_path = strdup(argv[optind++]);
> +	if (!cfg.c_img_path)
> +		return -ENOMEM;
> +
> +	if (optind < argc) {
> +		erofs_err("unexpected argument: %s\n", argv[optind]);
> +		return -EINVAL;
> +	}
> +	return 0;
> +}
> +
> +int main(int argc, char **argv)
> +{
> +	int err = 0;
> +
> +	erofs_init_configure();
> +	err = dumpfs_parse_options_cfg(argc, argv);
> +
> +	if (cfg.c_img_path)
> +		free(cfg.c_img_path);

I can see this has been moved into lib/config.c, how about changing this
patch? (call erofs_exit_configure() instead here.)

Thanks,
Gao Xiang

> +
> +	if (err) {
> +		if (err == -EINVAL)
> +			usage();
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> diff --git a/include/erofs/io.h b/include/erofs/io.h
> index 5574245..00e5de8 100644
> --- a/include/erofs/io.h
> +++ b/include/erofs/io.h
> @@ -10,6 +10,7 @@
>  #define __EROFS_IO_H
>  
>  #include <unistd.h>
> +#include <sys/types.h>
>  #include "internal.h"
>  
>  #ifndef O_BINARY
> @@ -25,6 +26,8 @@ int dev_fillzero(u64 offset, size_t len, bool padding);
>  int dev_fsync(void);
>  int dev_resize(erofs_blk_t nblocks);
>  u64 dev_length(void);
> +dev_t erofs_new_decode_dev(u32 dev);
> +int erofs_read_inode_from_disk(struct erofs_inode *vi);
>  
>  static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
>  			    u32 nblocks)
> diff --git a/lib/namei.c b/lib/namei.c
> index 4e06ba4..b45e9d8 100644
> --- a/lib/namei.c
> +++ b/lib/namei.c
> @@ -5,7 +5,6 @@
>   * Created by Li Guifu <blucerlee@gmail.com>
>   */
>  #include <linux/kdev_t.h>
> -#include <sys/types.h>
>  #include <unistd.h>
>  #include <stdio.h>
>  #include <errno.h>
> @@ -15,7 +14,7 @@
>  #include "erofs/print.h"
>  #include "erofs/io.h"
>  
> -static dev_t erofs_new_decode_dev(u32 dev)
> +dev_t erofs_new_decode_dev(u32 dev)
>  {
>  	const unsigned int major = (dev & 0xfff00) >> 8;
>  	const unsigned int minor = (dev & 0xff) | ((dev >> 12) & 0xfff00);
> @@ -23,7 +22,7 @@ static dev_t erofs_new_decode_dev(u32 dev)
>  	return makedev(major, minor);
>  }
>  
> -static int erofs_read_inode_from_disk(struct erofs_inode *vi)
> +int erofs_read_inode_from_disk(struct erofs_inode *vi)
>  {
>  	int ret, ifmt;
>  	char buf[sizeof(struct erofs_inode_extended)];
> -- 
> 2.25.4

Re: [PATCH v2 2/5] dump.erofs: add "-s" option to dump superblock information

[Gao Xiang]

Some nits...

On Tue, Sep 14, 2021 at 03:44:21PM +0800, Guo Xuenan wrote:
> From: Wang Qi <mpiglet@outlook.com>
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> Signed-off-by: Wang Qi <mpiglet@outlook.com>
> ---
>  dump/Makefile.am |  3 +-
>  dump/main.c      | 94 +++++++++++++++++++++++++++++++++++++++++++-----
>  2 files changed, 88 insertions(+), 9 deletions(-)
> 
> diff --git a/dump/Makefile.am b/dump/Makefile.am
> index 8e18c0f..1ba58da 100644
> --- a/dump/Makefile.am
> +++ b/dump/Makefile.am
> @@ -3,7 +3,8 @@
>  
>  AUTOMAKE_OPTIONS = foreign
>  bin_PROGRAMS     = dump.erofs
> +AM_CPPFLAGS = ${libuuid_CFLAGS}
>  dump_erofs_SOURCES = main.c
>  dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
> -dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}
> +dump_erofs_LDADD = ${libuuid_LIBS} $(top_builddir)/lib/liberofs.la ${liblz4_LIBS}

+dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libuuid_LIBS}

>  
> diff --git a/dump/main.c b/dump/main.c
> index 8f299ca..33521bf 100644
> --- a/dump/main.c
> +++ b/dump/main.c
> @@ -6,25 +6,57 @@
>   *            Guo Xuenan <guoxuenan@huawei.com>
>   */
>  
> +#define _GNU_SOURCE
>  #include <stdlib.h>
>  #include <getopt.h>
>  #include <sys/sysmacros.h>
>  #include <time.h>
>  #include <lz4.h>
> -

Should we get rid of this blank line in the patch 1 instead?

>  #include "erofs/print.h"
>  #include "erofs/io.h"
>  
> +#ifdef HAVE_LIBUUID
> +#include <uuid.h>
> +#endif
> +
> +struct dumpcfg {
> +	bool print_superblock;
> +	bool print_version;
> +};
> +static struct dumpcfg dumpcfg;
> +
>  static struct option long_options[] = {
>  	{"help", no_argument, 0, 1},
>  	{0, 0, 0, 0},
>  };
>  
> +#define EROFS_FEATURE_COMPAT	0
> +#define EROFS_FEATURE_INCOMPAT	1

How about getting rid of these?

> +
> +struct feature {
> +	int compat;

	bool compat;

> +	unsigned int mask;
> +	const char *name;
> +};
> +
> +static struct feature feature_lists[] = {
> +	{	EROFS_FEATURE_COMPAT, EROFS_FEATURE_COMPAT_SB_CHKSUM,
> +		"superblock-checksum"	},

sb_csum is enough... (refer to "metadata_csum" shown in dumpe2fs)

> +
> +	{	EROFS_FEATURE_INCOMPAT, EROFS_FEATURE_INCOMPAT_LZ4_0PADDING,
> +		"lz4-0padding"	},

How about showing "0padding" instead? Since I'll use this in the LZMA
patchset as well... (so not lz4-specific...)

> +	{	EROFS_FEATURE_INCOMPAT, EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER,
> +		"big-pcluster"	},

bigpcluster

> +
> +	{	0, 0, 0	},
> +};
> +
>  static void usage(void)
>  {
>  	fputs("usage: [options] erofs-image\n\n"
>  		"Dump erofs layout from erofs-image, and [options] are:\n"
>  		"--help  display this help and exit.\n"
> +		"-s          print information about superblock\n"
>  		"-V      print the version number of dump.erofs and exit.\n",
>  		stderr);
>  }
> @@ -37,9 +69,12 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
>  {
>  	int opt;
>  
> -	while ((opt = getopt_long(argc, argv, "V",
> +	while ((opt = getopt_long(argc, argv, "sV",
>  					long_options, NULL)) != -1) {
>  		switch (opt) {
> +		case 's':
> +			dumpcfg.print_superblock = true;
> +			break;
>  		case 'V':
>  			dumpfs_print_version();
>  			exit(0);
> @@ -65,21 +100,64 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
>  	return 0;
>  }
>  
> +static void dumpfs_print_superblock(void)
> +{
> +	time_t time = sbi.build_time;
> +	unsigned int features[] = {sbi.feature_compat, sbi.feature_incompat};
> +	char uuid_str[37] = "not available";
> +	int i = 0;
> +	int j = 0;
> +
> +	fprintf(stdout, "Filesystem magic number:			0x%04X\n", EROFS_SUPER_MAGIC_V1);
> +	fprintf(stdout, "Filesystem blocks:				%lu\n", sbi.blocks);
> +	fprintf(stdout, "Filesystem inode metadata start block:		%u\n", sbi.meta_blkaddr);
> +	fprintf(stdout, "Filesystem shared xattr metadata start block:	%u\n", sbi.xattr_blkaddr);
> +	fprintf(stdout, "Filesystem root nid:				%ld\n", sbi.root_nid);
> +	fprintf(stdout, "Filesystem valid inode count:			%lu\n", sbi.inos);

"Filesystem inode count" is enough since such "valid" expression makes
people think about what does the invalid inode mean.

Thanks,
Gao Xiang

> +	fprintf(stdout, "Filesystem created:				%s", ctime(&time));
> +	fprintf(stdout, "Filesystem features:				");
> +	for (; i < ARRAY_SIZE(features); i++) {
> +		for (; j < ARRAY_SIZE(feature_lists); j++) {
> +			if (i == feature_lists[j].compat
> +				&& (features[i] & feature_lists[j].mask))
> +				fprintf(stdout, "%s ", feature_lists[j].name);
> +		}
> +	}
> +	fprintf(stdout, "\n");
> +#ifdef HAVE_LIBUUID
> +	uuid_unparse_lower(sbi.uuid, uuid_str);
> +#endif
> +	fprintf(stdout, "Filesystem UUID:				%s\n", uuid_str);
> +}
> +
>  int main(int argc, char **argv)
>  {
>  	int err = 0;
>  
>  	erofs_init_configure();
>  	err = dumpfs_parse_options_cfg(argc, argv);
> -
> -	if (cfg.c_img_path)
> -		free(cfg.c_img_path);
> -
>  	if (err) {
>  		if (err == -EINVAL)
>  			usage();
> -		return -1;
> +		goto out;
>  	}
>  
> -	return 0;
> +	err = dev_open_ro(cfg.c_img_path);
> +	if (err) {
> +		erofs_err("open image file failed");
> +		goto out;
> +	}
> +
> +	err = erofs_read_superblock();
> +	if (err) {
> +		erofs_err("read superblock failed");
> +		goto out;
> +	}
> +
> +	if (dumpcfg.print_superblock)
> +		dumpfs_print_superblock();
> +out:
> +	if (cfg.c_img_path)
> +		free(cfg.c_img_path);
> +	return err;
>  }
> -- 
> 2.25.4

Re: [PATCH v2 3/5] dump.erofs: add -S options for collecting statistics of the whole filesystem

[Gao Xiang]

On Tue, Sep 14, 2021 at 03:44:22PM +0800, Guo Xuenan wrote:
> From: Wang Qi <mpiglet@outlook.com>
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> Signed-off-by: Wang Qi <mpiglet@outlook.com>
> ---
>  dump/main.c | 364 +++++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 361 insertions(+), 3 deletions(-)
> 
> diff --git a/dump/main.c b/dump/main.c
> index 33521bf..0778354 100644
> --- a/dump/main.c
> +++ b/dump/main.c
> @@ -21,10 +21,63 @@
>  
>  struct dumpcfg {
>  	bool print_superblock;
> +	bool print_statistic;
>  	bool print_version;
>  };
>  static struct dumpcfg dumpcfg;
>  
> +static const char chart_format[] = "%-16s	%-11d %8.2f%% |%-50s|\n";
> +static const char header_format[] = "%-16s %11s %16s |%-50s|\n";
> +static char *file_types[] = {
> +	".so", ".png", ".jpg", ".xml", ".html", ".odex",
> +	".vdex", ".apk", ".ttf", ".jar", ".json", ".ogg",
> +	".oat", ".art", ".rc", ".otf", ".txt", "others",
> +};
> +#define OTHERFILETYPE	ARRAY_SIZE(file_types)
> +/* (1 << FILE_MAX_SIZE_BITS)KB */
> +#define	FILE_MAX_SIZE_BITS	16
> +
> +static const char * const file_category_types[] = {
> +	[EROFS_FT_UNKNOWN] = "unknown type",
> +	[EROFS_FT_REG_FILE] = "regular file",
> +	[EROFS_FT_DIR] = "directory",
> +	[EROFS_FT_CHRDEV] = "char dev",
> +	[EROFS_FT_BLKDEV] = "block dev",
> +	[EROFS_FT_FIFO] = "FIFO file",
> +	[EROFS_FT_SOCK] = "SOCK file",
> +	[EROFS_FT_SYMLINK] = "symlink file",
> +};
> +
> +struct statistics {
> +	unsigned long blocks;
> +	unsigned long files;

what's the difference (blocks vs sbi.blocks) and
(files vs sbi.inos)?

If we consider fsck.erofs feature later, how about renaming
files to inos?

> +	unsigned long files_total_size;
> +	unsigned long files_total_origin_size;
> +	double compress_rate;
> +	unsigned long compressed_files;
> +	unsigned long uncompressed_files;
> +
> +	unsigned long regular_files;
> +	unsigned long dir_files;
> +	unsigned long chardev_files;
> +	unsigned long blkdev_files;
> +	unsigned long fifo_files;
> +	unsigned long sock_files;
> +	unsigned long symlink_files;

can we use an array indexed by EROFS_FT_UNKNOWN..EROFS_FT_SYMLINK as
well for these?

> +
> +	/* statistics the number of files based on inode_info->flags */
> +	unsigned long file_category_stat[EROFS_FT_MAX];
> +	/* statistics the number of files based on file name extensions */
> +	unsigned int file_type_stat[OTHERFILETYPE];
> +	/* statistics the number of files based on file orignial size */
> +	unsigned int file_original_size[FILE_MAX_SIZE_BITS + 1];
> +	/* statistics the number of files based on the compressed
> +	 * size of file
> +	 */
> +	unsigned int file_comp_size[FILE_MAX_SIZE_BITS + 1];
> +};
> +static struct statistics stats;
> +
>  static struct option long_options[] = {
>  	{"help", no_argument, 0, 1},
>  	{0, 0, 0, 0},
> @@ -57,24 +110,29 @@ static void usage(void)
>  		"Dump erofs layout from erofs-image, and [options] are:\n"
>  		"--help  display this help and exit.\n"
>  		"-s          print information about superblock\n"
> +		"-S      print statistic information of the erofs-image\n"
>  		"-V      print the version number of dump.erofs and exit.\n",
> -		stderr);
> +		stdout);

no need to modify this....

>  }
> +
>  static void dumpfs_print_version(void)
>  {
> -	fprintf(stderr, "dump.erofs %s\n", cfg.c_version);
> +	fprintf(stdout, "dump.erofs %s\n", cfg.c_version);

no need to modify this....

>  }
>  
>  static int dumpfs_parse_options_cfg(int argc, char **argv)
>  {
>  	int opt;
>  
> -	while ((opt = getopt_long(argc, argv, "sV",
> +	while ((opt = getopt_long(argc, argv, "sSV",
>  					long_options, NULL)) != -1) {
>  		switch (opt) {
>  		case 's':
>  			dumpcfg.print_superblock = true;
>  			break;
> +		case 'S':
> +			dumpcfg.print_statistic = true;
> +			break;
>  		case 'V':
>  			dumpfs_print_version();
>  			exit(0);
> @@ -100,6 +158,303 @@ static int dumpfs_parse_options_cfg(int argc, char **argv)
>  	return 0;
>  }
>  
> +static int get_file_compressed_size(struct erofs_inode *inode,
> +		erofs_off_t *size)
> +{
> +	*size = 0;
> +	switch (inode->datalayout) {
> +	case EROFS_INODE_FLAT_INLINE:
> +	case EROFS_INODE_FLAT_PLAIN:
> +		stats.uncompressed_files++;
> +		*size = inode->i_size;
> +		break;
> +	case EROFS_INODE_FLAT_COMPRESSION_LEGACY:
> +	case EROFS_INODE_FLAT_COMPRESSION:
> +		stats.compressed_files++;
> +		*size = inode->u.i_blocks * EROFS_BLKSIZ;
> +		break;
> +	default:
> +		erofs_err("unknown datalayout");
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +static int get_file_type(const char *filename)
> +{
> +	char *postfix = strrchr(filename, '.');
> +	int type = 0;
> +
> +	if (postfix == NULL)
> +		return OTHERFILETYPE - 1;
> +	while (type < OTHERFILETYPE - 1) {
> +		if (strcmp(postfix, file_types[type]) == 0)
> +			break;
> +		type++;
> +	}
> +	return type;
> +}
> +
> +static void update_file_size_statatics(erofs_off_t occupied_size,
> +		erofs_off_t original_size)
> +{
> +	int occupied_size_mark;
> +	int original_size_mark;
> +
> +	original_size_mark = 0;
> +	occupied_size_mark = 0;
> +	occupied_size >>= 10;
> +	original_size >>= 10;
> +
> +	while (occupied_size || original_size) {
> +		if (occupied_size) {
> +			occupied_size >>= 1;
> +			occupied_size_mark++;
> +		}
> +		if (original_size) {
> +			original_size >>= 1;
> +			original_size_mark++;
> +		}
> +	}
> +
> +	if (original_size_mark >= FILE_MAX_SIZE_BITS)
> +		stats.file_original_size[FILE_MAX_SIZE_BITS]++;
> +	else
> +		stats.file_original_size[original_size_mark]++;
> +
> +	if (occupied_size_mark >= FILE_MAX_SIZE_BITS)
> +		stats.file_comp_size[FILE_MAX_SIZE_BITS]++;
> +	else
> +		stats.file_comp_size[occupied_size_mark]++;
> +}
> +
> +static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid)
> +{
> +	struct erofs_inode vi = { .nid = nid};
> +	int err;
> +	char buf[EROFS_BLKSIZ];
> +	char filename[PATH_MAX + 1];
> +	erofs_off_t offset;
> +
> +	err = erofs_read_inode_from_disk(&vi);
> +	if (err)
> +		return err;
> +
> +	offset = 0;
> +	while (offset < vi.i_size) {
> +		erofs_off_t maxsize = min_t(erofs_off_t,
> +			vi.i_size - offset, EROFS_BLKSIZ);
> +		struct erofs_dirent *de = (void *)buf;
> +		struct erofs_dirent *end;
> +		unsigned int nameoff;
> +
> +		err = erofs_pread(&vi, buf, maxsize, offset);
> +		if (err)
> +			return err;
> +
> +		nameoff = le16_to_cpu(de->nameoff);
> +
> +		if (nameoff < sizeof(struct erofs_dirent) ||
> +		    nameoff >= PAGE_SIZE) {
> +			erofs_err("invalid de[0].nameoff %u @ nid %llu",
> +				  nameoff, nid | 0ULL);
> +			return -EFSCORRUPTED;
> +		}
> +		end = (void *)buf + nameoff;
> +		while (de < end) {
> +			const char *dname;
> +			unsigned int dname_len;
> +			struct erofs_inode inode = { .nid = de->nid };
> +			erofs_off_t occupied_size = 0;
> +
> +			nameoff = le16_to_cpu(de->nameoff);
> +			dname = (char *)buf + nameoff;
> +
> +			if (de + 1 >= end)
> +				dname_len = strnlen(dname, maxsize - nameoff);
> +			else
> +				dname_len =
> +					le16_to_cpu(de[1].nameoff) - nameoff;
> +
> +			/* a corrupted entry is found */
> +			if (nameoff + dname_len > maxsize ||
> +				dname_len > EROFS_NAME_LEN) {
> +				erofs_err("bogus dirent @ nid %llu",
> +						le64_to_cpu(de->nid) | 0ULL);
> +				DBG_BUGON(1);
> +				return -EFSCORRUPTED;
> +			}
> +			if (de->nid != nid && de->nid != parent_nid)
> +				stats.files++;
> +
> +			memset(filename, 0, PATH_MAX + 1);
> +			memcpy(filename, dname, dname_len);
> +			if (de->file_type >= EROFS_FT_MAX) {
> +				erofs_err("invalid file type %llu", de->nid);
> +				continue;
> +			}
> +			if (de->file_type != EROFS_FT_DIR)
> +				stats.file_category_stat[de->file_type]++;
> +			switch (de->file_type) {
> +			case EROFS_FT_REG_FILE:
> +				err = erofs_read_inode_from_disk(&inode);
> +				if (err) {
> +					erofs_err("read file inode from disk failed!");
> +					return err;
> +				}
> +				stats.files_total_origin_size += inode.i_size;
> +				stats.file_type_stat[get_file_type(filename)]++;
> +
> +				err = get_file_compressed_size(&inode,
> +						&occupied_size);
> +				if (err) {
> +					erofs_err("get file size failed\n");
> +					return err;
> +				}
> +				stats.files_total_size += occupied_size;
> +				update_file_size_statatics(occupied_size, inode.i_size);
> +				break;
> +
> +			case EROFS_FT_DIR:
> +				if (de->nid != nid && de->nid != parent_nid) {
> +					stats.uncompressed_files++;
> +					err = erofs_read_dir(de->nid, nid);
> +					if (err) {
> +						fprintf(stdout,
> +								"parse dir nid %llu error occurred\n",
> +								de->nid);
> +						return err;
> +					}
> +					stats.file_category_stat[EROFS_FT_DIR]++;
> +				}
> +				break;
> +			case EROFS_FT_UNKNOWN:
> +			case EROFS_FT_CHRDEV:
> +			case EROFS_FT_BLKDEV:
> +			case EROFS_FT_FIFO:
> +			case EROFS_FT_SOCK:
> +			case EROFS_FT_SYMLINK:
> +				stats.uncompressed_files++;

I'd like to check if the file is/isn't compressed just by inode
datalayout rather than specific file type....

I mean we could check if the file type is correct or not (...much like a
fsck functionality....) but if we just want to get # of (un)compressed
files, we could just check datalayout instead....

Thanks,
Gao Xiang

> +				break;
> +			default:
> +				erofs_err("%d file type not exists", de->file_type);
> +			}
> +			++de;
> +		}
> +		offset += maxsize;
> +	}
> +	return 0;
> +}
> +
> +static void dumpfs_print_statistic_of_filetype(void)
> +{
> +	fprintf(stdout, "Filesystem total file count:		%lu\n",
> +			stats.files);
> +	for (int i = 0; i < EROFS_FT_MAX; i++)
> +		fprintf(stdout, "Filesystem %s count:		%lu\n",
> +			file_category_types[i], stats.file_category_stat[i]);
> +}
> +
> +static void dumpfs_print_chart_row(char *col1, unsigned int col2,
> +		double col3, char *col4)
> +{
> +	char row[500] = {0};
> +
> +	sprintf(row, chart_format, col1, col2, col3, col4);
> +	fprintf(stdout, row);
> +}
> +
> +static void dumpfs_print_chart_of_file(unsigned int *file_counts,
> +		unsigned int len)
> +{
> +	char col1[30];
> +	unsigned int col2;
> +	double col3;
> +	char col4[400];
> +	unsigned int lowerbound = 0;
> +	unsigned int upperbound = 1;
> +
> +	fprintf(stdout, header_format, ">=(KB) .. <(KB) ", "count",
> +			"ratio", "distribution");
> +	for (int i = 0; i < len; i++) {
> +		memset(col1, 0, sizeof(col1));
> +		memset(col4, 0, sizeof(col4));
> +		if (i == len - 1)
> +			sprintf(col1, "%6d ..", lowerbound);
> +		else if (i <= 6)
> +			sprintf(col1, "%6d .. %-6d", lowerbound, upperbound);
> +		else
> +
> +			sprintf(col1, "%6d .. %-6d", lowerbound, upperbound);
> +		col2 = file_counts[i];
> +		col3 = (double)(100 * col2) / (double)stats.file_category_stat[EROFS_FT_REG_FILE];
> +		memset(col4, '#', col3 / 2);
> +		dumpfs_print_chart_row(col1, col2, col3, col4);
> +		lowerbound = upperbound;
> +		upperbound <<= 1;
> +	}
> +}
> +
> +static void dumpfs_print_chart_of_file_type(char **file_types, unsigned int len)
> +{
> +	char col1[30];
> +	unsigned int col2;
> +	double col3;
> +	char col4[401];
> +
> +	fprintf(stdout, header_format, "type", "count", "ratio",
> +			"distribution");
> +	for (int i = 0; i < len; i++) {
> +		memset(col1, 0, sizeof(col1));
> +		memset(col4, 0, sizeof(col4));
> +		sprintf(col1, "%-17s", file_types[i]);
> +		col2 = stats.file_type_stat[i];
> +		col3 = (double)(100 * col2) / (double)stats.file_category_stat[EROFS_FT_REG_FILE];
> +		memset(col4, '#', col3 / 2);
> +		dumpfs_print_chart_row(col1, col2, col3, col4);
> +	}
> +}
> +
> +static void dumpfs_print_statistic_of_compression(void)
> +{
> +	stats.compress_rate = (double)(100 * stats.files_total_size) /
> +		(double)(stats.files_total_origin_size);
> +	fprintf(stdout, "Filesystem compressed files:            %lu\n",
> +			stats.compressed_files);
> +	fprintf(stdout, "Filesystem uncompressed files:          %lu\n",
> +			stats.uncompressed_files);
> +	fprintf(stdout, "Filesystem total original file size:    %lu Bytes\n",
> +			stats.files_total_origin_size);
> +	fprintf(stdout, "Filesystem total file size:             %lu Bytes\n",
> +			stats.files_total_size);
> +	fprintf(stdout, "Filesystem compress rate:               %.2f%%\n",
> +			stats.compress_rate);
> +}
> +
> +static void dumpfs_print_statistic(void)
> +{
> +	int err;
> +
> +	stats.blocks = sbi.blocks;
> +	err = erofs_read_dir(sbi.root_nid, sbi.root_nid);
> +	if (err) {
> +		erofs_err("read dir failed");
> +		return;
> +	}
> +
> +	dumpfs_print_statistic_of_filetype();
> +	dumpfs_print_statistic_of_compression();
> +
> +	fprintf(stdout, "\nOriginal file size distribution:\n");
> +	dumpfs_print_chart_of_file(stats.file_original_size,
> +			ARRAY_SIZE(stats.file_original_size));
> +	fprintf(stdout, "\nOn-Disk file size distribution:\n");
> +	dumpfs_print_chart_of_file(stats.file_comp_size,
> +			ARRAY_SIZE(stats.file_comp_size));
> +	fprintf(stdout, "\nFile type distribution:\n");
> +	dumpfs_print_chart_of_file_type(file_types, OTHERFILETYPE);
> +}
> +
>  static void dumpfs_print_superblock(void)
>  {
>  	time_t time = sbi.build_time;
> @@ -156,6 +511,9 @@ int main(int argc, char **argv)
>  
>  	if (dumpcfg.print_superblock)
>  		dumpfs_print_superblock();
> +
> +	if (dumpcfg.print_statistic)
> +		dumpfs_print_statistic();
>  out:
>  	if (cfg.c_img_path)
>  		free(cfg.c_img_path);
> -- 
> 2.25.4

Re: [PATCH v2 4/5] dump.erofs: add -i options to dump file information of specific inode number

[Gao Xiang]

On Tue, Sep 14, 2021 at 03:44:23PM +0800, Guo Xuenan wrote:
> From: Wang Qi <mpiglet@outlook.com>
> 
> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
> Signed-off-by: Wang Qi <mpiglet@outlook.com>

Just a rough check, it still doesn't handle hardlink, but that is fine
anyway.

I think if the first patches get into shape, we could merge them in
advance.

Also, I'd like to add a feature to get the independent file
distribution in the fiemap-likewise form, which is useful to debug some
specific compressed files (by using the patch Jianan backported before.)


Thanks,
Gao Xiang

> ---
>  dump/main.c | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 159 insertions(+), 2 deletions(-)
> 
> diff --git a/dump/main.c b/dump/main.c
> index 0778354..233de50 100644
> --- a/dump/main.c
> +++ b/dump/main.c
> @@ -21,8 +21,10 @@
>  
>  struct dumpcfg {
>  	bool print_superblock;
> +	bool print_inode;
>  	bool print_statistic;
>  	bool print_version;
> +	u64 ino;
>  };
>  static struct dumpcfg dumpcfg;
>  
> @@ -109,8 +111,9 @@ static void usage(void)
>  	fputs("usage: [options] erofs-image\n\n"
>  		"Dump erofs layout from erofs-image, and [options] are:\n"
>  		"--help  display this help and exit.\n"
> -		"-s          print information about superblock\n"
> +		"-s      print information about superblock\n"
>  		"-S      print statistic information of the erofs-image\n"
> +		"-i #    print target # inode info\n"
>  		"-V      print the version number of dump.erofs and exit.\n",
>  		stdout);
>  }
> @@ -123,10 +126,16 @@ static void dumpfs_print_version(void)
>  static int dumpfs_parse_options_cfg(int argc, char **argv)
>  {
>  	int opt;
> +	u64 i;
>  
> -	while ((opt = getopt_long(argc, argv, "sSV",
> +	while ((opt = getopt_long(argc, argv, "i:sSV",
>  					long_options, NULL)) != -1) {
>  		switch (opt) {
> +		case 'i':
> +			i = atoll(optarg);
> +			dumpcfg.print_inode = true;
> +			dumpcfg.ino = i;
> +			break;
>  		case 's':
>  			dumpcfg.print_superblock = true;
>  			break;
> @@ -179,6 +188,151 @@ static int get_file_compressed_size(struct erofs_inode *inode,
>  	}
>  	return 0;
>  }
> +static int get_path_by_nid(erofs_nid_t nid, erofs_nid_t parent_nid,
> +		erofs_nid_t target, char *path, unsigned int pos)
> +{
> +	int err;
> +	struct erofs_inode inode = {.nid = nid};
> +	erofs_off_t offset;
> +	char buf[EROFS_BLKSIZ];
> +
> +	path[pos++] = '/';
> +	if (target == sbi.root_nid)
> +		return 0;
> +
> +	err = erofs_read_inode_from_disk(&inode);
> +	if (err) {
> +		erofs_err("read inode %lu failed", nid);
> +		return err;
> +	}
> +
> +	offset = 0;
> +	while (offset < inode.i_size) {
> +		erofs_off_t maxsize = min_t(erofs_off_t,
> +					inode.i_size - offset, EROFS_BLKSIZ);
> +		struct erofs_dirent *de = (void *)buf;
> +		struct erofs_dirent *end;
> +		unsigned int nameoff;
> +
> +		err = erofs_pread(&inode, buf, maxsize, offset);
> +		if (err)
> +			return err;
> +
> +		nameoff = le16_to_cpu(de->nameoff);
> +		if (nameoff < sizeof(struct erofs_dirent) ||
> +		    nameoff >= PAGE_SIZE) {
> +			erofs_err("invalid de[0].nameoff %u @ nid %llu",
> +				  nameoff, nid | 0ULL);
> +			return -EFSCORRUPTED;
> +		}
> +
> +		end = (void *)buf + nameoff;
> +		while (de < end) {
> +			const char *dname;
> +			unsigned int dname_len;
> +
> +			nameoff = le16_to_cpu(de->nameoff);
> +			dname = (char *)buf + nameoff;
> +			if (de + 1 >= end)
> +				dname_len = strnlen(dname, maxsize - nameoff);
> +			else
> +				dname_len = le16_to_cpu(de[1].nameoff)
> +					- nameoff;
> +
> +			/* a corrupted entry is found */
> +			if (nameoff + dname_len > maxsize ||
> +			    dname_len > EROFS_NAME_LEN) {
> +				erofs_err("bogus dirent @ nid %llu",
> +						le64_to_cpu(de->nid) | 0ULL);
> +				DBG_BUGON(1);
> +				return -EFSCORRUPTED;
> +			}
> +
> +			if (de->nid == target) {
> +				memcpy(path + pos, dname, dname_len);
> +				return 0;
> +			}
> +
> +			if (de->file_type == EROFS_FT_DIR &&
> +					de->nid != parent_nid &&
> +					de->nid != nid) {
> +				memcpy(path + pos, dname, dname_len);
> +				err = get_path_by_nid(de->nid, nid,
> +						target, path, pos + dname_len);
> +				if (!err)
> +					return 0;
> +				memset(path + pos, 0, dname_len);
> +			}
> +			++de;
> +		}
> +		offset += maxsize;
> +	}
> +	return -1;
> +}
> +
> +static void dumpfs_print_inode(void)
> +{
> +	int err;
> +	erofs_off_t size;
> +	u16 access_mode;
> +	time_t t;
> +	erofs_nid_t nid = dumpcfg.ino;
> +	struct erofs_inode inode = {.nid = nid};
> +	char path[PATH_MAX + 1] = {0};
> +	char access_mode_str[] = "rwxrwxrwx";
> +
> +	err = erofs_read_inode_from_disk(&inode);
> +	if (err) {
> +		erofs_err("read inode %lu from disk failed", nid);
> +		return;
> +	}
> +
> +	err = get_file_compressed_size(&inode, &size);
> +	if (err) {
> +		erofs_err("get file size failed\n");
> +		return;
> +	}
> +
> +	fprintf(stdout, "Inode %lu info:\n", dumpcfg.ino);
> +	err = get_path_by_nid(sbi.root_nid, sbi.root_nid, nid, path, 0);
> +
> +	fprintf(stdout, "File path:            %s\n",
> +			!err ? path : "path not found");
> +	fprintf(stdout, "File nid:             %lu\n", inode.nid);
> +	fprintf(stdout, "File inode core size: %d\n", inode.inode_isize);
> +	fprintf(stdout, "File original size:   %lu\n", inode.i_size);
> +	fprintf(stdout,	"File on-disk size:    %lu\n", size);
> +	fprintf(stdout, "File compress rate:   %.2f%%\n",
> +			(double)(100 * size) / (double)(inode.i_size));
> +	fprintf(stdout, "File extent size:     %u\n", inode.extent_isize);
> +	fprintf(stdout,	"File xattr size:      %u\n", inode.xattr_isize);
> +	fprintf(stdout, "File type:            ");
> +	switch (inode.i_mode & S_IFMT) {
> +	case S_IFBLK:  fprintf(stdout, "block device\n");     break;
> +	case S_IFCHR:  fprintf(stdout, "character device\n"); break;
> +	case S_IFDIR:  fprintf(stdout, "directory\n");        break;
> +	case S_IFIFO:  fprintf(stdout, "FIFO/pipe\n");        break;
> +	case S_IFLNK:  fprintf(stdout, "symlink\n");          break;
> +	case S_IFREG:  fprintf(stdout, "regular file\n");     break;
> +	case S_IFSOCK: fprintf(stdout, "socket\n");           break;
> +	default:       fprintf(stdout, "unknown?\n");         break;
> +	}
> +
> +	access_mode = inode.i_mode & 0777;
> +	t = inode.i_ctime;
> +	for (int i = 8; i >= 0; i--)
> +		if (((access_mode >> i) & 1) == 0)
> +			access_mode_str[8 - i] = '-';
> +	fprintf(stdout, "File access:          %04o/%s\n",
> +			access_mode, access_mode_str);
> +	fprintf(stdout, "File uid:             %u\n", inode.i_uid);
> +	fprintf(stdout, "File gid:             %u\n", inode.i_gid);
> +	fprintf(stdout, "File datalayout:      %d\n", inode.datalayout);
> +	fprintf(stdout,	"File nlink:           %u\n", inode.i_nlink);
> +	fprintf(stdout, "File create time:     %s", ctime(&t));
> +	fprintf(stdout, "File access time:     %s", ctime(&t));
> +	fprintf(stdout, "File modify time:     %s", ctime(&t));
> +}
>  
>  static int get_file_type(const char *filename)
>  {
> @@ -514,6 +668,9 @@ int main(int argc, char **argv)
>  
>  	if (dumpcfg.print_statistic)
>  		dumpfs_print_statistic();
> +
> +	if (dumpcfg.print_inode)
> +		dumpfs_print_inode();
>  out:
>  	if (cfg.c_img_path)
>  		free(cfg.c_img_path);
> -- 
> 2.25.4

Re: [PATCH v2 4/5] dump.erofs: add -i options to dump file information of specific inode number

[Guo Xuenan]

Hi Xiang：

Thanks for your detailed and thoughtful reply, we will modify it 
according to your suggestions.

As you said, we are also considering  handle hard-link issue in the 
follow-up work.

The filemap supporting will be added in patch v3. Previously, we only 
adapted the interface modified by Jianan, without realizing relevant 
functions, we'll study on it.

在 2021/9/14 21:20, Gao Xiang 写道:
> On Tue, Sep 14, 2021 at 03:44:23PM +0800, Guo Xuenan wrote:
>> From: Wang Qi <mpiglet@outlook.com>
>>
>> Signed-off-by: Guo Xuenan <guoxuenan@huawei.com>
>> Signed-off-by: Wang Qi <mpiglet@outlook.com>
> Just a rough check, it still doesn't handle hardlink, but that is fine
> anyway.
>
> I think if the first patches get into shape, we could merge them in
> advance.
>
> Also, I'd like to add a feature to get the independent file
> distribution in the fiemap-likewise form, which is useful to debug some
> specific compressed files (by using the patch Jianan backported before.)
>
>
> Thanks,
> Gao Xiang
>
>> ---
>>   dump/main.c | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++-
>>   1 file changed, 159 insertions(+), 2 deletions(-)
>>
>> diff --git a/dump/main.c b/dump/main.c
>> index 0778354..233de50 100644
>> --- a/dump/main.c
>> +++ b/dump/main.c
>> @@ -21,8 +21,10 @@
>>   
>>   struct dumpcfg {
>>   	bool print_superblock;
>> +	bool print_inode;
>>   	bool print_statistic;
>>   	bool print_version;
>> +	u64 ino;
>>   };
>>   static struct dumpcfg dumpcfg;
>>   
>> @@ -109,8 +111,9 @@ static void usage(void)
>>   	fputs("usage: [options] erofs-image\n\n"
>>   		"Dump erofs layout from erofs-image, and [options] are:\n"
>>   		"--help  display this help and exit.\n"
>> -		"-s          print information about superblock\n"
>> +		"-s      print information about superblock\n"
>>   		"-S      print statistic information of the erofs-image\n"
>> +		"-i #    print target # inode info\n"
>>   		"-V      print the version number of dump.erofs and exit.\n",
>>   		stdout);
>>   }
>> @@ -123,10 +126,16 @@ static void dumpfs_print_version(void)
>>   static int dumpfs_parse_options_cfg(int argc, char **argv)
>>   {
>>   	int opt;
>> +	u64 i;
>>   
>> -	while ((opt = getopt_long(argc, argv, "sSV",
>> +	while ((opt = getopt_long(argc, argv, "i:sSV",
>>   					long_options, NULL)) != -1) {
>>   		switch (opt) {
>> +		case 'i':
>> +			i = atoll(optarg);
>> +			dumpcfg.print_inode = true;
>> +			dumpcfg.ino = i;
>> +			break;
>>   		case 's':
>>   			dumpcfg.print_superblock = true;
>>   			break;
>> @@ -179,6 +188,151 @@ static int get_file_compressed_size(struct erofs_inode *inode,
>>   	}
>>   	return 0;
>>   }
>> +static int get_path_by_nid(erofs_nid_t nid, erofs_nid_t parent_nid,
>> +		erofs_nid_t target, char *path, unsigned int pos)
>> +{
>> +	int err;
>> +	struct erofs_inode inode = {.nid = nid};
>> +	erofs_off_t offset;
>> +	char buf[EROFS_BLKSIZ];
>> +
>> +	path[pos++] = '/';
>> +	if (target == sbi.root_nid)
>> +		return 0;
>> +
>> +	err = erofs_read_inode_from_disk(&inode);
>> +	if (err) {
>> +		erofs_err("read inode %lu failed", nid);
>> +		return err;
>> +	}
>> +
>> +	offset = 0;
>> +	while (offset < inode.i_size) {
>> +		erofs_off_t maxsize = min_t(erofs_off_t,
>> +					inode.i_size - offset, EROFS_BLKSIZ);
>> +		struct erofs_dirent *de = (void *)buf;
>> +		struct erofs_dirent *end;
>> +		unsigned int nameoff;
>> +
>> +		err = erofs_pread(&inode, buf, maxsize, offset);
>> +		if (err)
>> +			return err;
>> +
>> +		nameoff = le16_to_cpu(de->nameoff);
>> +		if (nameoff < sizeof(struct erofs_dirent) ||
>> +		    nameoff >= PAGE_SIZE) {
>> +			erofs_err("invalid de[0].nameoff %u @ nid %llu",
>> +				  nameoff, nid | 0ULL);
>> +			return -EFSCORRUPTED;
>> +		}
>> +
>> +		end = (void *)buf + nameoff;
>> +		while (de < end) {
>> +			const char *dname;
>> +			unsigned int dname_len;
>> +
>> +			nameoff = le16_to_cpu(de->nameoff);
>> +			dname = (char *)buf + nameoff;
>> +			if (de + 1 >= end)
>> +				dname_len = strnlen(dname, maxsize - nameoff);
>> +			else
>> +				dname_len = le16_to_cpu(de[1].nameoff)
>> +					- nameoff;
>> +
>> +			/* a corrupted entry is found */
>> +			if (nameoff + dname_len > maxsize ||
>> +			    dname_len > EROFS_NAME_LEN) {
>> +				erofs_err("bogus dirent @ nid %llu",
>> +						le64_to_cpu(de->nid) | 0ULL);
>> +				DBG_BUGON(1);
>> +				return -EFSCORRUPTED;
>> +			}
>> +
>> +			if (de->nid == target) {
>> +				memcpy(path + pos, dname, dname_len);
>> +				return 0;
>> +			}
>> +
>> +			if (de->file_type == EROFS_FT_DIR &&
>> +					de->nid != parent_nid &&
>> +					de->nid != nid) {
>> +				memcpy(path + pos, dname, dname_len);
>> +				err = get_path_by_nid(de->nid, nid,
>> +						target, path, pos + dname_len);
>> +				if (!err)
>> +					return 0;
>> +				memset(path + pos, 0, dname_len);
>> +			}
>> +			++de;
>> +		}
>> +		offset += maxsize;
>> +	}
>> +	return -1;
>> +}
>> +
>> +static void dumpfs_print_inode(void)
>> +{
>> +	int err;
>> +	erofs_off_t size;
>> +	u16 access_mode;
>> +	time_t t;
>> +	erofs_nid_t nid = dumpcfg.ino;
>> +	struct erofs_inode inode = {.nid = nid};
>> +	char path[PATH_MAX + 1] = {0};
>> +	char access_mode_str[] = "rwxrwxrwx";
>> +
>> +	err = erofs_read_inode_from_disk(&inode);
>> +	if (err) {
>> +		erofs_err("read inode %lu from disk failed", nid);
>> +		return;
>> +	}
>> +
>> +	err = get_file_compressed_size(&inode, &size);
>> +	if (err) {
>> +		erofs_err("get file size failed\n");
>> +		return;
>> +	}
>> +
>> +	fprintf(stdout, "Inode %lu info:\n", dumpcfg.ino);
>> +	err = get_path_by_nid(sbi.root_nid, sbi.root_nid, nid, path, 0);
>> +
>> +	fprintf(stdout, "File path:            %s\n",
>> +			!err ? path : "path not found");
>> +	fprintf(stdout, "File nid:             %lu\n", inode.nid);
>> +	fprintf(stdout, "File inode core size: %d\n", inode.inode_isize);
>> +	fprintf(stdout, "File original size:   %lu\n", inode.i_size);
>> +	fprintf(stdout,	"File on-disk size:    %lu\n", size);
>> +	fprintf(stdout, "File compress rate:   %.2f%%\n",
>> +			(double)(100 * size) / (double)(inode.i_size));
>> +	fprintf(stdout, "File extent size:     %u\n", inode.extent_isize);
>> +	fprintf(stdout,	"File xattr size:      %u\n", inode.xattr_isize);
>> +	fprintf(stdout, "File type:            ");
>> +	switch (inode.i_mode & S_IFMT) {
>> +	case S_IFBLK:  fprintf(stdout, "block device\n");     break;
>> +	case S_IFCHR:  fprintf(stdout, "character device\n"); break;
>> +	case S_IFDIR:  fprintf(stdout, "directory\n");        break;
>> +	case S_IFIFO:  fprintf(stdout, "FIFO/pipe\n");        break;
>> +	case S_IFLNK:  fprintf(stdout, "symlink\n");          break;
>> +	case S_IFREG:  fprintf(stdout, "regular file\n");     break;
>> +	case S_IFSOCK: fprintf(stdout, "socket\n");           break;
>> +	default:       fprintf(stdout, "unknown?\n");         break;
>> +	}
>> +
>> +	access_mode = inode.i_mode & 0777;
>> +	t = inode.i_ctime;
>> +	for (int i = 8; i >= 0; i--)
>> +		if (((access_mode >> i) & 1) == 0)
>> +			access_mode_str[8 - i] = '-';
>> +	fprintf(stdout, "File access:          %04o/%s\n",
>> +			access_mode, access_mode_str);
>> +	fprintf(stdout, "File uid:             %u\n", inode.i_uid);
>> +	fprintf(stdout, "File gid:             %u\n", inode.i_gid);
>> +	fprintf(stdout, "File datalayout:      %d\n", inode.datalayout);
>> +	fprintf(stdout,	"File nlink:           %u\n", inode.i_nlink);
>> +	fprintf(stdout, "File create time:     %s", ctime(&t));
>> +	fprintf(stdout, "File access time:     %s", ctime(&t));
>> +	fprintf(stdout, "File modify time:     %s", ctime(&t));
>> +}
>>   
>>   static int get_file_type(const char *filename)
>>   {
>> @@ -514,6 +668,9 @@ int main(int argc, char **argv)
>>   
>>   	if (dumpcfg.print_statistic)
>>   		dumpfs_print_statistic();
>> +
>> +	if (dumpcfg.print_inode)
>> +		dumpfs_print_inode();
>>   out:
>>   	if (cfg.c_img_path)
>>   		free(cfg.c_img_path);
>> -- 
>> 2.25.4
> .