[PATCH 1/2] erofs-utils: support 64-bit internal buffer cache

[PATCH 1/2] erofs-utils: support 64-bit internal buffer cache

[Gao Xiang via Linux-erofs]

From: Gao Xiang <gaoxiang25@huawei.com>

Previously, the type of off in struct erofs_buffer_head
is unsigned int, it's not enough for large files.

Fix to a 64-bit field in order to support large files.

Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
---
Hi Guifu,
Please take some time to review these patches
since I'd like to release erofs-utils 1.0 recently.

Thanks,
Gao Xiang

 include/erofs/cache.h |  8 ++++----
 include/erofs/defs.h  |  4 ++++
 lib/cache.c           | 15 ++++++++-------
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/include/erofs/cache.h b/include/erofs/cache.h
index 71df811..10a6aac 100644
--- a/include/erofs/cache.h
+++ b/include/erofs/cache.h
@@ -31,7 +31,7 @@ struct erofs_buffer_head {
 	struct list_head list;
 	struct erofs_buffer_block *block;
 
-	unsigned int off;
+	erofs_off_t off;
 	struct erofs_bhops *op;
 
 	void *fsprivate;
@@ -87,13 +87,13 @@ static inline bool erofs_bh_flush_generic_end(struct erofs_buffer_head *bh)
 }
 
 struct erofs_buffer_head *erofs_buffer_init(void);
-int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr);
+int erofs_bh_balloon(struct erofs_buffer_head *bh, erofs_off_t incr);
 
-struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+struct erofs_buffer_head *erofs_balloc(int type, erofs_off_t size,
 				       unsigned int required_ext,
 				       unsigned int inline_ext);
 struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
-					int type, int size);
+					int type, unsigned int size);
 
 erofs_blk_t erofs_mapbh(struct erofs_buffer_block *bb, bool end);
 bool erofs_bflush(struct erofs_buffer_block *bb);
diff --git a/include/erofs/defs.h b/include/erofs/defs.h
index 0d9910c..15db4e3 100644
--- a/include/erofs/defs.h
+++ b/include/erofs/defs.h
@@ -136,6 +136,10 @@ typedef int64_t         s64;
 	type __max2 = (y);			\
 	__max1 > __max2 ? __max1: __max2; })
 
+#define sgn(x) ({		\
+	typeof(x) _x = (x);	\
+(_x > 0) - (_x < 0); })
+
 #define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
 
 #define BIT(nr)             (1UL << (nr))
diff --git a/lib/cache.c b/lib/cache.c
index 4f03cb9..41d2d5d 100644
--- a/lib/cache.c
+++ b/lib/cache.c
@@ -74,14 +74,15 @@ struct erofs_buffer_head *erofs_buffer_init(void)
 /* return occupied bytes in specific buffer block if succeed */
 static int __erofs_battach(struct erofs_buffer_block *bb,
 			   struct erofs_buffer_head *bh,
-			   unsigned int incr,
+			   erofs_off_t incr,
 			   unsigned int alignsize,
 			   unsigned int extrasize,
 			   bool dryrun)
 {
-	const unsigned int alignedoffset = roundup(bb->buffers.off, alignsize);
-	const int oob = alignedoffset + incr + extrasize -
-			roundup(bb->buffers.off + 1, EROFS_BLKSIZ);
+	const erofs_off_t alignedoffset = roundup(bb->buffers.off, alignsize);
+	const int oob = sgn(roundup(bb->buffers.off % EROFS_BLKSIZ,
+				    alignsize) + incr + extrasize -
+			    EROFS_BLKSIZ);
 	bool tailupdate = false;
 	erofs_blk_t blkaddr;
 
@@ -113,7 +114,7 @@ static int __erofs_battach(struct erofs_buffer_block *bb,
 	return (alignedoffset + incr) % EROFS_BLKSIZ;
 }
 
-int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr)
+int erofs_bh_balloon(struct erofs_buffer_head *bh, erofs_off_t incr)
 {
 	struct erofs_buffer_block *const bb = bh->block;
 
@@ -124,7 +125,7 @@ int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr)
 	return __erofs_battach(bb, NULL, incr, 1, 0, false);
 }
 
-struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+struct erofs_buffer_head *erofs_balloc(int type, erofs_off_t size,
 				       unsigned int required_ext,
 				       unsigned int inline_ext)
 {
@@ -214,7 +215,7 @@ found:
 }
 
 struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
-					int type, int size)
+					int type, unsigned int size)
 {
 	struct erofs_buffer_block *const bb = bh->block;
 	struct erofs_buffer_head *nbh;
-- 
2.17.1

[PATCH 2/2] erofs-utils: complete extended inode support

[Gao Xiang via Linux-erofs]

From: Gao Xiang <gaoxiang25@huawei.com>

There is no requirement to use extended inode (v2)
for Android, but it's obviously useful for wider
use cases.

extended inode was partially supported by obsoleted_mkfs,
complete for new erofs-utils as well.

Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
---
 include/erofs/config.h   |   6 ++
 include/erofs/internal.h |   2 +
 lib/compress.c           |   4 +-
 lib/config.c             |   1 +
 lib/inode.c              | 119 +++++++++++++++++++++++++++++++--------
 mkfs/main.c              |  26 +++++++--
 6 files changed, 125 insertions(+), 33 deletions(-)

diff --git a/include/erofs/config.h b/include/erofs/config.h
index 05fe6b2..8b09430 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -11,6 +11,11 @@
 
 #include "defs.h"
 
+enum {
+	FORCE_INODE_COMPACT = 1,
+	FORCE_INODE_EXTENDED,
+};
+
 struct erofs_configure {
 	const char *c_version;
 	int c_dbg_lvl;
@@ -22,6 +27,7 @@ struct erofs_configure {
 	char *c_src_path;
 	char *c_compr_alg_master;
 	int c_compr_level_master;
+	int c_force_inodeversion;
 };
 
 extern struct erofs_configure cfg;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 07a32d2..5384946 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -53,6 +53,8 @@ struct erofs_sb_info {
 	erofs_blk_t xattr_blkaddr;
 
 	u32 feature_incompat;
+	u64 build_time;
+	u32 build_time_nsec;
 };
 
 /* global sbi */
diff --git a/lib/compress.c b/lib/compress.c
index 7935fce..7f65e5e 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -423,8 +423,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 	remaining = inode->i_size;
 
 	while (remaining) {
-		const uint readcount = min_t(uint, remaining,
-					     sizeof(ctx.queue) - ctx.tail);
+		const u64 readcount = min_t(u64, remaining,
+					    sizeof(ctx.queue) - ctx.tail);
 
 		ret = read(fd, ctx.queue + ctx.tail, readcount);
 		if (ret != readcount) {
diff --git a/lib/config.c b/lib/config.c
index 9c78142..110c8b6 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -22,6 +22,7 @@ void erofs_init_configure(void)
 	cfg.c_legacy_compress = false;
 	cfg.c_compr_level_master = -1;
 	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
+	cfg.c_force_inodeversion = 0;
 }
 
 void erofs_show_config(void)
diff --git a/lib/inode.c b/lib/inode.c
index 4e1e29f..d1f294a 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -8,6 +8,7 @@
  * with heavy changes by Gao Xiang <gaoxiang25@huawei.com>
  */
 #define _GNU_SOURCE
+#include <limits.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -365,40 +366,81 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
 {
 	struct erofs_inode *const inode = bh->fsprivate;
 	erofs_off_t off = erofs_btell(bh, false);
-
-	/* let's support compact inode currently */
-	struct erofs_inode_compact dic = {0};
+	union {
+		struct erofs_inode_compact dic;
+		struct erofs_inode_extended die;
+	} u = { {0}, };
 	int ret;
 
-	dic.i_format = cpu_to_le16(0 | (inode->datalayout << 1));
-	dic.i_mode = cpu_to_le16(inode->i_mode);
-	dic.i_nlink = cpu_to_le16(inode->i_nlink);
-	dic.i_size = cpu_to_le32((u32)inode->i_size);
+	switch (inode->inode_isize) {
+	case sizeof(struct erofs_inode_compact):
+		u.dic.i_format = cpu_to_le16(0 | (inode->datalayout << 1));
+		u.dic.i_mode = cpu_to_le16(inode->i_mode);
+		u.dic.i_nlink = cpu_to_le16(inode->i_nlink);
+		u.dic.i_size = cpu_to_le32((u32)inode->i_size);
 
-	dic.i_ino = cpu_to_le32(inode->i_ino[0]);
+		u.dic.i_ino = cpu_to_le32(inode->i_ino[0]);
 
-	dic.i_uid = cpu_to_le16((u16)inode->i_uid);
-	dic.i_gid = cpu_to_le16((u16)inode->i_gid);
+		u.dic.i_uid = cpu_to_le16((u16)inode->i_uid);
+		u.dic.i_gid = cpu_to_le16((u16)inode->i_gid);
 
-	switch ((inode->i_mode) >> S_SHIFT) {
-	case S_IFCHR:
-	case S_IFBLK:
-	case S_IFIFO:
-	case S_IFSOCK:
-		dic.i_u.rdev = cpu_to_le32(inode->u.i_rdev);
+		switch ((inode->i_mode) >> S_SHIFT) {
+		case S_IFCHR:
+		case S_IFBLK:
+		case S_IFIFO:
+		case S_IFSOCK:
+			u.dic.i_u.rdev = cpu_to_le32(inode->u.i_rdev);
+			break;
+
+		default:
+			if (is_inode_layout_compression(inode))
+				u.dic.i_u.compressed_blocks =
+					cpu_to_le32(inode->u.i_blocks);
+			else
+				u.dic.i_u.raw_blkaddr =
+					cpu_to_le32(inode->u.i_blkaddr);
+			break;
+		}
 		break;
+	case sizeof(struct erofs_inode_extended):
+		u.die.i_format = cpu_to_le16(1 | (inode->datalayout << 1));
+		u.die.i_mode = cpu_to_le16(inode->i_mode);
+		u.die.i_nlink = cpu_to_le32(inode->i_nlink);
+		u.die.i_size = cpu_to_le64(inode->i_size);
+
+		u.die.i_ino = cpu_to_le32(inode->i_ino[0]);
+
+		u.die.i_uid = cpu_to_le16(inode->i_uid);
+		u.die.i_gid = cpu_to_le16(inode->i_gid);
+
+		u.die.i_ctime = cpu_to_le64(inode->i_ctime);
+		u.die.i_ctime_nsec = cpu_to_le32(inode->i_ctime_nsec);
+
+		switch ((inode->i_mode) >> S_SHIFT) {
+		case S_IFCHR:
+		case S_IFBLK:
+		case S_IFIFO:
+		case S_IFSOCK:
+			u.die.i_u.rdev = cpu_to_le32(inode->u.i_rdev);
+			break;
 
-	default:
-		if (is_inode_layout_compression(inode))
-			dic.i_u.compressed_blocks =
-				cpu_to_le32(inode->u.i_blocks);
-		else
-			dic.i_u.raw_blkaddr =
-				cpu_to_le32(inode->u.i_blkaddr);
+		default:
+			if (is_inode_layout_compression(inode))
+				u.die.i_u.compressed_blocks =
+					cpu_to_le32(inode->u.i_blocks);
+			else
+				u.die.i_u.raw_blkaddr =
+					cpu_to_le32(inode->u.i_blkaddr);
+			break;
+		}
 		break;
+	default:
+		erofs_err("unsupported on-disk inode version of nid %llu",
+			  (unsigned long long)inode->nid);
+		BUG_ON(1);
 	}
 
-	ret = dev_write(&dic, off, sizeof(struct erofs_inode_compact));
+	ret = dev_write(&u, off, inode->inode_isize);
 	if (ret)
 		return false;
 	off += inode->inode_isize;
@@ -578,6 +620,21 @@ out:
 	return 0;
 }
 
+bool erofs_should_use_inode_extended(struct erofs_inode *inode)
+{
+	if (cfg.c_force_inodeversion == FORCE_INODE_EXTENDED)
+		return true;
+	if (inode->i_size > UINT_MAX)
+		return true;
+	if (inode->i_uid > USHRT_MAX)
+		return true;
+	if (inode->i_gid > USHRT_MAX)
+		return true;
+	if (inode->i_nlink > USHRT_MAX)
+		return true;
+	return false;
+}
+
 static u32 erofs_new_encode_dev(dev_t dev)
 {
 	const unsigned int major = major(dev);
@@ -593,6 +650,8 @@ int erofs_fill_inode(struct erofs_inode *inode,
 	inode->i_mode = st->st_mode;
 	inode->i_uid = st->st_uid;
 	inode->i_gid = st->st_gid;
+	inode->i_ctime = sbi.build_time;
+	inode->i_ctime_nsec = sbi.build_time_nsec;
 	inode->i_nlink = 1;	/* fix up later if needed */
 
 	switch (inode->i_mode & S_IFMT) {
@@ -616,7 +675,17 @@ int erofs_fill_inode(struct erofs_inode *inode,
 	inode->i_srcpath[sizeof(inode->i_srcpath) - 1] = '\0';
 
 	inode->i_ino[1] = st->st_ino;
-	inode->inode_isize = sizeof(struct erofs_inode_compact);
+
+	if (erofs_should_use_inode_extended(inode)) {
+		if (cfg.c_force_inodeversion == FORCE_INODE_COMPACT) {
+			erofs_err("file %s cannot be in compact form",
+				  inode->i_srcpath);
+			return -EINVAL;
+		}
+		inode->inode_isize = sizeof(struct erofs_inode_extended);
+	} else {
+		inode->inode_isize = sizeof(struct erofs_inode_compact);
+	}
 
 	list_add(&inode->i_hash,
 		 &inode_hashtable[st->st_ino % NR_INODE_HASHTABLE]);
diff --git a/mkfs/main.c b/mkfs/main.c
index effc26b..4b279c0 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -72,6 +72,18 @@ static int parse_extended_opts(const char *opts)
 			sbi.feature_incompat &=
 				~EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
 		}
+
+		if (MATCH_EXTENTED_OPT("force-inode-compact", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_force_inodeversion = FORCE_INODE_COMPACT;
+		}
+
+		if (MATCH_EXTENTED_OPT("force-inode-extended", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
+		}
 	}
 	return 0;
 }
@@ -153,6 +165,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 		.magic     = cpu_to_le32(EROFS_SUPER_MAGIC_V1),
 		.blkszbits = LOG_BLOCK_SIZE,
 		.inos   = 0,
+		.build_time = cpu_to_le64(sbi.build_time),
+		.build_time_nsec = cpu_to_le32(sbi.build_time_nsec),
 		.blocks = 0,
 		.meta_blkaddr  = sbi.meta_blkaddr,
 		.xattr_blkaddr = 0,
@@ -161,12 +175,6 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 	const unsigned int sb_blksize =
 		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
 	char *buf;
-	struct timeval t;
-
-	if (!gettimeofday(&t, NULL)) {
-		sb.build_time      = cpu_to_le64(t.tv_sec);
-		sb.build_time_nsec = cpu_to_le32(t.tv_usec);
-	}
 
 	*blocks         = erofs_mapbh(NULL, true);
 	sb.blocks       = cpu_to_le32(*blocks);
@@ -193,6 +201,7 @@ int main(int argc, char **argv)
 	erofs_nid_t root_nid;
 	struct stat64 st;
 	erofs_blk_t nblocks;
+	struct timeval t;
 
 	erofs_init_configure();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
@@ -214,6 +223,11 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
+	if (!gettimeofday(&t, NULL)) {
+		sbi.build_time      = t.tv_sec;
+		sbi.build_time_nsec = t.tv_usec;
+	}
+
 	err = dev_open(cfg.c_img_path);
 	if (err) {
 		usage();
-- 
2.17.1

Re: [PATCH 2/2] erofs-utils: complete extended inode support

[Li Guifu]

> From: Gao Xiang <gaoxiang25@huawei.com>
> 
> There is no requirement to use extended inode (v2)
> for Android, but it's obviously useful for wider
> use cases.
> 
> extended inode was partially supported by obsoleted_mkfs,
> complete for new erofs-utils as well.
> 
> Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>
It looks good
Reviewed-by: Li Guifu <blucerlee@gmail.com>
Tested-by: Li Guifu <blucerlee@gmail.com>

Thanks,

Re: [PATCH 1/2] erofs-utils: support 64-bit internal buffer cache

[Li Guifu]

> From: Gao Xiang <gaoxiang25@huawei.com>
> 
> Previously, the type of off in struct erofs_buffer_head
> is unsigned int, it's not enough for large files.
> 
> Fix to a 64-bit field in order to support large files.
> 
> Signed-off-by: Gao Xiang <gaoxiang25@huawei.com>

It looks good
Reviewed-by: Li Guifu <blucerlee@gmail.com>
Tested-by: Li Guifu <blucerlee@gmail.com>

Thanks,