[PATCH] AOSP: erofs-utils: add block list support

[PATCH] AOSP: erofs-utils: add block list support

[Yue Hu]

From: Yue Hu <huyue2@yulong.com>

Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.

Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
Signed-off-by: Yue Hu <huyue2@yulong.com>
---
 include/erofs/block_list.h | 19 ++++++++++
 include/erofs/config.h     |  1 +
 lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/compress.c             |  8 +++++
 lib/inode.c                | 21 ++++++++++-
 mkfs/main.c                | 17 +++++++++
 6 files changed, 151 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/block_list.h
 create mode 100644 lib/block_list.c

diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index 0000000..cbf1050
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+int block_list_fopen(void);
+void block_list_fclose(void);
+void write_block_list(const char *path, erofs_blk_t blk_start,
+                      erofs_blk_t nblocks, bool has_tail);
+void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
+                               bool inline_data, erofs_blk_t blkaddr);
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a73..67e7a0f 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
 	char *mount_point;
 	char *target_out_path;
 	char *fs_config_file;
+	char *block_list_file;
 #endif
 };
 
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index 0000000..6ebe0f9
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifdef WITH_ANDROID
+#include <stdio.h>
+
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int block_list_fopen(void)
+{
+	if (block_list_fp)
+		return 0;
+
+	block_list_fp = fopen(cfg.block_list_file, "w");
+
+	if (block_list_fp == NULL)
+		return -1;
+
+	return 0;
+}
+
+void block_list_fclose(void)
+{
+	if (block_list_fp) {
+		fclose(block_list_fp);
+		block_list_fp = NULL;
+	}
+}
+
+void write_block_list(const char *path, erofs_blk_t blk_start,
+		      erofs_blk_t nblocks, bool has_tail)
+{
+	const char *fspath = erofs_fspath(path);
+
+	if (!block_list_fp || !cfg.mount_point)
+		return;
+
+	/* only tail-end data */
+	if (!nblocks)
+		return;
+
+	fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+	if (fspath[0] != '/')
+		fprintf(block_list_fp, "/");
+
+	if (nblocks == 1) {
+		fprintf(block_list_fp, "%s %u", fspath, blk_start);
+	} else {
+		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+			blk_start + nblocks - 1);
+	}
+
+	if (!has_tail)
+		fprintf(block_list_fp, "\n");
+}
+
+void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
+			       bool inline_data, erofs_blk_t blkaddr)
+{
+	if (!block_list_fp || !cfg.mount_point)
+		return;
+
+	if (!nblocks && !inline_data) {
+		erofs_dbg("%s : only tail-end non-inline data", path);
+		write_block_list(path, blkaddr, 1, false);
+		return;
+	}
+
+	if (nblocks) {
+		if (!inline_data)
+			fprintf(block_list_fp, " %u", blkaddr);
+
+		fprintf(block_list_fp, "\n");
+	}
+}
+#endif
diff --git a/lib/compress.c b/lib/compress.c
index 2093bfd..5dec0c3 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -19,6 +19,10 @@
 #include "erofs/compress.h"
 #include "compressor.h"
 
+#ifdef WITH_ANDROID
+#include "erofs/block_list.h"
+#endif
+
 static struct erofs_compress compresshandle;
 static int compressionlevel;
 
@@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 		   inode->i_srcpath, (unsigned long long)inode->i_size,
 		   compressed_blocks);
 
+#ifdef WITH_ANDROID
+	write_block_list(inode->i_srcpath, blkaddr, compressed_blocks, false);
+#endif
+
 	/*
 	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
 	 *       when both mkfs & kernel support compression inline.
diff --git a/lib/inode.c b/lib/inode.c
index 787e5b4..6be23cb 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -22,6 +22,10 @@
 #include "erofs/xattr.h"
 #include "erofs/exclude.h"
 
+#ifdef WITH_ANDROID
+#include "erofs/block_list.h"
+#endif
+
 #define S_SHIFT                 12
 static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
 	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
@@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
 			return -EIO;
 		}
 	}
+
+#ifdef WITH_ANDROID
+	if (nblocks)
+		write_block_list(inode->i_srcpath, inode->u.i_blkaddr,
+				 nblocks, inode->idata_size ? true : false);
+#endif
 	return 0;
 }
 
@@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {
 int erofs_write_tail_end(struct erofs_inode *inode)
 {
 	struct erofs_buffer_head *bh, *ibh;
+	erofs_off_t pos;
 
 	bh = inode->bh_data;
 
@@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 		ibh->op = &erofs_write_inline_bhops;
 	} else {
 		int ret;
-		erofs_off_t pos;
 
 		erofs_mapbh(bh->block);
 		pos = erofs_btell(bh, true) - EROFS_BLKSIZ;
@@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 		free(inode->idata);
 		inode->idata = NULL;
 	}
+
+#ifdef WITH_ANDROID
+	if (!S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))
+		write_block_list_tail_end(inode->i_srcpath,
+					  inode->i_size / EROFS_BLKSIZ,
+					  inode->bh_inline ? true: false,
+					  erofs_blknr(pos));
+#endif
+
 out:
 	/* now bh_data can drop directly */
 	if (bh) {
diff --git a/mkfs/main.c b/mkfs/main.c
index e476189..d5a5e07 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -27,6 +27,10 @@
 #include <uuid.h>
 #endif
 
+#ifdef WITH_ANDROID
+#include "erofs/block_list.h"
+#endif
+
 #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
 
 static struct option long_options[] = {
@@ -47,6 +51,7 @@ static struct option long_options[] = {
 	{"mount-point", required_argument, NULL, 10},
 	{"product-out", required_argument, NULL, 11},
 	{"fs-config-file", required_argument, NULL, 12},
+	{"block-list-file", required_argument, NULL, 13},
 #endif
 	{0, 0, 0, 0},
 };
@@ -95,6 +100,7 @@ static void usage(void)
 	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
 	      " --product-out=X       X=product_out directory\n"
 	      " --fs-config-file=X    X=fs_config file\n"
+	      " --block-list-file=X    X=block_list file\n"
 #endif
 	      "\nAvailable compressors are: ", stderr);
 	print_available_compressors(stderr, ", ");
@@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 12:
 			cfg.fs_config_file = optarg;
 			break;
+		case 13:
+			cfg.block_list_file = optarg;
+			break;
 #endif
 		case 'C':
 			i = strtoull(optarg, &endptr, 0);
@@ -541,6 +550,11 @@ int main(int argc, char **argv)
 		erofs_err("failed to load fs config %s", cfg.fs_config_file);
 		return 1;
 	}
+
+	if (cfg.block_list_file && block_list_fopen() < 0) {
+		erofs_err("failed to open %s", cfg.block_list_file);
+		return 1;
+	}
 #endif
 
 	erofs_show_config();
@@ -607,6 +621,9 @@ int main(int argc, char **argv)
 		err = erofs_mkfs_superblock_csum_set();
 exit:
 	z_erofs_compress_exit();
+#ifdef WITH_ANDROID
+	block_list_fclose();
+#endif
 	dev_close();
 	erofs_cleanup_exclude_rules();
 	erofs_exit_configure();
-- 
1.9.1

Re: [PATCH] AOSP: erofs-utils: add block list support

[Gao Xiang]

Hi Yue,

Thanks for your contribution!

On Tue, Jun 22, 2021 at 11:02:32AM +0800, Yue Hu wrote:
> From: Yue Hu <huyue2@yulong.com>
> 
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
> 
> Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d

Change-Id is not strictly necessary here.

I'm not quite familiar with Android block list format (Guifu knows better
than me), may be Jianan and Guifu could review it first. I'll seek time
on this off work.

Hi Jianan,
I heard that you guys have some internal implementation as well, could
you also help review this patch?

Hi Guifu,
Could you also take a look at this against the previous huawei in-house
Android block list implementation and review this as well?

Thanks,
Gao Xiang

> Signed-off-by: Yue Hu <huyue2@yulong.com>
> ---
>  include/erofs/block_list.h | 19 ++++++++++
>  include/erofs/config.h     |  1 +
>  lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
>  lib/compress.c             |  8 +++++
>  lib/inode.c                | 21 ++++++++++-
>  mkfs/main.c                | 17 +++++++++
>  6 files changed, 151 insertions(+), 1 deletion(-)
>  create mode 100644 include/erofs/block_list.h
>  create mode 100644 lib/block_list.c
> 
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 0000000..cbf1050
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,19 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +int block_list_fopen(void);
> +void block_list_fclose(void);
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +                      erofs_blk_t nblocks, bool has_tail);
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +                               bool inline_data, erofs_blk_t blkaddr);
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a73..67e7a0f 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>  	char *mount_point;
>  	char *target_out_path;
>  	char *fs_config_file;
> +	char *block_list_file;
>  #endif
>  };
>  
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 0000000..6ebe0f9
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int block_list_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (block_list_fp == NULL)
> +		return -1;
> +
> +	return 0;
> +}
> +
> +void block_list_fclose(void)
> +{
> +	if (block_list_fp) {
> +		fclose(block_list_fp);
> +		block_list_fp = NULL;
> +	}
> +}
> +
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +		      erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* only tail-end data */
> +	if (!nblocks)
> +		return;
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1) {
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	} else {
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	}
> +
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +			       bool inline_data, erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	if (!nblocks && !inline_data) {
> +		erofs_dbg("%s : only tail-end non-inline data", path);
> +		write_block_list(path, blkaddr, 1, false);
> +		return;
> +	}
> +
> +	if (nblocks) {
> +		if (!inline_data)
> +			fprintf(block_list_fp, " %u", blkaddr);
> +
> +		fprintf(block_list_fp, "\n");
> +	}
> +}
> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd..5dec0c3 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -19,6 +19,10 @@
>  #include "erofs/compress.h"
>  #include "compressor.h"
>  
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>  static struct erofs_compress compresshandle;
>  static int compressionlevel;
>  
> @@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>  		   inode->i_srcpath, (unsigned long long)inode->i_size,
>  		   compressed_blocks);
>  
> +#ifdef WITH_ANDROID
> +	write_block_list(inode->i_srcpath, blkaddr, compressed_blocks, false);
> +#endif
> +
>  	/*
>  	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
>  	 *       when both mkfs & kernel support compression inline.
> diff --git a/lib/inode.c b/lib/inode.c
> index 787e5b4..6be23cb 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -22,6 +22,10 @@
>  #include "erofs/xattr.h"
>  #include "erofs/exclude.h"
>  
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>  #define S_SHIFT                 12
>  static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
>  	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
> @@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>  			return -EIO;
>  		}
>  	}
> +
> +#ifdef WITH_ANDROID
> +	if (nblocks)
> +		write_block_list(inode->i_srcpath, inode->u.i_blkaddr,
> +				 nblocks, inode->idata_size ? true : false);
> +#endif
>  	return 0;
>  }
>  
> @@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {
>  int erofs_write_tail_end(struct erofs_inode *inode)
>  {
>  	struct erofs_buffer_head *bh, *ibh;
> +	erofs_off_t pos;
>  
>  	bh = inode->bh_data;
>  
> @@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>  		ibh->op = &erofs_write_inline_bhops;
>  	} else {
>  		int ret;
> -		erofs_off_t pos;
>  
>  		erofs_mapbh(bh->block);
>  		pos = erofs_btell(bh, true) - EROFS_BLKSIZ;
> @@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>  		free(inode->idata);
>  		inode->idata = NULL;
>  	}
> +
> +#ifdef WITH_ANDROID
> +	if (!S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))
> +		write_block_list_tail_end(inode->i_srcpath,
> +					  inode->i_size / EROFS_BLKSIZ,
> +					  inode->bh_inline ? true: false,
> +					  erofs_blknr(pos));
> +#endif
> +
>  out:
>  	/* now bh_data can drop directly */
>  	if (bh) {
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189..d5a5e07 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -27,6 +27,10 @@
>  #include <uuid.h>
>  #endif
>  
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>  #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>  
>  static struct option long_options[] = {
> @@ -47,6 +51,7 @@ static struct option long_options[] = {
>  	{"mount-point", required_argument, NULL, 10},
>  	{"product-out", required_argument, NULL, 11},
>  	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>  #endif
>  	{0, 0, 0, 0},
>  };
> @@ -95,6 +100,7 @@ static void usage(void)
>  	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>  	      " --product-out=X       X=product_out directory\n"
>  	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>  #endif
>  	      "\nAvailable compressors are: ", stderr);
>  	print_available_compressors(stderr, ", ");
> @@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>  		case 12:
>  			cfg.fs_config_file = optarg;
>  			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>  #endif
>  		case 'C':
>  			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +550,11 @@ int main(int argc, char **argv)
>  		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>  		return 1;
>  	}
> +
> +	if (cfg.block_list_file && block_list_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>  #endif
>  
>  	erofs_show_config();
> @@ -607,6 +621,9 @@ int main(int argc, char **argv)
>  		err = erofs_mkfs_superblock_csum_set();
>  exit:
>  	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	block_list_fclose();
> +#endif
>  	dev_close();
>  	erofs_cleanup_exclude_rules();
>  	erofs_exit_configure();
> -- 
> 1.9.1

Re: [PATCH] AOSP: erofs-utils: add block list support

[Huang Jianan via Linux-erofs]

Hi all,

This patch works well for us.

Tested-by: Huang Jianan <huangjianan@oppo.com>

Thanks,

Jianan

On 2021/6/22 11:02, Yue Hu wrote:
> From: Yue Hu <huyue2@yulong.com>
>
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
>
> Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
> Signed-off-by: Yue Hu <huyue2@yulong.com>
> ---
>   include/erofs/block_list.h | 19 ++++++++++
>   include/erofs/config.h     |  1 +
>   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
>   lib/compress.c             |  8 +++++
>   lib/inode.c                | 21 ++++++++++-
>   mkfs/main.c                | 17 +++++++++
>   6 files changed, 151 insertions(+), 1 deletion(-)
>   create mode 100644 include/erofs/block_list.h
>   create mode 100644 lib/block_list.c
>
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 0000000..cbf1050
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,19 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +int block_list_fopen(void);
> +void block_list_fclose(void);
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +                      erofs_blk_t nblocks, bool has_tail);
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +                               bool inline_data, erofs_blk_t blkaddr);
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a73..67e7a0f 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>   	char *mount_point;
>   	char *target_out_path;
>   	char *fs_config_file;
> +	char *block_list_file;
>   #endif
>   };
>   
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 0000000..6ebe0f9
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int block_list_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (block_list_fp == NULL)
> +		return -1;
> +
> +	return 0;
> +}
> +
> +void block_list_fclose(void)
> +{
> +	if (block_list_fp) {
> +		fclose(block_list_fp);
> +		block_list_fp = NULL;
> +	}
> +}
> +
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +		      erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* only tail-end data */
> +	if (!nblocks)
> +		return;
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1) {
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	} else {
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	}
> +
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +			       bool inline_data, erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	if (!nblocks && !inline_data) {
> +		erofs_dbg("%s : only tail-end non-inline data", path);
> +		write_block_list(path, blkaddr, 1, false);
> +		return;
> +	}
> +
> +	if (nblocks) {
> +		if (!inline_data)
> +			fprintf(block_list_fp, " %u", blkaddr);
> +
> +		fprintf(block_list_fp, "\n");
> +	}
> +}
> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd..5dec0c3 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -19,6 +19,10 @@
>   #include "erofs/compress.h"
>   #include "compressor.h"
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   static struct erofs_compress compresshandle;
>   static int compressionlevel;
>   
> @@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>   		   inode->i_srcpath, (unsigned long long)inode->i_size,
>   		   compressed_blocks);
>   
> +#ifdef WITH_ANDROID
> +	write_block_list(inode->i_srcpath, blkaddr, compressed_blocks, false);
> +#endif
> +
>   	/*
>   	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
>   	 *       when both mkfs & kernel support compression inline.
> diff --git a/lib/inode.c b/lib/inode.c
> index 787e5b4..6be23cb 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -22,6 +22,10 @@
>   #include "erofs/xattr.h"
>   #include "erofs/exclude.h"
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   #define S_SHIFT                 12
>   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
>   	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
> @@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>   			return -EIO;
>   		}
>   	}
> +
> +#ifdef WITH_ANDROID
> +	if (nblocks)
> +		write_block_list(inode->i_srcpath, inode->u.i_blkaddr,
> +				 nblocks, inode->idata_size ? true : false);
> +#endif
>   	return 0;
>   }
>   
> @@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {
>   int erofs_write_tail_end(struct erofs_inode *inode)
>   {
>   	struct erofs_buffer_head *bh, *ibh;
> +	erofs_off_t pos;
>   
>   	bh = inode->bh_data;
>   
> @@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   		ibh->op = &erofs_write_inline_bhops;
>   	} else {
>   		int ret;
> -		erofs_off_t pos;
>   
>   		erofs_mapbh(bh->block);
>   		pos = erofs_btell(bh, true) - EROFS_BLKSIZ;
> @@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   		free(inode->idata);
>   		inode->idata = NULL;
>   	}
> +
> +#ifdef WITH_ANDROID
> +	if (!S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))
> +		write_block_list_tail_end(inode->i_srcpath,
> +					  inode->i_size / EROFS_BLKSIZ,
> +					  inode->bh_inline ? true: false,
> +					  erofs_blknr(pos));
> +#endif
> +
>   out:
>   	/* now bh_data can drop directly */
>   	if (bh) {
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189..d5a5e07 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -27,6 +27,10 @@
>   #include <uuid.h>
>   #endif
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>   
>   static struct option long_options[] = {
> @@ -47,6 +51,7 @@ static struct option long_options[] = {
>   	{"mount-point", required_argument, NULL, 10},
>   	{"product-out", required_argument, NULL, 11},
>   	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>   #endif
>   	{0, 0, 0, 0},
>   };
> @@ -95,6 +100,7 @@ static void usage(void)
>   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>   	      " --product-out=X       X=product_out directory\n"
>   	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>   #endif
>   	      "\nAvailable compressors are: ", stderr);
>   	print_available_compressors(stderr, ", ");
> @@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   		case 12:
>   			cfg.fs_config_file = optarg;
>   			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>   #endif
>   		case 'C':
>   			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +550,11 @@ int main(int argc, char **argv)
>   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>   		return 1;
>   	}
> +
> +	if (cfg.block_list_file && block_list_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>   #endif
>   
>   	erofs_show_config();
> @@ -607,6 +621,9 @@ int main(int argc, char **argv)
>   		err = erofs_mkfs_superblock_csum_set();
>   exit:
>   	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	block_list_fclose();
> +#endif
>   	dev_close();
>   	erofs_cleanup_exclude_rules();
>   	erofs_exit_configure();

Re: [PATCH] AOSP: erofs-utils: add block list support

[Gao Xiang]

Hi Jianan,

On Wed, Jun 23, 2021 at 06:32:37PM +0800, Huang Jianan wrote:
> Hi all,
> 
> This patch works well for us.
> 
> Tested-by: Huang Jianan <huangjianan@oppo.com>

Thanks for confirmation.

> 
> Thanks,
> 
> Jianan
> 

Hi Yue,

> On 2021/6/22 11:02, Yue Hu wrote:
> > From: Yue Hu <huyue2@yulong.com>
> > 
> > Android update engine will treat EROFS filesystem image as one single
> > file. Let's add block list support to optimize OTA size.
> > 
> > Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
> > Signed-off-by: Yue Hu <huyue2@yulong.com>
> > ---
> >   include/erofs/block_list.h | 19 ++++++++++
> >   include/erofs/config.h     |  1 +
> >   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
> >   lib/compress.c             |  8 +++++
> >   lib/inode.c                | 21 ++++++++++-
> >   mkfs/main.c                | 17 +++++++++
> >   6 files changed, 151 insertions(+), 1 deletion(-)
> >   create mode 100644 include/erofs/block_list.h
> >   create mode 100644 lib/block_list.c
> > 
> > diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> > new file mode 100644
> > index 0000000..cbf1050
> > --- /dev/null
> > +++ b/include/erofs/block_list.h
> > @@ -0,0 +1,19 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/include/erofs/block_list.h
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifndef __EROFS_BLOCK_LIST_H
> > +#define __EROFS_BLOCK_LIST_H
> > +
> > +#include "internal.h"
> > +
> > +int block_list_fopen(void);
> > +void block_list_fclose(void);
> > +void write_block_list(const char *path, erofs_blk_t blk_start,
> > +                      erofs_blk_t nblocks, bool has_tail);
> > +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> > +                               bool inline_data, erofs_blk_t blkaddr);

some nitpick...

How about use "erofs_droid_blocklist_" prefix here?
even though it may not be quite useful, but I'd like to avoid potential
namespace polluation...

> > +#endif
> > diff --git a/include/erofs/config.h b/include/erofs/config.h
> > index d140a73..67e7a0f 100644
> > --- a/include/erofs/config.h
> > +++ b/include/erofs/config.h
> > @@ -65,6 +65,7 @@ struct erofs_configure {
> >   	char *mount_point;
> >   	char *target_out_path;
> >   	char *fs_config_file;
> > +	char *block_list_file;
> >   #endif
> >   };
> > diff --git a/lib/block_list.c b/lib/block_list.c
> > new file mode 100644
> > index 0000000..6ebe0f9
> > --- /dev/null
> > +++ b/lib/block_list.c
> > @@ -0,0 +1,86 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/lib/block_list.c
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifdef WITH_ANDROID
> > +#include <stdio.h>
> > +
> > +#include "erofs/block_list.h"
> > +
> > +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> > +#include "erofs/print.h"
> > +
> > +static FILE *block_list_fp = NULL;
> > +
> > +int block_list_fopen(void)
> > +{
> > +	if (block_list_fp)
> > +		return 0;
> > +
> > +	block_list_fp = fopen(cfg.block_list_file, "w");
> > +
> > +	if (block_list_fp == NULL)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> > +void block_list_fclose(void)
> > +{
> > +	if (block_list_fp) {
> > +		fclose(block_list_fp);
> > +		block_list_fp = NULL;
> > +	}
> > +}
> > +
> > +void write_block_list(const char *path, erofs_blk_t blk_start,
> > +		      erofs_blk_t nblocks, bool has_tail)
> > +{
> > +	const char *fspath = erofs_fspath(path);
> > +
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	/* only tail-end data */
> > +	if (!nblocks)
> > +		return;
> > +
> > +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> > +
> > +	if (fspath[0] != '/')
> > +		fprintf(block_list_fp, "/");
> > +
> > +	if (nblocks == 1) {
> > +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> > +	} else {
> > +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> > +			blk_start + nblocks - 1);
> > +	}
> > +
> > +	if (!has_tail)
> > +		fprintf(block_list_fp, "\n");
> > +}
> > +
> > +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> > +			       bool inline_data, erofs_blk_t blkaddr)

This makes me wondering...

1) how about passing in struct erofs_inode *inode directly instead?
2) how about getting rid of inline_data boolean and using
blkaddr != NULL_ADDR to replace such case, e.g.

> > +{
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	if (!nblocks && !inline_data) {

	if (!nblocks && blkaddr != NULL_ADDR) {

> > +		erofs_dbg("%s : only tail-end non-inline data", path);
> > +		write_block_list(path, blkaddr, 1, false);
> > +		return;
> > +	}
> > +
> > +	if (nblocks) {
> > +		if (!inline_data)
> > +			fprintf(block_list_fp, " %u", blkaddr);
> > +
> > +		fprintf(block_list_fp, "\n");
> > +	}
> > +}
> > +#endif
> > diff --git a/lib/compress.c b/lib/compress.c
> > index 2093bfd..5dec0c3 100644
> > --- a/lib/compress.c
> > +++ b/lib/compress.c
> > @@ -19,6 +19,10 @@
> >   #include "erofs/compress.h"
> >   #include "compressor.h"
> > +#ifdef WITH_ANDROID
> > +#include "erofs/block_list.h"
> > +#endif
> > +
> >   static struct erofs_compress compresshandle;
> >   static int compressionlevel;
> > @@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> >   		   inode->i_srcpath, (unsigned long long)inode->i_size,
> >   		   compressed_blocks);
> > +#ifdef WITH_ANDROID
> > +	write_block_list(inode->i_srcpath, blkaddr, compressed_blocks, false);
> > +#endif
> > +
> >   	/*
> >   	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
> >   	 *       when both mkfs & kernel support compression inline.
> > diff --git a/lib/inode.c b/lib/inode.c
> > index 787e5b4..6be23cb 100644
> > --- a/lib/inode.c
> > +++ b/lib/inode.c
> > @@ -22,6 +22,10 @@
> >   #include "erofs/xattr.h"
> >   #include "erofs/exclude.h"
> > +#ifdef WITH_ANDROID
> > +#include "erofs/block_list.h"
> > +#endif
> > +
> >   #define S_SHIFT                 12
> >   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> >   	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
> > @@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
> >   			return -EIO;
> >   		}
> >   	}
> > +
> > +#ifdef WITH_ANDROID
> > +	if (nblocks)
> > +		write_block_list(inode->i_srcpath, inode->u.i_blkaddr,
> > +				 nblocks, inode->idata_size ? true : false);
> > +#endif
> >   	return 0;
> >   }
> > @@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {
> >   int erofs_write_tail_end(struct erofs_inode *inode)
> >   {
> >   	struct erofs_buffer_head *bh, *ibh;
> > +	erofs_off_t pos;
> >   	bh = inode->bh_data;
> > @@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >   		ibh->op = &erofs_write_inline_bhops;
> >   	} else {
> >   		int ret;
> > -		erofs_off_t pos;
> >   		erofs_mapbh(bh->block);
> >   		pos = erofs_btell(bh, true) - EROFS_BLKSIZ;
> > @@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >   		free(inode->idata);
> >   		inode->idata = NULL;
> >   	}
> > +
> > +#ifdef WITH_ANDROID
> > +	if (!S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))

hmmm... that seems some hacky, but I can leave it as-is for now.
but how about moving it into erofs_droid_blocklist_write_tail_end()?

> > +		write_block_list_tail_end(inode->i_srcpath,
> > +					  inode->i_size / EROFS_BLKSIZ,
> > +					  inode->bh_inline ? true: false,
> > +					  erofs_blknr(pos));

		erofs_droid_blocklist_write_tail_end(inode,
				inode->bh_inline ? NULL_ADDR :
				erofs_blknr(pos));

I will send a rough delta patch later, please check it out...

Thanks,
Gao Xiang

> > +#endif
> > +
> >   out:
> >   	/* now bh_data can drop directly */
> >   	if (bh) {
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index e476189..d5a5e07 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -27,6 +27,10 @@
> >   #include <uuid.h>
> >   #endif
> > +#ifdef WITH_ANDROID
> > +#include "erofs/block_list.h"
> > +#endif
> > +
> >   #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
> >   static struct option long_options[] = {
> > @@ -47,6 +51,7 @@ static struct option long_options[] = {
> >   	{"mount-point", required_argument, NULL, 10},
> >   	{"product-out", required_argument, NULL, 11},
> >   	{"fs-config-file", required_argument, NULL, 12},
> > +	{"block-list-file", required_argument, NULL, 13},
> >   #endif
> >   	{0, 0, 0, 0},
> >   };
> > @@ -95,6 +100,7 @@ static void usage(void)
> >   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
> >   	      " --product-out=X       X=product_out directory\n"
> >   	      " --fs-config-file=X    X=fs_config file\n"
> > +	      " --block-list-file=X    X=block_list file\n"
> >   #endif
> >   	      "\nAvailable compressors are: ", stderr);
> >   	print_available_compressors(stderr, ", ");
> > @@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> >   		case 12:
> >   			cfg.fs_config_file = optarg;
> >   			break;
> > +		case 13:
> > +			cfg.block_list_file = optarg;
> > +			break;
> >   #endif
> >   		case 'C':
> >   			i = strtoull(optarg, &endptr, 0);
> > @@ -541,6 +550,11 @@ int main(int argc, char **argv)
> >   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
> >   		return 1;
> >   	}
> > +
> > +	if (cfg.block_list_file && block_list_fopen() < 0) {
> > +		erofs_err("failed to open %s", cfg.block_list_file);
> > +		return 1;
> > +	}
> >   #endif
> >   	erofs_show_config();
> > @@ -607,6 +621,9 @@ int main(int argc, char **argv)
> >   		err = erofs_mkfs_superblock_csum_set();
> >   exit:
> >   	z_erofs_compress_exit();
> > +#ifdef WITH_ANDROID
> > +	block_list_fclose();
> > +#endif
> >   	dev_close();
> >   	erofs_cleanup_exclude_rules();
> >   	erofs_exit_configure();

[PATCH v2] AOSP: erofs-utils: add block list support

[Gao Xiang]

From: Yue Hu <huyue2@yulong.com>

Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.

Signed-off-by: Yue Hu <huyue2@yulong.com>
Signed-off-by: Gao Xiang <xiang@kernel.org>
---
some cleanups.

Hi all,
please kindly check it again if it works since I dont have such
environment. If it doesn't work, please help fix it..

Thanks,
Gao Xiang

 include/erofs/block_list.h | 27 +++++++++++++++
 include/erofs/config.h     |  1 +
 lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/compress.c             |  2 ++
 lib/inode.c                |  6 ++++
 mkfs/main.c                | 14 ++++++++
 6 files changed, 136 insertions(+)
 create mode 100644 include/erofs/block_list.h
 create mode 100644 lib/block_list.c

diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index 000000000000..7756d8a5784c
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+#ifdef WITH_ANDROID
+int erofs_droid_blocklist_fopen(void);
+void erofs_droid_blocklist_fclose(void);
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks);
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr);
+#else
+static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
+static inline
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr) {}
+#endif
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a735bd49..67e7a0fed24c 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
 	char *mount_point;
 	char *target_out_path;
 	char *fs_config_file;
+	char *block_list_file;
 #endif
 };
 
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index 000000000000..ffe780e91900
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifdef WITH_ANDROID
+#include <stdio.h>
+#include <sys/stat.h>
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int erofs_droid_blocklist_fopen(void)
+{
+	if (block_list_fp)
+		return 0;
+
+	block_list_fp = fopen(cfg.block_list_file, "w");
+
+	if (!block_list_fp)
+		return -1;
+	return 0;
+}
+
+void erofs_droid_blocklist_fclose(void)
+{
+	if (!block_list_fp)
+		return;
+
+	fclose(block_list_fp);
+	block_list_fp = NULL;
+}
+
+static void blocklist_write(const char *path, erofs_blk_t blk_start,
+			    erofs_blk_t nblocks, bool has_tail)
+{
+	const char *fspath = erofs_fspath(path);
+
+	fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+	if (fspath[0] != '/')
+		fprintf(block_list_fp, "/");
+
+	if (nblocks == 1)
+		fprintf(block_list_fp, "%s %u", fspath, blk_start);
+	else if (nblocks > 1)
+		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+			blk_start + nblocks - 1);
+	else
+		fprintf(block_list_fp, "%s", fspath);
+ 
+	if (!has_tail)
+		fprintf(block_list_fp, "\n");
+}
+
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks)
+{
+	if (!block_list_fp || !cfg.mount_point)
+		return;
+
+	blocklist_write(inode->i_srcpath, blk_start, nblocks,
+			!!inode->idata_size);
+}
+
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr)
+{
+	if (!block_list_fp || !cfg.mount_point)
+		return;
+
+	/* XXX: a bit hacky.. may need a better approach */
+	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
+		return;
+
+	if (blkaddr != NULL_ADDR)
+		fprintf(block_list_fp, " %u\n", blkaddr);
+	else
+		fprintf(block_list_fp, "\n");
+}
+#endif
diff --git a/lib/compress.c b/lib/compress.c
index 2093bfd68b71..af0c72037281 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -18,6 +18,7 @@
 #include "erofs/cache.h"
 #include "erofs/compress.h"
 #include "compressor.h"
+#include "erofs/block_list.h"
 
 static struct erofs_compress compresshandle;
 static int compressionlevel;
@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 		DBG_BUGON(ret);
 	}
 	inode->compressmeta = compressmeta;
+	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
 	return 0;
 
 err_bdrop:
diff --git a/lib/inode.c b/lib/inode.c
index 787e5b4485a2..4134f8a0e91b 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -21,6 +21,7 @@
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
 #include "erofs/exclude.h"
+#include "erofs/block_list.h"
 
 #define S_SHIFT                 12
 static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
 			return -EIO;
 		}
 	}
+	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
 	return 0;
 }
 
@@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 
 		ibh->fsprivate = erofs_igrab(inode);
 		ibh->op = &erofs_write_inline_bhops;
+
+		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
 	} else {
 		int ret;
 		erofs_off_t pos;
@@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 		inode->idata_size = 0;
 		free(inode->idata);
 		inode->idata = NULL;
+
+		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
 	}
 out:
 	/* now bh_data can drop directly */
diff --git a/mkfs/main.c b/mkfs/main.c
index e476189f0731..28539da5ea5f 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -22,6 +22,7 @@
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
 #include "erofs/exclude.h"
+#include "erofs/block_list.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid.h>
@@ -47,6 +48,7 @@ static struct option long_options[] = {
 	{"mount-point", required_argument, NULL, 10},
 	{"product-out", required_argument, NULL, 11},
 	{"fs-config-file", required_argument, NULL, 12},
+	{"block-list-file", required_argument, NULL, 13},
 #endif
 	{0, 0, 0, 0},
 };
@@ -95,6 +97,7 @@ static void usage(void)
 	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
 	      " --product-out=X       X=product_out directory\n"
 	      " --fs-config-file=X    X=fs_config file\n"
+	      " --block-list-file=X    X=block_list file\n"
 #endif
 	      "\nAvailable compressors are: ", stderr);
 	print_available_compressors(stderr, ", ");
@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 12:
 			cfg.fs_config_file = optarg;
 			break;
+		case 13:
+			cfg.block_list_file = optarg;
+			break;
 #endif
 		case 'C':
 			i = strtoull(optarg, &endptr, 0);
@@ -541,6 +547,11 @@ int main(int argc, char **argv)
 		erofs_err("failed to load fs config %s", cfg.fs_config_file);
 		return 1;
 	}
+
+	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
+		erofs_err("failed to open %s", cfg.block_list_file);
+		return 1;
+	}
 #endif
 
 	erofs_show_config();
@@ -607,6 +618,9 @@ int main(int argc, char **argv)
 		err = erofs_mkfs_superblock_csum_set();
 exit:
 	z_erofs_compress_exit();
+#ifdef WITH_ANDROID
+	erofs_droid_blocklist_fclose();
+#endif
 	dev_close();
 	erofs_cleanup_exclude_rules();
 	erofs_exit_configure();
-- 
1.8.3.1

Re: [PATCH v2] AOSP: erofs-utils: add block list support

[Yue Hu]

On Thu, 24 Jun 2021 01:31:29 +0800
Gao Xiang <hsiangkao@linux.alibaba.com> wrote:

> From: Yue Hu <huyue2@yulong.com>
> 
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
> 
> Signed-off-by: Yue Hu <huyue2@yulong.com>
> Signed-off-by: Gao Xiang <xiang@kernel.org>
> ---
> some cleanups.
> 
> Hi all,
> please kindly check it again if it works since I dont have such
> environment. If it doesn't work, please help fix it..
> 
> Thanks,
> Gao Xiang
> 
>  include/erofs/block_list.h | 27 +++++++++++++++
>  include/erofs/config.h     |  1 +
>  lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
>  lib/compress.c             |  2 ++
>  lib/inode.c                |  6 ++++
>  mkfs/main.c                | 14 ++++++++
>  6 files changed, 136 insertions(+)
>  create mode 100644 include/erofs/block_list.h
>  create mode 100644 lib/block_list.c
> 
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 000000000000..7756d8a5784c
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,27 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +#ifdef WITH_ANDROID
> +int erofs_droid_blocklist_fopen(void);
> +void erofs_droid_blocklist_fclose(void);
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr);
> +#else
> +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> +static inline
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr) {}
> +#endif
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a735bd49..67e7a0fed24c 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>  	char *mount_point;
>  	char *target_out_path;
>  	char *fs_config_file;
> +	char *block_list_file;
>  #endif
>  };
>  
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 000000000000..ffe780e91900
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +#include <sys/stat.h>
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int erofs_droid_blocklist_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (!block_list_fp)
> +		return -1;
> +	return 0;
> +}
> +
> +void erofs_droid_blocklist_fclose(void)
> +{
> +	if (!block_list_fp)
> +		return;
> +
> +	fclose(block_list_fp);
> +	block_list_fp = NULL;
> +}
> +
> +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> +			    erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1)
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	else if (nblocks > 1)
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	else
> +		fprintf(block_list_fp, "%s", fspath);

Seems the last else branch is not correct, it will only record the fspath without
block map due to only inline tail-end data. It's not intention in my v1 patch.

> + 
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> +			!!inode->idata_size);
> +}
> +
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* XXX: a bit hacky.. may need a better approach */
> +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> +		return;
> +
> +	if (blkaddr != NULL_ADDR)
> +		fprintf(block_list_fp, " %u\n", blkaddr);
> +	else
> +		fprintf(block_list_fp, "\n");
> +}
> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd68b71..af0c72037281 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -18,6 +18,7 @@
>  #include "erofs/cache.h"
>  #include "erofs/compress.h"
>  #include "compressor.h"
> +#include "erofs/block_list.h"
>  
>  static struct erofs_compress compresshandle;
>  static int compressionlevel;
> @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>  		DBG_BUGON(ret);
>  	}
>  	inode->compressmeta = compressmeta;
> +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
>  	return 0;
>  
>  err_bdrop:
> diff --git a/lib/inode.c b/lib/inode.c
> index 787e5b4485a2..4134f8a0e91b 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -21,6 +21,7 @@
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
>  #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>  
>  #define S_SHIFT                 12
>  static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>  			return -EIO;
>  		}
>  	}
> +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
>  	return 0;
>  }
>  
> @@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>  
>  		ibh->fsprivate = erofs_igrab(inode);
>  		ibh->op = &erofs_write_inline_bhops;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
>  	} else {
>  		int ret;
>  		erofs_off_t pos;
> @@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>  		inode->idata_size = 0;
>  		free(inode->idata);
>  		inode->idata = NULL;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
>  	}
>  out:
>  	/* now bh_data can drop directly */
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189f0731..28539da5ea5f 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,7 @@
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
>  #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>  
>  #ifdef HAVE_LIBUUID
>  #include <uuid.h>
> @@ -47,6 +48,7 @@ static struct option long_options[] = {
>  	{"mount-point", required_argument, NULL, 10},
>  	{"product-out", required_argument, NULL, 11},
>  	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>  #endif
>  	{0, 0, 0, 0},
>  };
> @@ -95,6 +97,7 @@ static void usage(void)
>  	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>  	      " --product-out=X       X=product_out directory\n"
>  	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>  #endif
>  	      "\nAvailable compressors are: ", stderr);
>  	print_available_compressors(stderr, ", ");
> @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>  		case 12:
>  			cfg.fs_config_file = optarg;
>  			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>  #endif
>  		case 'C':
>  			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +547,11 @@ int main(int argc, char **argv)
>  		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>  		return 1;
>  	}
> +
> +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>  #endif
>  
>  	erofs_show_config();
> @@ -607,6 +618,9 @@ int main(int argc, char **argv)
>  		err = erofs_mkfs_superblock_csum_set();
>  exit:
>  	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	erofs_droid_blocklist_fclose();
> +#endif
>  	dev_close();
>  	erofs_cleanup_exclude_rules();
>  	erofs_exit_configure();

Re: [PATCH v2] AOSP: erofs-utils: add block list support

[Gao Xiang]

Hi Yue,

On Thu, Jun 24, 2021 at 12:33:12PM +0800, Yue Hu wrote:
> On Thu, 24 Jun 2021 01:31:29 +0800
> Gao Xiang <hsiangkao@linux.alibaba.com> wrote:
> 
> > From: Yue Hu <huyue2@yulong.com>
> > 
> > Android update engine will treat EROFS filesystem image as one single
> > file. Let's add block list support to optimize OTA size.
> > 
> > Signed-off-by: Yue Hu <huyue2@yulong.com>
> > Signed-off-by: Gao Xiang <xiang@kernel.org>
> > ---
> > some cleanups.
> > 
> > Hi all,
> > please kindly check it again if it works since I dont have such
> > environment. If it doesn't work, please help fix it..
> > 
> > Thanks,
> > Gao Xiang
> > 
> >  include/erofs/block_list.h | 27 +++++++++++++++
> >  include/erofs/config.h     |  1 +
> >  lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
> >  lib/compress.c             |  2 ++
> >  lib/inode.c                |  6 ++++
> >  mkfs/main.c                | 14 ++++++++
> >  6 files changed, 136 insertions(+)
> >  create mode 100644 include/erofs/block_list.h
> >  create mode 100644 lib/block_list.c
> > 
> > diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> > new file mode 100644
> > index 000000000000..7756d8a5784c
> > --- /dev/null
> > +++ b/include/erofs/block_list.h
> > @@ -0,0 +1,27 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/include/erofs/block_list.h
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifndef __EROFS_BLOCK_LIST_H
> > +#define __EROFS_BLOCK_LIST_H
> > +
> > +#include "internal.h"
> > +
> > +#ifdef WITH_ANDROID
> > +int erofs_droid_blocklist_fopen(void);
> > +void erofs_droid_blocklist_fclose(void);
> > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr);
> > +#else
> > +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> > +static inline
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr) {}
> > +#endif
> > +#endif
> > diff --git a/include/erofs/config.h b/include/erofs/config.h
> > index d140a735bd49..67e7a0fed24c 100644
> > --- a/include/erofs/config.h
> > +++ b/include/erofs/config.h
> > @@ -65,6 +65,7 @@ struct erofs_configure {
> >  	char *mount_point;
> >  	char *target_out_path;
> >  	char *fs_config_file;
> > +	char *block_list_file;
> >  #endif
> >  };
> >  
> > diff --git a/lib/block_list.c b/lib/block_list.c
> > new file mode 100644
> > index 000000000000..ffe780e91900
> > --- /dev/null
> > +++ b/lib/block_list.c
> > @@ -0,0 +1,86 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/lib/block_list.c
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifdef WITH_ANDROID
> > +#include <stdio.h>
> > +#include <sys/stat.h>
> > +#include "erofs/block_list.h"
> > +
> > +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> > +#include "erofs/print.h"
> > +
> > +static FILE *block_list_fp = NULL;
> > +
> > +int erofs_droid_blocklist_fopen(void)
> > +{
> > +	if (block_list_fp)
> > +		return 0;
> > +
> > +	block_list_fp = fopen(cfg.block_list_file, "w");
> > +
> > +	if (!block_list_fp)
> > +		return -1;
> > +	return 0;
> > +}
> > +
> > +void erofs_droid_blocklist_fclose(void)
> > +{
> > +	if (!block_list_fp)
> > +		return;
> > +
> > +	fclose(block_list_fp);
> > +	block_list_fp = NULL;
> > +}
> > +
> > +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> > +			    erofs_blk_t nblocks, bool has_tail)
> > +{
> > +	const char *fspath = erofs_fspath(path);
> > +
> > +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> > +
> > +	if (fspath[0] != '/')
> > +		fprintf(block_list_fp, "/");
> > +
> > +	if (nblocks == 1)
> > +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> > +	else if (nblocks > 1)
> > +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> > +			blk_start + nblocks - 1);
> > +	else
> > +		fprintf(block_list_fp, "%s", fspath);
> 
> Seems the last else branch is not correct, it will only record the fspath without
> block map due to only inline tail-end data. It's not intention in my v1 patch.

Thanks for pointing out... yeah, that is a bit awkward, my intention was
to simplify it as much as possible, some better idea for this?

Thanks,
Gao Xiang

> 
> > + 
> > +	if (!has_tail)
> > +		fprintf(block_list_fp, "\n");
> > +}
> > +
> > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> > +{
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> > +			!!inode->idata_size);
> > +}
> > +
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr)
> > +{
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	/* XXX: a bit hacky.. may need a better approach */
> > +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> > +		return;
> > +
> > +	if (blkaddr != NULL_ADDR)
> > +		fprintf(block_list_fp, " %u\n", blkaddr);
> > +	else
> > +		fprintf(block_list_fp, "\n");
> > +}
> > +#endif
> > diff --git a/lib/compress.c b/lib/compress.c
> > index 2093bfd68b71..af0c72037281 100644
> > --- a/lib/compress.c
> > +++ b/lib/compress.c
> > @@ -18,6 +18,7 @@
> >  #include "erofs/cache.h"
> >  #include "erofs/compress.h"
> >  #include "compressor.h"
> > +#include "erofs/block_list.h"
> >  
> >  static struct erofs_compress compresshandle;
> >  static int compressionlevel;
> > @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> >  		DBG_BUGON(ret);
> >  	}
> >  	inode->compressmeta = compressmeta;
> > +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
> >  	return 0;
> >  
> >  err_bdrop:
> > diff --git a/lib/inode.c b/lib/inode.c
> > index 787e5b4485a2..4134f8a0e91b 100644
> > --- a/lib/inode.c
> > +++ b/lib/inode.c
> > @@ -21,6 +21,7 @@
> >  #include "erofs/compress.h"
> >  #include "erofs/xattr.h"
> >  #include "erofs/exclude.h"
> > +#include "erofs/block_list.h"
> >  
> >  #define S_SHIFT                 12
> >  static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> > @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
> >  			return -EIO;
> >  		}
> >  	}
> > +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
> >  	return 0;
> >  }
> >  
> > @@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >  
> >  		ibh->fsprivate = erofs_igrab(inode);
> >  		ibh->op = &erofs_write_inline_bhops;
> > +
> > +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
> >  	} else {
> >  		int ret;
> >  		erofs_off_t pos;
> > @@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >  		inode->idata_size = 0;
> >  		free(inode->idata);
> >  		inode->idata = NULL;
> > +
> > +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
> >  	}
> >  out:
> >  	/* now bh_data can drop directly */
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index e476189f0731..28539da5ea5f 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -22,6 +22,7 @@
> >  #include "erofs/compress.h"
> >  #include "erofs/xattr.h"
> >  #include "erofs/exclude.h"
> > +#include "erofs/block_list.h"
> >  
> >  #ifdef HAVE_LIBUUID
> >  #include <uuid.h>
> > @@ -47,6 +48,7 @@ static struct option long_options[] = {
> >  	{"mount-point", required_argument, NULL, 10},
> >  	{"product-out", required_argument, NULL, 11},
> >  	{"fs-config-file", required_argument, NULL, 12},
> > +	{"block-list-file", required_argument, NULL, 13},
> >  #endif
> >  	{0, 0, 0, 0},
> >  };
> > @@ -95,6 +97,7 @@ static void usage(void)
> >  	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
> >  	      " --product-out=X       X=product_out directory\n"
> >  	      " --fs-config-file=X    X=fs_config file\n"
> > +	      " --block-list-file=X    X=block_list file\n"
> >  #endif
> >  	      "\nAvailable compressors are: ", stderr);
> >  	print_available_compressors(stderr, ", ");
> > @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> >  		case 12:
> >  			cfg.fs_config_file = optarg;
> >  			break;
> > +		case 13:
> > +			cfg.block_list_file = optarg;
> > +			break;
> >  #endif
> >  		case 'C':
> >  			i = strtoull(optarg, &endptr, 0);
> > @@ -541,6 +547,11 @@ int main(int argc, char **argv)
> >  		erofs_err("failed to load fs config %s", cfg.fs_config_file);
> >  		return 1;
> >  	}
> > +
> > +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> > +		erofs_err("failed to open %s", cfg.block_list_file);
> > +		return 1;
> > +	}
> >  #endif
> >  
> >  	erofs_show_config();
> > @@ -607,6 +618,9 @@ int main(int argc, char **argv)
> >  		err = erofs_mkfs_superblock_csum_set();
> >  exit:
> >  	z_erofs_compress_exit();
> > +#ifdef WITH_ANDROID
> > +	erofs_droid_blocklist_fclose();
> > +#endif
> >  	dev_close();
> >  	erofs_cleanup_exclude_rules();
> >  	erofs_exit_configure();

Re: [PATCH v2] AOSP: erofs-utils: add block list support

[Li Guifu via Linux-erofs]

Gao Xiang
	There are three condition need to be fixed.
	The whole inline file should not be printed anything even *\n*
	The last inline block should add *\n* to trigger a new line
	The last block and not inline should be print independently
	

On 6/23/21 1:31 PM, Gao Xiang wrote:
> From: Yue Hu <huyue2@yulong.com>
> 
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
> 
> Signed-off-by: Yue Hu <huyue2@yulong.com>
> Signed-off-by: Gao Xiang <xiang@kernel.org>
> ---
> some cleanups.
> 
> Hi all,
> please kindly check it again if it works since I dont have such
> environment. If it doesn't work, please help fix it..
> 
> Thanks,
> Gao Xiang
> 
>   include/erofs/block_list.h | 27 +++++++++++++++
>   include/erofs/config.h     |  1 +
>   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
>   lib/compress.c             |  2 ++
>   lib/inode.c                |  6 ++++
>   mkfs/main.c                | 14 ++++++++
>   6 files changed, 136 insertions(+)
>   create mode 100644 include/erofs/block_list.h
>   create mode 100644 lib/block_list.c
> 
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 000000000000..7756d8a5784c
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,27 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +#ifdef WITH_ANDROID
> +int erofs_droid_blocklist_fopen(void);
> +void erofs_droid_blocklist_fclose(void);
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr);
> +#else
> +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> +static inline
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr) {}
> +#endif
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a735bd49..67e7a0fed24c 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>   	char *mount_point;
>   	char *target_out_path;
>   	char *fs_config_file;
> +	char *block_list_file;
>   #endif
>   };
>   
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 000000000000..ffe780e91900
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +#include <sys/stat.h>
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int erofs_droid_blocklist_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (!block_list_fp)
> +		return -1;
> +	return 0;
> +}
> +
> +void erofs_droid_blocklist_fclose(void)
> +{
> +	if (!block_list_fp)
> +		return;
> +
> +	fclose(block_list_fp);
> +	block_list_fp = NULL;
> +}
> +
> +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> +			    erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1)
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	else if (nblocks > 1)
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	else
> +		fprintf(block_list_fp, "%s", fspath);
> +
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> +			!!inode->idata_size);
> +}
> +
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* XXX: a bit hacky.. may need a better approach */
> +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> +		return;
> +
> +	if (blkaddr != NULL_ADDR)
> +		fprintf(block_list_fp, " %u\n", blkaddr);
> +	else
> +		fprintf(block_list_fp, "\n");
> +}
void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
					  erofs_blk_t blkaddr)
{
	if (!block_list_fp || !cfg.mount_point)
		return;

	/* XXX: a bit hacky.. may need a better approach */
	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
		return;

	if (erofs_blknr(inode->i_size)) { // its block list has been output before
		if (blkaddr == NULL_ADDR)
			fprintf(block_list_fp, "\n");
		else
			fprintf(block_list_fp, " %u\n", blkaddr);
		return;
	}
	if (blkaddr != NULL_ADDR)
		blocklist_write(inode->i_srcpath, blkaddr, 1, false);
}


> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd68b71..af0c72037281 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -18,6 +18,7 @@
>   #include "erofs/cache.h"
>   #include "erofs/compress.h"
>   #include "compressor.h"
> +#include "erofs/block_list.h"
>   
>   static struct erofs_compress compresshandle;
>   static int compressionlevel;
> @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>   		DBG_BUGON(ret);
>   	}
>   	inode->compressmeta = compressmeta;
> +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
>   	return 0;
>   
>   err_bdrop:
> diff --git a/lib/inode.c b/lib/inode.c
> index 787e5b4485a2..4134f8a0e91b 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -21,6 +21,7 @@
>   #include "erofs/compress.h"
>   #include "erofs/xattr.h"
>   #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>   
>   #define S_SHIFT                 12
>   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>   			return -EIO;
>   		}
>   	}
> +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
>   	return 0;
>   }
>   
> @@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   
>   		ibh->fsprivate = erofs_igrab(inode);
>   		ibh->op = &erofs_write_inline_bhops;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
>   	} else {
>   		int ret;
>   		erofs_off_t pos;
> @@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   		inode->idata_size = 0;
>   		free(inode->idata);
>   		inode->idata = NULL;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
>   	}
>   out:
>   	/* now bh_data can drop directly */
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189f0731..28539da5ea5f 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,7 @@
>   #include "erofs/compress.h"
>   #include "erofs/xattr.h"
>   #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>   
>   #ifdef HAVE_LIBUUID
>   #include <uuid.h>
> @@ -47,6 +48,7 @@ static struct option long_options[] = {
>   	{"mount-point", required_argument, NULL, 10},
>   	{"product-out", required_argument, NULL, 11},
>   	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>   #endif
>   	{0, 0, 0, 0},
>   };
> @@ -95,6 +97,7 @@ static void usage(void)
>   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>   	      " --product-out=X       X=product_out directory\n"
>   	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>   #endif
>   	      "\nAvailable compressors are: ", stderr);
>   	print_available_compressors(stderr, ", ");
> @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   		case 12:
>   			cfg.fs_config_file = optarg;
>   			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>   #endif
>   		case 'C':
>   			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +547,11 @@ int main(int argc, char **argv)
>   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>   		return 1;
>   	}
> +
> +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>   #endif
>   
>   	erofs_show_config();
> @@ -607,6 +618,9 @@ int main(int argc, char **argv)
>   		err = erofs_mkfs_superblock_csum_set();
>   exit:
>   	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	erofs_droid_blocklist_fclose();
> +#endif
>   	dev_close();
>   	erofs_cleanup_exclude_rules();
>   	erofs_exit_configure();
> 

Re: [PATCH v2] AOSP: erofs-utils: add block list support

[Gao Xiang]

On Thu, Jun 24, 2021 at 01:23:25AM -0400, Li Guifu wrote:
> Gao Xiang
> 	There are three condition need to be fixed.
> 	The whole inline file should not be printed anything even *\n*

Yes.

> 	The last inline block should add *\n* to trigger a new line

doesn't it?

> 	The last block and not inline should be print independently

Could you give some output example of this?

btw, do you also have some idea or help fix/improve this patch?
I'd like to merge it asap if it gets into shape but I don't have
aosp environment at hand.

> 	
> 
> On 6/23/21 1:31 PM, Gao Xiang wrote:
> > From: Yue Hu <huyue2@yulong.com>
> > 
> > Android update engine will treat EROFS filesystem image as one single
> > file. Let's add block list support to optimize OTA size.
> > 
> > Signed-off-by: Yue Hu <huyue2@yulong.com>
> > Signed-off-by: Gao Xiang <xiang@kernel.org>
> > ---
> > some cleanups.
> > 
> > Hi all,
> > please kindly check it again if it works since I dont have such
> > environment. If it doesn't work, please help fix it..
> > 
> > Thanks,
> > Gao Xiang
> > 
> >   include/erofs/block_list.h | 27 +++++++++++++++
> >   include/erofs/config.h     |  1 +
> >   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
> >   lib/compress.c             |  2 ++
> >   lib/inode.c                |  6 ++++
> >   mkfs/main.c                | 14 ++++++++
> >   6 files changed, 136 insertions(+)
> >   create mode 100644 include/erofs/block_list.h
> >   create mode 100644 lib/block_list.c
> > 
> > diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> > new file mode 100644
> > index 000000000000..7756d8a5784c
> > --- /dev/null
> > +++ b/include/erofs/block_list.h
> > @@ -0,0 +1,27 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/include/erofs/block_list.h
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifndef __EROFS_BLOCK_LIST_H
> > +#define __EROFS_BLOCK_LIST_H
> > +
> > +#include "internal.h"
> > +
> > +#ifdef WITH_ANDROID
> > +int erofs_droid_blocklist_fopen(void);
> > +void erofs_droid_blocklist_fclose(void);
> > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr);
> > +#else
> > +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> > +static inline
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr) {}
> > +#endif
> > +#endif
> > diff --git a/include/erofs/config.h b/include/erofs/config.h
> > index d140a735bd49..67e7a0fed24c 100644
> > --- a/include/erofs/config.h
> > +++ b/include/erofs/config.h
> > @@ -65,6 +65,7 @@ struct erofs_configure {
> >   	char *mount_point;
> >   	char *target_out_path;
> >   	char *fs_config_file;
> > +	char *block_list_file;
> >   #endif
> >   };
> > diff --git a/lib/block_list.c b/lib/block_list.c
> > new file mode 100644
> > index 000000000000..ffe780e91900
> > --- /dev/null
> > +++ b/lib/block_list.c
> > @@ -0,0 +1,86 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/lib/block_list.c
> > + *
> > + * Copyright (C), 2021, Coolpad Group Limited.
> > + * Created by Yue Hu <huyue2@yulong.com>
> > + */
> > +#ifdef WITH_ANDROID
> > +#include <stdio.h>
> > +#include <sys/stat.h>
> > +#include "erofs/block_list.h"
> > +
> > +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> > +#include "erofs/print.h"
> > +
> > +static FILE *block_list_fp = NULL;
> > +
> > +int erofs_droid_blocklist_fopen(void)
> > +{
> > +	if (block_list_fp)
> > +		return 0;
> > +
> > +	block_list_fp = fopen(cfg.block_list_file, "w");
> > +
> > +	if (!block_list_fp)
> > +		return -1;
> > +	return 0;
> > +}
> > +
> > +void erofs_droid_blocklist_fclose(void)
> > +{
> > +	if (!block_list_fp)
> > +		return;
> > +
> > +	fclose(block_list_fp);
> > +	block_list_fp = NULL;
> > +}
> > +
> > +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> > +			    erofs_blk_t nblocks, bool has_tail)
> > +{
> > +	const char *fspath = erofs_fspath(path);
> > +
> > +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> > +
> > +	if (fspath[0] != '/')
> > +		fprintf(block_list_fp, "/");
> > +
> > +	if (nblocks == 1)
> > +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> > +	else if (nblocks > 1)
> > +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> > +			blk_start + nblocks - 1);
> > +	else
> > +		fprintf(block_list_fp, "%s", fspath);
> > +
> > +	if (!has_tail)
> > +		fprintf(block_list_fp, "\n");
> > +}
> > +
> > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> > +{
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> > +			!!inode->idata_size);
> > +}
> > +
> > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > +					  erofs_blk_t blkaddr)
> > +{
> > +	if (!block_list_fp || !cfg.mount_point)
> > +		return;
> > +
> > +	/* XXX: a bit hacky.. may need a better approach */
> > +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> > +		return;
> > +
> > +	if (blkaddr != NULL_ADDR)
> > +		fprintf(block_list_fp, " %u\n", blkaddr);
> > +	else
> > +		fprintf(block_list_fp, "\n");
> > +}
> void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> 					  erofs_blk_t blkaddr)
> {
> 	if (!block_list_fp || !cfg.mount_point)
> 		return;
> 
> 	/* XXX: a bit hacky.. may need a better approach */
> 	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> 		return;
> 
> 	if (erofs_blknr(inode->i_size)) { // its block list has been output before
> 		if (blkaddr == NULL_ADDR)
> 			fprintf(block_list_fp, "\n");
> 		else
> 			fprintf(block_list_fp, " %u\n", blkaddr);
> 		return;
> 	}
> 	if (blkaddr != NULL_ADDR)
> 		blocklist_write(inode->i_srcpath, blkaddr, 1, false);

That still sounds a bit hacky. But if no better idea, we could leave it
as-is... honestly, I don't quite like this.

Thanks,
Gao Xiang

> }
> 
> 
> > +#endif
> > diff --git a/lib/compress.c b/lib/compress.c
> > index 2093bfd68b71..af0c72037281 100644
> > --- a/lib/compress.c
> > +++ b/lib/compress.c
> > @@ -18,6 +18,7 @@
> >   #include "erofs/cache.h"
> >   #include "erofs/compress.h"
> >   #include "compressor.h"
> > +#include "erofs/block_list.h"
> >   static struct erofs_compress compresshandle;
> >   static int compressionlevel;
> > @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> >   		DBG_BUGON(ret);
> >   	}
> >   	inode->compressmeta = compressmeta;
> > +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
> >   	return 0;
> >   err_bdrop:
> > diff --git a/lib/inode.c b/lib/inode.c
> > index 787e5b4485a2..4134f8a0e91b 100644
> > --- a/lib/inode.c
> > +++ b/lib/inode.c
> > @@ -21,6 +21,7 @@
> >   #include "erofs/compress.h"
> >   #include "erofs/xattr.h"
> >   #include "erofs/exclude.h"
> > +#include "erofs/block_list.h"
> >   #define S_SHIFT                 12
> >   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> > @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
> >   			return -EIO;
> >   		}
> >   	}
> > +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
> >   	return 0;
> >   }
> > @@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >   		ibh->fsprivate = erofs_igrab(inode);
> >   		ibh->op = &erofs_write_inline_bhops;
> > +
> > +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
> >   	} else {
> >   		int ret;
> >   		erofs_off_t pos;
> > @@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> >   		inode->idata_size = 0;
> >   		free(inode->idata);
> >   		inode->idata = NULL;
> > +
> > +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
> >   	}
> >   out:
> >   	/* now bh_data can drop directly */
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index e476189f0731..28539da5ea5f 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -22,6 +22,7 @@
> >   #include "erofs/compress.h"
> >   #include "erofs/xattr.h"
> >   #include "erofs/exclude.h"
> > +#include "erofs/block_list.h"
> >   #ifdef HAVE_LIBUUID
> >   #include <uuid.h>
> > @@ -47,6 +48,7 @@ static struct option long_options[] = {
> >   	{"mount-point", required_argument, NULL, 10},
> >   	{"product-out", required_argument, NULL, 11},
> >   	{"fs-config-file", required_argument, NULL, 12},
> > +	{"block-list-file", required_argument, NULL, 13},
> >   #endif
> >   	{0, 0, 0, 0},
> >   };
> > @@ -95,6 +97,7 @@ static void usage(void)
> >   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
> >   	      " --product-out=X       X=product_out directory\n"
> >   	      " --fs-config-file=X    X=fs_config file\n"
> > +	      " --block-list-file=X    X=block_list file\n"
> >   #endif
> >   	      "\nAvailable compressors are: ", stderr);
> >   	print_available_compressors(stderr, ", ");
> > @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> >   		case 12:
> >   			cfg.fs_config_file = optarg;
> >   			break;
> > +		case 13:
> > +			cfg.block_list_file = optarg;
> > +			break;
> >   #endif
> >   		case 'C':
> >   			i = strtoull(optarg, &endptr, 0);
> > @@ -541,6 +547,11 @@ int main(int argc, char **argv)
> >   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
> >   		return 1;
> >   	}
> > +
> > +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> > +		erofs_err("failed to open %s", cfg.block_list_file);
> > +		return 1;
> > +	}
> >   #endif
> >   	erofs_show_config();
> > @@ -607,6 +618,9 @@ int main(int argc, char **argv)
> >   		err = erofs_mkfs_superblock_csum_set();
> >   exit:
> >   	z_erofs_compress_exit();
> > +#ifdef WITH_ANDROID
> > +	erofs_droid_blocklist_fclose();
> > +#endif
> >   	dev_close();
> >   	erofs_cleanup_exclude_rules();
> >   	erofs_exit_configure();
> > 

Re: [PATCH v2] AOSP: erofs-utils: add block list support

[Yue Hu]

On Thu, 24 Jun 2021 13:37:53 +0800
Gao Xiang <hsiangkao@linux.alibaba.com> wrote:

> On Thu, Jun 24, 2021 at 01:23:25AM -0400, Li Guifu wrote:
> > Gao Xiang
> > 	There are three condition need to be fixed.
> > 	The whole inline file should not be printed anything even *\n*  
> 
> Yes.
> 
> > 	The last inline block should add *\n* to trigger a new line  
> 
> doesn't it?
> 
> > 	The last block and not inline should be print independently  
> 
> Could you give some output example of this?
> 
> btw, do you also have some idea or help fix/improve this patch?

Gao Xiang,

I'm improving the patch based on v2 and some suggestions from Guifu.
Of course, i will test it firstly then send it out.

Thanks.

> I'd like to merge it asap if it gets into shape but I don't have
> aosp environment at hand.
> 
> > 	
> > 
> > On 6/23/21 1:31 PM, Gao Xiang wrote:  
> > > From: Yue Hu <huyue2@yulong.com>
> > > 
> > > Android update engine will treat EROFS filesystem image as one single
> > > file. Let's add block list support to optimize OTA size.
> > > 
> > > Signed-off-by: Yue Hu <huyue2@yulong.com>
> > > Signed-off-by: Gao Xiang <xiang@kernel.org>
> > > ---
> > > some cleanups.
> > > 
> > > Hi all,
> > > please kindly check it again if it works since I dont have such
> > > environment. If it doesn't work, please help fix it..
> > > 
> > > Thanks,
> > > Gao Xiang
> > > 
> > >   include/erofs/block_list.h | 27 +++++++++++++++
> > >   include/erofs/config.h     |  1 +
> > >   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
> > >   lib/compress.c             |  2 ++
> > >   lib/inode.c                |  6 ++++
> > >   mkfs/main.c                | 14 ++++++++
> > >   6 files changed, 136 insertions(+)
> > >   create mode 100644 include/erofs/block_list.h
> > >   create mode 100644 lib/block_list.c
> > > 
> > > diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> > > new file mode 100644
> > > index 000000000000..7756d8a5784c
> > > --- /dev/null
> > > +++ b/include/erofs/block_list.h
> > > @@ -0,0 +1,27 @@
> > > +// SPDX-License-Identifier: GPL-2.0+
> > > +/*
> > > + * erofs-utils/include/erofs/block_list.h
> > > + *
> > > + * Copyright (C), 2021, Coolpad Group Limited.
> > > + * Created by Yue Hu <huyue2@yulong.com>
> > > + */
> > > +#ifndef __EROFS_BLOCK_LIST_H
> > > +#define __EROFS_BLOCK_LIST_H
> > > +
> > > +#include "internal.h"
> > > +
> > > +#ifdef WITH_ANDROID
> > > +int erofs_droid_blocklist_fopen(void);
> > > +void erofs_droid_blocklist_fclose(void);
> > > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > > +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> > > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > > +					  erofs_blk_t blkaddr);
> > > +#else
> > > +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > > +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> > > +static inline
> > > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > > +					  erofs_blk_t blkaddr) {}
> > > +#endif
> > > +#endif
> > > diff --git a/include/erofs/config.h b/include/erofs/config.h
> > > index d140a735bd49..67e7a0fed24c 100644
> > > --- a/include/erofs/config.h
> > > +++ b/include/erofs/config.h
> > > @@ -65,6 +65,7 @@ struct erofs_configure {
> > >   	char *mount_point;
> > >   	char *target_out_path;
> > >   	char *fs_config_file;
> > > +	char *block_list_file;
> > >   #endif
> > >   };
> > > diff --git a/lib/block_list.c b/lib/block_list.c
> > > new file mode 100644
> > > index 000000000000..ffe780e91900
> > > --- /dev/null
> > > +++ b/lib/block_list.c
> > > @@ -0,0 +1,86 @@
> > > +// SPDX-License-Identifier: GPL-2.0+
> > > +/*
> > > + * erofs-utils/lib/block_list.c
> > > + *
> > > + * Copyright (C), 2021, Coolpad Group Limited.
> > > + * Created by Yue Hu <huyue2@yulong.com>
> > > + */
> > > +#ifdef WITH_ANDROID
> > > +#include <stdio.h>
> > > +#include <sys/stat.h>
> > > +#include "erofs/block_list.h"
> > > +
> > > +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> > > +#include "erofs/print.h"
> > > +
> > > +static FILE *block_list_fp = NULL;
> > > +
> > > +int erofs_droid_blocklist_fopen(void)
> > > +{
> > > +	if (block_list_fp)
> > > +		return 0;
> > > +
> > > +	block_list_fp = fopen(cfg.block_list_file, "w");
> > > +
> > > +	if (!block_list_fp)
> > > +		return -1;
> > > +	return 0;
> > > +}
> > > +
> > > +void erofs_droid_blocklist_fclose(void)
> > > +{
> > > +	if (!block_list_fp)
> > > +		return;
> > > +
> > > +	fclose(block_list_fp);
> > > +	block_list_fp = NULL;
> > > +}
> > > +
> > > +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> > > +			    erofs_blk_t nblocks, bool has_tail)
> > > +{
> > > +	const char *fspath = erofs_fspath(path);
> > > +
> > > +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> > > +
> > > +	if (fspath[0] != '/')
> > > +		fprintf(block_list_fp, "/");
> > > +
> > > +	if (nblocks == 1)
> > > +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> > > +	else if (nblocks > 1)
> > > +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> > > +			blk_start + nblocks - 1);
> > > +	else
> > > +		fprintf(block_list_fp, "%s", fspath);
> > > +
> > > +	if (!has_tail)
> > > +		fprintf(block_list_fp, "\n");
> > > +}
> > > +
> > > +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> > > +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> > > +{
> > > +	if (!block_list_fp || !cfg.mount_point)
> > > +		return;
> > > +
> > > +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> > > +			!!inode->idata_size);
> > > +}
> > > +
> > > +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > > +					  erofs_blk_t blkaddr)
> > > +{
> > > +	if (!block_list_fp || !cfg.mount_point)
> > > +		return;
> > > +
> > > +	/* XXX: a bit hacky.. may need a better approach */
> > > +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> > > +		return;
> > > +
> > > +	if (blkaddr != NULL_ADDR)
> > > +		fprintf(block_list_fp, " %u\n", blkaddr);
> > > +	else
> > > +		fprintf(block_list_fp, "\n");
> > > +}  
> > void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> > 					  erofs_blk_t blkaddr)
> > {
> > 	if (!block_list_fp || !cfg.mount_point)
> > 		return;
> > 
> > 	/* XXX: a bit hacky.. may need a better approach */
> > 	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> > 		return;
> > 
> > 	if (erofs_blknr(inode->i_size)) { // its block list has been output before
> > 		if (blkaddr == NULL_ADDR)
> > 			fprintf(block_list_fp, "\n");
> > 		else
> > 			fprintf(block_list_fp, " %u\n", blkaddr);
> > 		return;
> > 	}
> > 	if (blkaddr != NULL_ADDR)
> > 		blocklist_write(inode->i_srcpath, blkaddr, 1, false);  
> 
> That still sounds a bit hacky. But if no better idea, we could leave it
> as-is... honestly, I don't quite like this.
> 
> Thanks,
> Gao Xiang
> 
> > }
> > 
> >   
> > > +#endif
> > > diff --git a/lib/compress.c b/lib/compress.c
> > > index 2093bfd68b71..af0c72037281 100644
> > > --- a/lib/compress.c
> > > +++ b/lib/compress.c
> > > @@ -18,6 +18,7 @@
> > >   #include "erofs/cache.h"
> > >   #include "erofs/compress.h"
> > >   #include "compressor.h"
> > > +#include "erofs/block_list.h"
> > >   static struct erofs_compress compresshandle;
> > >   static int compressionlevel;
> > > @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
> > >   		DBG_BUGON(ret);
> > >   	}
> > >   	inode->compressmeta = compressmeta;
> > > +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
> > >   	return 0;
> > >   err_bdrop:
> > > diff --git a/lib/inode.c b/lib/inode.c
> > > index 787e5b4485a2..4134f8a0e91b 100644
> > > --- a/lib/inode.c
> > > +++ b/lib/inode.c
> > > @@ -21,6 +21,7 @@
> > >   #include "erofs/compress.h"
> > >   #include "erofs/xattr.h"
> > >   #include "erofs/exclude.h"
> > > +#include "erofs/block_list.h"
> > >   #define S_SHIFT                 12
> > >   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> > > @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
> > >   			return -EIO;
> > >   		}
> > >   	}
> > > +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
> > >   	return 0;
> > >   }
> > > @@ -638,6 +640,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> > >   		ibh->fsprivate = erofs_igrab(inode);
> > >   		ibh->op = &erofs_write_inline_bhops;
> > > +
> > > +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
> > >   	} else {
> > >   		int ret;
> > >   		erofs_off_t pos;
> > > @@ -657,6 +661,8 @@ int erofs_write_tail_end(struct erofs_inode *inode)
> > >   		inode->idata_size = 0;
> > >   		free(inode->idata);
> > >   		inode->idata = NULL;
> > > +
> > > +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
> > >   	}
> > >   out:
> > >   	/* now bh_data can drop directly */
> > > diff --git a/mkfs/main.c b/mkfs/main.c
> > > index e476189f0731..28539da5ea5f 100644
> > > --- a/mkfs/main.c
> > > +++ b/mkfs/main.c
> > > @@ -22,6 +22,7 @@
> > >   #include "erofs/compress.h"
> > >   #include "erofs/xattr.h"
> > >   #include "erofs/exclude.h"
> > > +#include "erofs/block_list.h"
> > >   #ifdef HAVE_LIBUUID
> > >   #include <uuid.h>
> > > @@ -47,6 +48,7 @@ static struct option long_options[] = {
> > >   	{"mount-point", required_argument, NULL, 10},
> > >   	{"product-out", required_argument, NULL, 11},
> > >   	{"fs-config-file", required_argument, NULL, 12},
> > > +	{"block-list-file", required_argument, NULL, 13},
> > >   #endif
> > >   	{0, 0, 0, 0},
> > >   };
> > > @@ -95,6 +97,7 @@ static void usage(void)
> > >   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
> > >   	      " --product-out=X       X=product_out directory\n"
> > >   	      " --fs-config-file=X    X=fs_config file\n"
> > > +	      " --block-list-file=X    X=block_list file\n"
> > >   #endif
> > >   	      "\nAvailable compressors are: ", stderr);
> > >   	print_available_compressors(stderr, ", ");
> > > @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> > >   		case 12:
> > >   			cfg.fs_config_file = optarg;
> > >   			break;
> > > +		case 13:
> > > +			cfg.block_list_file = optarg;
> > > +			break;
> > >   #endif
> > >   		case 'C':
> > >   			i = strtoull(optarg, &endptr, 0);
> > > @@ -541,6 +547,11 @@ int main(int argc, char **argv)
> > >   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
> > >   		return 1;
> > >   	}
> > > +
> > > +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> > > +		erofs_err("failed to open %s", cfg.block_list_file);
> > > +		return 1;
> > > +	}
> > >   #endif
> > >   	erofs_show_config();
> > > @@ -607,6 +618,9 @@ int main(int argc, char **argv)
> > >   		err = erofs_mkfs_superblock_csum_set();
> > >   exit:
> > >   	z_erofs_compress_exit();
> > > +#ifdef WITH_ANDROID
> > > +	erofs_droid_blocklist_fclose();
> > > +#endif
> > >   	dev_close();
> > >   	erofs_cleanup_exclude_rules();
> > >   	erofs_exit_configure();
> > >   

[PATCH v3] AOSP: erofs-utils: add block list support

[Li GuiFu via Linux-erofs]

From: Yue Hu <huyue2@yulong.com>

Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.

Signed-off-by: Yue Hu <huyue2@yulong.com>
Signed-off-by: Gao Xiang <xiang@kernel.org>
Signed-off-by: Li Guifu <blucerlee@gmail.com>
---
 include/erofs/block_list.h | 27 +++++++++++
 include/erofs/config.h     |  1 +
 lib/block_list.c           | 91 ++++++++++++++++++++++++++++++++++++++
 lib/compress.c             |  2 +
 lib/inode.c                |  6 +++
 mkfs/main.c                | 14 ++++++
 6 files changed, 141 insertions(+)
 create mode 100644 include/erofs/block_list.h
 create mode 100644 lib/block_list.c

diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index 0000000..7756d8a
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+#ifdef WITH_ANDROID
+int erofs_droid_blocklist_fopen(void);
+void erofs_droid_blocklist_fclose(void);
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks);
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr);
+#else
+static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
+static inline
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr) {}
+#endif
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a73..67e7a0f 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
 	char *mount_point;
 	char *target_out_path;
 	char *fs_config_file;
+	char *block_list_file;
 #endif
 };
 
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index 0000000..90fe0fd
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu <huyue2@yulong.com>
+ */
+#ifdef WITH_ANDROID
+#include <stdio.h>
+#include <sys/stat.h>
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int erofs_droid_blocklist_fopen(void)
+{
+	if (block_list_fp)
+		return 0;
+
+	block_list_fp = fopen(cfg.block_list_file, "w");
+
+	if (!block_list_fp)
+		return -1;
+	return 0;
+}
+
+void erofs_droid_blocklist_fclose(void)
+{
+	if (!block_list_fp)
+		return;
+
+	fclose(block_list_fp);
+	block_list_fp = NULL;
+}
+
+static void blocklist_write(const char *path, erofs_blk_t blk_start,
+			    erofs_blk_t nblocks, bool has_tail)
+{
+	const char *fspath = erofs_fspath(path);
+
+	fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+	if (fspath[0] != '/')
+		fprintf(block_list_fp, "/");
+
+	if (nblocks == 1)
+		fprintf(block_list_fp, "%s %u", fspath, blk_start);
+	else if (nblocks > 1)
+		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+			blk_start + nblocks - 1);
+	else
+		fprintf(block_list_fp, "%s", fspath);
+
+	if (!has_tail)
+		fprintf(block_list_fp, "\n");
+}
+
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+				 erofs_blk_t blk_start, erofs_blk_t nblocks)
+{
+	if (!block_list_fp || !cfg.mount_point || !nblocks)
+		return;
+
+	blocklist_write(inode->i_srcpath, blk_start, nblocks,
+			!!inode->idata_size);
+}
+
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+					  erofs_blk_t blkaddr)
+{
+	if (!block_list_fp || !cfg.mount_point)
+		return;
+
+	/* XXX: a bit hacky.. may need a better approach */
+	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
+		return;
+
+	if (erofs_blknr(inode->i_size)) { // its block list has been output before
+		if (blkaddr == NULL_ADDR)
+			fprintf(block_list_fp, "\n");
+		else
+			fprintf(block_list_fp, " %u\n", blkaddr);
+		return;
+	}
+	if (blkaddr != NULL_ADDR)
+		blocklist_write(inode->i_srcpath, blkaddr, 1, false);
+}
+#endif
diff --git a/lib/compress.c b/lib/compress.c
index 2093bfd..af0c720 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -18,6 +18,7 @@
 #include "erofs/cache.h"
 #include "erofs/compress.h"
 #include "compressor.h"
+#include "erofs/block_list.h"
 
 static struct erofs_compress compresshandle;
 static int compressionlevel;
@@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
 		DBG_BUGON(ret);
 	}
 	inode->compressmeta = compressmeta;
+	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
 	return 0;
 
 err_bdrop:
diff --git a/lib/inode.c b/lib/inode.c
index b6108db..97f0cf7 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -21,6 +21,7 @@
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
 #include "erofs/exclude.h"
+#include "erofs/block_list.h"
 
 #define S_SHIFT                 12
 static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
@@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
 			return -EIO;
 		}
 	}
+	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
 	return 0;
 }
 
@@ -638,6 +640,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode)
 
 		ibh->fsprivate = erofs_igrab(inode);
 		ibh->op = &erofs_write_inline_bhops;
+
+		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
 	} else {
 		int ret;
 		erofs_off_t pos;
@@ -657,6 +661,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode)
 		inode->idata_size = 0;
 		free(inode->idata);
 		inode->idata = NULL;
+
+		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
 	}
 out:
 	/* now bh_data can drop directly */
diff --git a/mkfs/main.c b/mkfs/main.c
index e476189..28539da 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -22,6 +22,7 @@
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
 #include "erofs/exclude.h"
+#include "erofs/block_list.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid.h>
@@ -47,6 +48,7 @@ static struct option long_options[] = {
 	{"mount-point", required_argument, NULL, 10},
 	{"product-out", required_argument, NULL, 11},
 	{"fs-config-file", required_argument, NULL, 12},
+	{"block-list-file", required_argument, NULL, 13},
 #endif
 	{0, 0, 0, 0},
 };
@@ -95,6 +97,7 @@ static void usage(void)
 	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
 	      " --product-out=X       X=product_out directory\n"
 	      " --fs-config-file=X    X=fs_config file\n"
+	      " --block-list-file=X    X=block_list file\n"
 #endif
 	      "\nAvailable compressors are: ", stderr);
 	print_available_compressors(stderr, ", ");
@@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 		case 12:
 			cfg.fs_config_file = optarg;
 			break;
+		case 13:
+			cfg.block_list_file = optarg;
+			break;
 #endif
 		case 'C':
 			i = strtoull(optarg, &endptr, 0);
@@ -541,6 +547,11 @@ int main(int argc, char **argv)
 		erofs_err("failed to load fs config %s", cfg.fs_config_file);
 		return 1;
 	}
+
+	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
+		erofs_err("failed to open %s", cfg.block_list_file);
+		return 1;
+	}
 #endif
 
 	erofs_show_config();
@@ -607,6 +618,9 @@ int main(int argc, char **argv)
 		err = erofs_mkfs_superblock_csum_set();
 exit:
 	z_erofs_compress_exit();
+#ifdef WITH_ANDROID
+	erofs_droid_blocklist_fclose();
+#endif
 	dev_close();
 	erofs_cleanup_exclude_rules();
 	erofs_exit_configure();
-- 
2.25.1

Re: [PATCH v3] AOSP: erofs-utils: add block list support

[Yue Hu]

On Thu, 24 Jun 2021 02:03:05 -0400
Li GuiFu <bluce.lee@aliyun.com> wrote:

> From: Yue Hu <huyue2@yulong.com>
> 
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
> 
> Signed-off-by: Yue Hu <huyue2@yulong.com>
> Signed-off-by: Gao Xiang <xiang@kernel.org>
> Signed-off-by: Li Guifu <blucerlee@gmail.com>
> ---
>  include/erofs/block_list.h | 27 +++++++++++
>  include/erofs/config.h     |  1 +
>  lib/block_list.c           | 91 ++++++++++++++++++++++++++++++++++++++
>  lib/compress.c             |  2 +
>  lib/inode.c                |  6 +++
>  mkfs/main.c                | 14 ++++++
>  6 files changed, 141 insertions(+)
>  create mode 100644 include/erofs/block_list.h
>  create mode 100644 lib/block_list.c
> 
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 0000000..7756d8a
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,27 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +#ifdef WITH_ANDROID
> +int erofs_droid_blocklist_fopen(void);
> +void erofs_droid_blocklist_fclose(void);
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks);
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr);
> +#else
> +static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks) {}
> +static inline
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr) {}
> +#endif
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a73..67e7a0f 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>  	char *mount_point;
>  	char *target_out_path;
>  	char *fs_config_file;
> +	char *block_list_file;
>  #endif
>  };
>  
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 0000000..90fe0fd
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,91 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +#include <sys/stat.h>
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int erofs_droid_blocklist_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (!block_list_fp)
> +		return -1;
> +	return 0;
> +}
> +
> +void erofs_droid_blocklist_fclose(void)
> +{
> +	if (!block_list_fp)
> +		return;
> +
> +	fclose(block_list_fp);
> +	block_list_fp = NULL;
> +}
> +
> +static void blocklist_write(const char *path, erofs_blk_t blk_start,
> +			    erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1)
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	else if (nblocks > 1)
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	else
> +		fprintf(block_list_fp, "%s", fspath);

The last else is not needed since nblocks = 0 is not possible.

> +
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void erofs_droid_blocklist_write(struct erofs_inode *inode,
> +				 erofs_blk_t blk_start, erofs_blk_t nblocks)
> +{
> +	if (!block_list_fp || !cfg.mount_point || !nblocks)
> +		return;
> +
> +	blocklist_write(inode->i_srcpath, blk_start, nblocks,
> +			!!inode->idata_size);
> +}
> +
> +void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
> +					  erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* XXX: a bit hacky.. may need a better approach */
> +	if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
> +		return;
> +
> +	if (erofs_blknr(inode->i_size)) { // its block list has been output before
> +		if (blkaddr == NULL_ADDR)
> +			fprintf(block_list_fp, "\n");
> +		else
> +			fprintf(block_list_fp, " %u\n", blkaddr);
> +		return;
> +	}
> +	if (blkaddr != NULL_ADDR)
> +		blocklist_write(inode->i_srcpath, blkaddr, 1, false);
> +}
> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd..af0c720 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -18,6 +18,7 @@
>  #include "erofs/cache.h"
>  #include "erofs/compress.h"
>  #include "compressor.h"
> +#include "erofs/block_list.h"
>  
>  static struct erofs_compress compresshandle;
>  static int compressionlevel;
> @@ -571,6 +572,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>  		DBG_BUGON(ret);
>  	}
>  	inode->compressmeta = compressmeta;
> +	erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
>  	return 0;
>  
>  err_bdrop:
> diff --git a/lib/inode.c b/lib/inode.c
> index b6108db..97f0cf7 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -21,6 +21,7 @@
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
>  #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>  
>  #define S_SHIFT                 12
>  static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
> @@ -369,6 +370,7 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>  			return -EIO;
>  		}
>  	}
> +	erofs_droid_blocklist_write(inode, inode->u.i_blkaddr, nblocks);
>  	return 0;
>  }
>  
> @@ -638,6 +640,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode)
>  
>  		ibh->fsprivate = erofs_igrab(inode);
>  		ibh->op = &erofs_write_inline_bhops;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR);
>  	} else {
>  		int ret;
>  		erofs_off_t pos;
> @@ -657,6 +661,8 @@ static int erofs_write_tail_end(struct erofs_inode *inode)
>  		inode->idata_size = 0;
>  		free(inode->idata);
>  		inode->idata = NULL;
> +
> +		erofs_droid_blocklist_write_tail_end(inode, erofs_blknr(pos));
>  	}
>  out:
>  	/* now bh_data can drop directly */
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189..28539da 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,7 @@
>  #include "erofs/compress.h"
>  #include "erofs/xattr.h"
>  #include "erofs/exclude.h"
> +#include "erofs/block_list.h"
>  
>  #ifdef HAVE_LIBUUID
>  #include <uuid.h>
> @@ -47,6 +48,7 @@ static struct option long_options[] = {
>  	{"mount-point", required_argument, NULL, 10},
>  	{"product-out", required_argument, NULL, 11},
>  	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>  #endif
>  	{0, 0, 0, 0},
>  };
> @@ -95,6 +97,7 @@ static void usage(void)
>  	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>  	      " --product-out=X       X=product_out directory\n"
>  	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>  #endif
>  	      "\nAvailable compressors are: ", stderr);
>  	print_available_compressors(stderr, ", ");
> @@ -293,6 +296,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>  		case 12:
>  			cfg.fs_config_file = optarg;
>  			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>  #endif
>  		case 'C':
>  			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +547,11 @@ int main(int argc, char **argv)
>  		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>  		return 1;
>  	}
> +
> +	if (cfg.block_list_file && erofs_droid_blocklist_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>  #endif
>  
>  	erofs_show_config();
> @@ -607,6 +618,9 @@ int main(int argc, char **argv)
>  		err = erofs_mkfs_superblock_csum_set();
>  exit:
>  	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	erofs_droid_blocklist_fclose();
> +#endif
>  	dev_close();
>  	erofs_cleanup_exclude_rules();
>  	erofs_exit_configure();

Re: [PATCH] AOSP: erofs-utils: add block list support

[Li Guifu via Linux-erofs]

Hu Yue

   Thanks to your contribution.
   Base test shows it could record block map list correctly.
   Some codes need be refactored for further ahead.

On 6/21/21 11:02 PM, Yue Hu wrote:
> From: Yue Hu <huyue2@yulong.com>
> 
> Android update engine will treat EROFS filesystem image as one single
> file. Let's add block list support to optimize OTA size.
> 
> Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
> Signed-off-by: Yue Hu <huyue2@yulong.com>
> ---
>   include/erofs/block_list.h | 19 ++++++++++
>   include/erofs/config.h     |  1 +
>   lib/block_list.c           | 86 ++++++++++++++++++++++++++++++++++++++++++++++
>   lib/compress.c             |  8 +++++
>   lib/inode.c                | 21 ++++++++++-
>   mkfs/main.c                | 17 +++++++++
>   6 files changed, 151 insertions(+), 1 deletion(-)
>   create mode 100644 include/erofs/block_list.h
>   create mode 100644 lib/block_list.c

step1:
     block_list.c/h are new file, please add them to lib/Makefile.am or 
it will not be built.
     You would also add a Android.mk/bp for android target build

> 
> diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
> new file mode 100644
> index 0000000..cbf1050
> --- /dev/null
> +++ b/include/erofs/block_list.h
> @@ -0,0 +1,19 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/include/erofs/block_list.h
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifndef __EROFS_BLOCK_LIST_H
> +#define __EROFS_BLOCK_LIST_H
> +
> +#include "internal.h"
> +
> +int block_list_fopen(void);
> +void block_list_fclose(void);
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +                      erofs_blk_t nblocks, bool has_tail);
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +                               bool inline_data, erofs_blk_t blkaddr);
> +#endif
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index d140a73..67e7a0f 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -65,6 +65,7 @@ struct erofs_configure {
>   	char *mount_point;
>   	char *target_out_path;
>   	char *fs_config_file;
> +	char *block_list_file;
>   #endif
>   };
>   
> diff --git a/lib/block_list.c b/lib/block_list.c
> new file mode 100644
> index 0000000..6ebe0f9
> --- /dev/null
> +++ b/lib/block_list.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/block_list.c
> + *
> + * Copyright (C), 2021, Coolpad Group Limited.
> + * Created by Yue Hu <huyue2@yulong.com>
> + */
> +#ifdef WITH_ANDROID
> +#include <stdio.h>
> +
> +#include "erofs/block_list.h"
> +
> +#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
> +#include "erofs/print.h"
> +
> +static FILE *block_list_fp = NULL;
> +
> +int block_list_fopen(void)
> +{
> +	if (block_list_fp)
> +		return 0;
> +
> +	block_list_fp = fopen(cfg.block_list_file, "w");
> +
> +	if (block_list_fp == NULL)
> +		return -1;
step2:
	The return code would be replace with errno please.
	So further error message could be showed up.
> +
> +	return 0;
> +}
> +
> +void block_list_fclose(void)
> +{
> +	if (block_list_fp) {
> +		fclose(block_list_fp);
> +		block_list_fp = NULL;
> +	}
> +}
> +
> +void write_block_list(const char *path, erofs_blk_t blk_start,
> +		      erofs_blk_t nblocks, bool has_tail)
> +{
> +	const char *fspath = erofs_fspath(path);
> +
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
> +
> +	/* only tail-end data */
> +	if (!nblocks)
> +		return;
> +
> +	fprintf(block_list_fp, "/%s", cfg.mount_point);
> +
> +	if (fspath[0] != '/')
> +		fprintf(block_list_fp, "/");
> +
> +	if (nblocks == 1) {
> +		fprintf(block_list_fp, "%s %u", fspath, blk_start);
> +	} else {
> +		fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
> +			blk_start + nblocks - 1);
> +	}
> +
> +	if (!has_tail)
> +		fprintf(block_list_fp, "\n");
> +}
> +
> +void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
> +			       bool inline_data, erofs_blk_t blkaddr)
> +{
> +	if (!block_list_fp || !cfg.mount_point)
> +		return;
step3:
	if (!block_list_fp || !cfg.mount_point)
	These codes are double checked at write_block_list() function.
	Could you do a further optimize ?

> +
> +	if (!nblocks && !inline_data) {
> +		erofs_dbg("%s : only tail-end non-inline data", path);
> +		write_block_list(path, blkaddr, 1, false);
> +		return;
> +	}
> +
> +	if (nblocks) {
> +		if (!inline_data)
> +			fprintf(block_list_fp, " %u", blkaddr);
> +
> +		fprintf(block_list_fp, "\n");
> +	}
> +}
step4:
	write_block_list_tail_end() has much if-condition nesting,
	It could return early at the begin of it.
	

> +#endif
> diff --git a/lib/compress.c b/lib/compress.c
> index 2093bfd..5dec0c3 100644
> --- a/lib/compress.c
> +++ b/lib/compress.c
> @@ -19,6 +19,10 @@
>   #include "erofs/compress.h"
>   #include "compressor.h"
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   static struct erofs_compress compresshandle;
>   static int compressionlevel;
>   
> @@ -553,6 +557,10 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
>   		   inode->i_srcpath, (unsigned long long)inode->i_size,
>   		   compressed_blocks);
>   
> +#ifdef WITH_ANDROID
> +	write_block_list(inode->i_srcpath, blkaddr, compressed_blocks, false);
> +#endif
> +
>   	/*
>   	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
>   	 *       when both mkfs & kernel support compression inline.
> diff --git a/lib/inode.c b/lib/inode.c
> index 787e5b4..6be23cb 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -22,6 +22,10 @@
>   #include "erofs/xattr.h"
>   #include "erofs/exclude.h"
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   #define S_SHIFT                 12
>   static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
>   	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
> @@ -369,6 +373,12 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
>   			return -EIO;
>   		}
>   	}
> +
> +#ifdef WITH_ANDROID
> +	if (nblocks)
> +		write_block_list(inode->i_srcpath, inode->u.i_blkaddr,
> +				 nblocks, inode->idata_size ? true : false);
> +#endif
>   	return 0;
>   }
>   
> @@ -626,6 +636,7 @@ static struct erofs_bhops erofs_write_inline_bhops = {
>   int erofs_write_tail_end(struct erofs_inode *inode)
>   {
>   	struct erofs_buffer_head *bh, *ibh;
> +	erofs_off_t pos;
>   
>   	bh = inode->bh_data;
>   
> @@ -640,7 +651,6 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   		ibh->op = &erofs_write_inline_bhops;
>   	} else {
>   		int ret;
> -		erofs_off_t pos;
>   
>   		erofs_mapbh(bh->block);
>   		pos = erofs_btell(bh, true) - EROFS_BLKSIZ;
> @@ -658,6 +668,15 @@ int erofs_write_tail_end(struct erofs_inode *inode)
>   		free(inode->idata);
>   		inode->idata = NULL;
>   	}
> +
> +#ifdef WITH_ANDROID
> +	if (!S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))
> +		write_block_list_tail_end(inode->i_srcpath,
> +					  inode->i_size / EROFS_BLKSIZ,
> +					  inode->bh_inline ? true: false,
> +					  erofs_blknr(pos));
step5:
	inode->i_size / EROFS_BLKSIZ,
	It could be replace with erofs_blknr() just like pos.
	The *pos* here would cause build errorly with message,
	"./include/erofs/internal.h:55:41: error: ‘pos’ may be used 
uninitialized in this function [-Werror=maybe-uninitialized]"
> +#endif
> +
>   out:
>   	/* now bh_data can drop directly */
>   	if (bh) {
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e476189..d5a5e07 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -27,6 +27,10 @@
>   #include <uuid.h>
>   #endif
>   
> +#ifdef WITH_ANDROID
> +#include "erofs/block_list.h"
> +#endif
> +
>   #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>   
>   static struct option long_options[] = {
> @@ -47,6 +51,7 @@ static struct option long_options[] = {
>   	{"mount-point", required_argument, NULL, 10},
>   	{"product-out", required_argument, NULL, 11},
>   	{"fs-config-file", required_argument, NULL, 12},
> +	{"block-list-file", required_argument, NULL, 13},
>   #endif
>   	{0, 0, 0, 0},
>   };
> @@ -95,6 +100,7 @@ static void usage(void)
>   	      " --mount-point=X       X=prefix of target fs path (default: /)\n"
>   	      " --product-out=X       X=product_out directory\n"
>   	      " --fs-config-file=X    X=fs_config file\n"
> +	      " --block-list-file=X    X=block_list file\n"
>   #endif
>   	      "\nAvailable compressors are: ", stderr);
>   	print_available_compressors(stderr, ", ");
> @@ -293,6 +299,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   		case 12:
>   			cfg.fs_config_file = optarg;
>   			break;
> +		case 13:
> +			cfg.block_list_file = optarg;
> +			break;
>   #endif
>   		case 'C':
>   			i = strtoull(optarg, &endptr, 0);
> @@ -541,6 +550,11 @@ int main(int argc, char **argv)
>   		erofs_err("failed to load fs config %s", cfg.fs_config_file);
>   		return 1;
>   	}
> +
> +	if (cfg.block_list_file && block_list_fopen() < 0) {
> +		erofs_err("failed to open %s", cfg.block_list_file);
> +		return 1;
> +	}
>   #endif
>   
>   	erofs_show_config();
> @@ -607,6 +621,9 @@ int main(int argc, char **argv)
>   		err = erofs_mkfs_superblock_csum_set();
>   exit:
>   	z_erofs_compress_exit();
> +#ifdef WITH_ANDROID
> +	block_list_fclose();
> +#endif
>   	dev_close();
>   	erofs_cleanup_exclude_rules();
>   	erofs_exit_configure();
> 

Re: [PATCH] AOSP: erofs-utils: add block list support

[Gao Xiang]

On Wed, Jun 23, 2021 at 11:58:15PM -0400, Li Guifu wrote:
> Hu Yue
> 
>   Thanks to your contribution.
>   Base test shows it could record block map list correctly.
>   Some codes need be refactored for further ahead.

Guifu,
could you check my v2 as well?
https://lore.kernel.org/linux-erofs/1624469489-40907-1-git-send-email-hsiangkao@linux.alibaba.com

does it look good to you?

Thanks,
Gao Xiang