Re: [PATCH-v5] erofs-utils:code for calculating crc checksum of erofs blocks

[Gao Xiang via Linux-erofs <linux-erofs@lists.ozlabs.org>]

Hi Pratik,

I was attending "China Linux Storage and File System (CLSF) Workshop
2019" (with EROFS roadmap discussed) these days, sorry about delaying
... I will take this version with little print messages added this
evening... Don't worry, I will certainly do. :-)

Thanks,
Gao Xiang

On Sat, Oct 19, 2019 at 08:38:03PM +0530, Pratik Shinde wrote:
> Added code for calculating crc of erofs blocks (4K size).for now it calculates
> checksum of first block. but can modified to calculate crc for any no. of blocks
> 
> Fill 'feature_compat' field of erofs_super_block so that it
> can be used on kernel side. also fixing one typo.
> 
> Signed-off-by: Pratik Shinde <pratikshinde320@gmail.com>
> ---
>  include/erofs/internal.h |  1 +
>  include/erofs/io.h       |  8 ++++++
>  include/erofs_fs.h       |  5 ++--
>  lib/io.c                 | 27 ++++++++++++++++++
>  mkfs/main.c              | 71 ++++++++++++++++++++++++++++++++++++++++++++++++
>  5 files changed, 110 insertions(+), 2 deletions(-)
> 
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 5384946..53335bc 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -55,6 +55,7 @@ struct erofs_sb_info {
>  	u32 feature_incompat;
>  	u64 build_time;
>  	u32 build_time_nsec;
> +	u32 feature;
>  };
>  
>  /* global sbi */
> diff --git a/include/erofs/io.h b/include/erofs/io.h
> index 9775047..e0ca8d9 100644
> --- a/include/erofs/io.h
> +++ b/include/erofs/io.h
> @@ -19,6 +19,7 @@
>  int dev_open(const char *devname);
>  void dev_close(void);
>  int dev_write(const void *buf, u64 offset, size_t len);
> +int dev_read(void *buf, u64 offset, size_t len);
>  int dev_fillzero(u64 offset, size_t len, bool padding);
>  int dev_fsync(void);
>  int dev_resize(erofs_blk_t nblocks);
> @@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
>  			 blknr_to_addr(nblocks));
>  }
>  
> +static inline int blk_read(void *buf, erofs_blk_t start,
> +			    u32 nblocks)
> +{
> +	return dev_read(buf, blknr_to_addr(start),
> +			 blknr_to_addr(nblocks));
> +}
> +
>  #endif
>  
> diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> index f29aa25..9eda6c2 100644
> --- a/include/erofs_fs.h
> +++ b/include/erofs_fs.h
> @@ -19,6 +19,7 @@
>   */
>  #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
>  #define EROFS_ALL_FEATURE_INCOMPAT		EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> +#define EROFS_FEATURE_SB_CHKSUM	0x0001
>  
>  /* 128-byte erofs on-disk super block */
>  struct erofs_super_block {
> @@ -39,8 +40,8 @@ struct erofs_super_block {
>  	__u8 uuid[16];          /* 128-bit uuid for volume */
>  	__u8 volume_name[16];   /* volume name */
>  	__le32 feature_incompat;
> -
> -	__u8 reserved2[44];
> +	__le32 chksum_blocks;	/* number of blocks used for checksum */
> +	__u8 reserved2[40];
>  };
>  
>  /*
> diff --git a/lib/io.c b/lib/io.c
> index 7f5f94d..52f9424 100644
> --- a/lib/io.c
> +++ b/lib/io.c
> @@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
>  	return dev_fillzero(st.st_size, length, true);
>  }
>  
> +int dev_read(void *buf, u64 offset, size_t len)
> +{
> +	int ret;
> +
> +	if (cfg.c_dry_run)
> +		return 0;
> +
> +	if (!buf) {
> +		erofs_err("buf is NULL");
> +		return -EINVAL;
> +	}
> +	if (offset >= erofs_devsz || len > erofs_devsz ||
> +	    offset > erofs_devsz - len) {
> +		erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
> +			  "the end of device(%" PRIu64 ").",
> +			  offset, len, erofs_devsz);
> +		return -EINVAL;
> +	}
> +
> +	ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
> +	if (ret != (int)len) {
> +		erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
> +			  erofs_devname, offset, len);
> +		return -errno;
> +	}
> +	return 0;
> +}
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 91a018f..c7cb923 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,9 @@
>  
>  #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>  
> +/* number of blocks for calculating checksum */
> +#define EROFS_CKSUM_BLOCKS	1
> +
>  static void usage(void)
>  {
>  	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
> @@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
>  				return -EINVAL;
>  			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
>  		}
> +
> +		if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
> +			sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
> +		}
>  	}
>  	return 0;
>  }
> @@ -180,6 +187,9 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>  		.meta_blkaddr  = sbi.meta_blkaddr,
>  		.xattr_blkaddr = 0,
>  		.feature_incompat = cpu_to_le32(sbi.feature_incompat),
> +		.feature_compat = cpu_to_le32(sbi.feature),
> +		.checksum = 0,
> +		.chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
>  	};
>  	const unsigned int sb_blksize =
>  		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
> @@ -202,6 +212,64 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>  	return 0;
>  }
>  
> +#define CRCPOLY	0x82F63B78
> +static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
> +{
> +	int i;
> +	u32 crc = seed;
> +
> +	while (len--) {
> +		crc ^= *in++;
> +		for (i = 0; i < 8; i++) {
> +			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
> +		}
> +	}
> +	return crc;
> +}
> +
> +/* calculate checksum for first n blocks */
> +u32 erofs_calc_blk_checksum(erofs_blk_t nblks, u32 *crc)
> +{
> +	char *buf;
> +	int err = 0;
> +
> +	buf = malloc(nblks * EROFS_BLKSIZ);
> +	err = blk_read(buf, 0, nblks);
> +	if (err) {
> +		return err;
> +	}
> +	*crc = crc32c(0, (const unsigned char *)buf, nblks * EROFS_BLKSIZ);
> +	free(buf);
> +	return 0;
> +}
> +
> +void erofs_write_sb_checksum()
> +{
> +	struct erofs_super_block *sb;
> +	char buf[EROFS_BLKSIZ];
> +	int ret = 0;
> +	u32 crc;
> +
> +	ret = erofs_calc_blk_checksum(EROFS_CKSUM_BLOCKS, &crc);
> +	if (ret) {
> +		return;
> +	}
> +	ret = blk_read(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +
> +	sb = (struct erofs_super_block *)((u8 *)buf + EROFS_SUPER_OFFSET);
> +	if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
> +		return;
> +	}
> +	sb->checksum = cpu_to_le32(crc);
> +	ret = blk_write(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +}
> +
>  int main(int argc, char **argv)
>  {
>  	int err = 0;
> @@ -217,6 +285,7 @@ int main(int argc, char **argv)
>  
>  	cfg.c_legacy_compress = false;
>  	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
> +	sbi.feature = EROFS_FEATURE_SB_CHKSUM;
>  
>  	err = mkfs_parse_options_cfg(argc, argv);
>  	if (err) {
> @@ -301,6 +370,8 @@ int main(int argc, char **argv)
>  		err = -EIO;
>  	else
>  		err = dev_resize(nblocks);
> +	if (sbi.feature & EROFS_FEATURE_SB_CHKSUM)
> +		erofs_write_sb_checksum();
>  exit:
>  	z_erofs_compress_exit();
>  	dev_close();
> -- 
> 2.9.3
>