Linux-EROFS Archive on lore.kernel.org
 help / color / Atom feed
* [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks.
@ 2019-10-15  5:18 Pratik Shinde
  2019-10-15  6:16 ` Gao Xiang
  2019-10-15 15:11 ` Li Guifu
  0 siblings, 2 replies; 4+ messages in thread
From: Pratik Shinde @ 2019-10-15  5:18 UTC (permalink / raw)
  To: linux-erofs, bluce.liguifu, miaoxie, fangwei1

Added code for calculating crc of erofs blocks (4K size).for now it calculates
checksum of first block. but can modified to calculate crc for any no. of blocks

1) Added 'chksum_blocks' field to erofs_super_block
2) removed unnecessary prints
3) moved EROFS_FEATURE_SB_CHKSUM include/erofs_fs.h

Signed-off-by: Pratik Shinde <pratikshinde320@gmail.com>
---
 include/erofs/internal.h |  1 +
 include/erofs/io.h       |  8 ++++++
 include/erofs_fs.h       |  3 ++-
 lib/io.c                 | 27 +++++++++++++++++++
 mkfs/main.c              | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 5384946..53335bc 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -55,6 +55,7 @@ struct erofs_sb_info {
 	u32 feature_incompat;
 	u64 build_time;
 	u32 build_time_nsec;
+	u32 feature;
 };
 
 /* global sbi */
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 9775047..e0ca8d9 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -19,6 +19,7 @@
 int dev_open(const char *devname);
 void dev_close(void);
 int dev_write(const void *buf, u64 offset, size_t len);
+int dev_read(void *buf, u64 offset, size_t len);
 int dev_fillzero(u64 offset, size_t len, bool padding);
 int dev_fsync(void);
 int dev_resize(erofs_blk_t nblocks);
@@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
 			 blknr_to_addr(nblocks));
 }
 
+static inline int blk_read(void *buf, erofs_blk_t start,
+			    u32 nblocks)
+{
+	return dev_read(buf, blknr_to_addr(start),
+			 blknr_to_addr(nblocks));
+}
+
 #endif
 
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index f29aa25..8bd29d6 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -19,6 +19,7 @@
  */
 #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
 #define EROFS_ALL_FEATURE_INCOMPAT		EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
+#define EROFS_FEATURE_SB_CHKSUM	0x0001
 
 /* 128-byte erofs on-disk super block */
 struct erofs_super_block {
@@ -39,7 +40,7 @@ struct erofs_super_block {
 	__u8 uuid[16];          /* 128-bit uuid for volume */
 	__u8 volume_name[16];   /* volume name */
 	__le32 feature_incompat;
-
+	__le32 chksum_blocks;	/* number of blocks used for checksum */
 	__u8 reserved2[44];
 };
 
diff --git a/lib/io.c b/lib/io.c
index 7f5f94d..52f9424 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
 	return dev_fillzero(st.st_size, length, true);
 }
 
+int dev_read(void *buf, u64 offset, size_t len)
+{
+	int ret;
+
+	if (cfg.c_dry_run)
+		return 0;
+
+	if (!buf) {
+		erofs_err("buf is NULL");
+		return -EINVAL;
+	}
+	if (offset >= erofs_devsz || len > erofs_devsz ||
+	    offset > erofs_devsz - len) {
+		erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
+			  "the end of device(%" PRIu64 ").",
+			  offset, len, erofs_devsz);
+		return -EINVAL;
+	}
+
+	ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
+	if (ret != (int)len) {
+		erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
+			  erofs_devname, offset, len);
+		return -errno;
+	}
+	return 0;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index 91a018f..0633e83 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -22,6 +22,9 @@
 
 #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
 
+/* number of blocks for calculating checksum */
+#define EROFS_CKSUM_BLOCKS	1
+
 static void usage(void)
 {
 	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
@@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
 				return -EINVAL;
 			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
 		}
+
+		if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
+			sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
+		}
 	}
 	return 0;
 }
@@ -180,6 +187,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 		.meta_blkaddr  = sbi.meta_blkaddr,
 		.xattr_blkaddr = 0,
 		.feature_incompat = cpu_to_le32(sbi.feature_incompat),
+		.checksum = 0,
+		.chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
 	};
 	const unsigned int sb_blksize =
 		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -202,6 +211,64 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 	return 0;
 }
 
+#define CRCPOLY	0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+	int i;
+	u32 crc = seed;
+
+	while (len--) {
+		crc ^= *in++;
+		for (i = 0; i < 8; i++) {
+			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+		}
+	}
+	return crc;
+}
+
+/* calculate checksum for first n blocks */
+u32 erofs_calc_blk_checksum(erofs_blk_t nblks, u32 *crc)
+{
+	char *buf;
+	int err = 0;
+
+	buf = malloc(nblks * EROFS_BLKSIZ);
+	err = blk_read(buf, 0, nblks);
+	if (err) {
+		return err;
+	}
+	*crc = crc32c(0, (const unsigned char *)buf, nblks * EROFS_BLKSIZ);
+	free(buf);
+	return 0;
+}
+
+void erofs_write_sb_checksum()
+{
+	struct erofs_super_block *sb;
+	char buf[EROFS_BLKSIZ];
+	int ret = 0;
+	u32 crc;
+
+	ret = erofs_calc_blk_checksum(EROFS_CKSUM_BLOCKS, &crc);
+	if (ret) {
+		return;
+	}
+	ret = blk_read(buf, 0, 1);
+	if (ret) {
+		return;
+	}
+
+	sb = (struct erofs_super_block *)((u8 *)buf + EROFS_SUPER_OFFSET);
+	if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
+		return;
+	}
+	sb->checksum = cpu_to_le32(crc);
+	ret = blk_write(buf, 0, 1);
+	if (ret) {
+		return;
+	}
+}
+
 int main(int argc, char **argv)
 {
 	int err = 0;
@@ -217,6 +284,7 @@ int main(int argc, char **argv)
 
 	cfg.c_legacy_compress = false;
 	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
+	sbi.feature = EROFS_FEATURE_SB_CHKSUM;
 
 	err = mkfs_parse_options_cfg(argc, argv);
 	if (err) {
@@ -301,6 +369,8 @@ int main(int argc, char **argv)
 		err = -EIO;
 	else
 		err = dev_resize(nblocks);
+	if (sbi.feature & EROFS_FEATURE_SB_CHKSUM)
+		erofs_write_sb_checksum();
 exit:
 	z_erofs_compress_exit();
 	dev_close();
-- 
2.9.3


^ permalink raw reply	[flat|nested] 4+ messages in thread

* Re: [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks.
  2019-10-15  5:18 [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks Pratik Shinde
@ 2019-10-15  6:16 ` Gao Xiang
  2019-10-15 15:11 ` Li Guifu
  1 sibling, 0 replies; 4+ messages in thread
From: Gao Xiang @ 2019-10-15  6:16 UTC (permalink / raw)
  To: Pratik Shinde; +Cc: miaoxie, linux-erofs

Hi Pratik,

On Tue, Oct 15, 2019 at 10:48:30AM +0530, Pratik Shinde wrote:
> Added code for calculating crc of erofs blocks (4K size).for now it calculates
> checksum of first block. but can modified to calculate crc for any no. of blocks
> 
> 1) Added 'chksum_blocks' field to erofs_super_block
> 2) removed unnecessary prints

I think leaving some prints is necessary, but users should know
something is wrong when doing checksumming, so these prints should
be better with words "checksumming"....

> 3) moved EROFS_FEATURE_SB_CHKSUM include/erofs_fs.h
> 
> Signed-off-by: Pratik Shinde <pratikshinde320@gmail.com>
> ---
>  include/erofs/internal.h |  1 +
>  include/erofs/io.h       |  8 ++++++
>  include/erofs_fs.h       |  3 ++-
>  lib/io.c                 | 27 +++++++++++++++++++
>  mkfs/main.c              | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
>  5 files changed, 108 insertions(+), 1 deletion(-)
> 
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 5384946..53335bc 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -55,6 +55,7 @@ struct erofs_sb_info {
>  	u32 feature_incompat;
>  	u64 build_time;
>  	u32 build_time_nsec;
> +	u32 feature;
>  };
>  
>  /* global sbi */
> diff --git a/include/erofs/io.h b/include/erofs/io.h
> index 9775047..e0ca8d9 100644
> --- a/include/erofs/io.h
> +++ b/include/erofs/io.h
> @@ -19,6 +19,7 @@
>  int dev_open(const char *devname);
>  void dev_close(void);
>  int dev_write(const void *buf, u64 offset, size_t len);
> +int dev_read(void *buf, u64 offset, size_t len);
>  int dev_fillzero(u64 offset, size_t len, bool padding);
>  int dev_fsync(void);
>  int dev_resize(erofs_blk_t nblocks);
> @@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
>  			 blknr_to_addr(nblocks));
>  }
>  
> +static inline int blk_read(void *buf, erofs_blk_t start,
> +			    u32 nblocks)
> +{
> +	return dev_read(buf, blknr_to_addr(start),
> +			 blknr_to_addr(nblocks));
> +}
> +
>  #endif
>  
> diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> index f29aa25..8bd29d6 100644
> --- a/include/erofs_fs.h
> +++ b/include/erofs_fs.h
> @@ -19,6 +19,7 @@
>   */
>  #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
>  #define EROFS_ALL_FEATURE_INCOMPAT		EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> +#define EROFS_FEATURE_SB_CHKSUM	0x0001
>  
>  /* 128-byte erofs on-disk super block */
>  struct erofs_super_block {
> @@ -39,7 +40,7 @@ struct erofs_super_block {
>  	__u8 uuid[16];          /* 128-bit uuid for volume */
>  	__u8 volume_name[16];   /* volume name */
>  	__le32 feature_incompat;
> -
> +	__le32 chksum_blocks;	/* number of blocks used for checksum */
>  	__u8 reserved2[44];

reserved2[44]  --> reserved2[40];

I think totally this patch is ok, you can resend or I will take
this version to experimental branch with small modification
tonight... And after the kernel side is ready, we can test
properly and merge into mainline.

Thanks,
Gao Xiang


>  };
>  
> diff --git a/lib/io.c b/lib/io.c
> index 7f5f94d..52f9424 100644
> --- a/lib/io.c
> +++ b/lib/io.c
> @@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
>  	return dev_fillzero(st.st_size, length, true);
>  }
>  
> +int dev_read(void *buf, u64 offset, size_t len)
> +{
> +	int ret;
> +
> +	if (cfg.c_dry_run)
> +		return 0;
> +
> +	if (!buf) {
> +		erofs_err("buf is NULL");
> +		return -EINVAL;
> +	}
> +	if (offset >= erofs_devsz || len > erofs_devsz ||
> +	    offset > erofs_devsz - len) {
> +		erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
> +			  "the end of device(%" PRIu64 ").",
> +			  offset, len, erofs_devsz);
> +		return -EINVAL;
> +	}
> +
> +	ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
> +	if (ret != (int)len) {
> +		erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
> +			  erofs_devname, offset, len);
> +		return -errno;
> +	}
> +	return 0;
> +}
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 91a018f..0633e83 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,9 @@
>  
>  #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>  
> +/* number of blocks for calculating checksum */
> +#define EROFS_CKSUM_BLOCKS	1
> +
>  static void usage(void)
>  {
>  	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
> @@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
>  				return -EINVAL;
>  			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
>  		}
> +
> +		if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
> +			sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
> +		}
>  	}
>  	return 0;
>  }
> @@ -180,6 +187,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>  		.meta_blkaddr  = sbi.meta_blkaddr,
>  		.xattr_blkaddr = 0,
>  		.feature_incompat = cpu_to_le32(sbi.feature_incompat),
> +		.checksum = 0,
> +		.chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
>  	};
>  	const unsigned int sb_blksize =
>  		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
> @@ -202,6 +211,64 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>  	return 0;
>  }
>  
> +#define CRCPOLY	0x82F63B78
> +static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
> +{
> +	int i;
> +	u32 crc = seed;
> +
> +	while (len--) {
> +		crc ^= *in++;
> +		for (i = 0; i < 8; i++) {
> +			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
> +		}
> +	}
> +	return crc;
> +}
> +
> +/* calculate checksum for first n blocks */
> +u32 erofs_calc_blk_checksum(erofs_blk_t nblks, u32 *crc)
> +{
> +	char *buf;
> +	int err = 0;
> +
> +	buf = malloc(nblks * EROFS_BLKSIZ);
> +	err = blk_read(buf, 0, nblks);
> +	if (err) {
> +		return err;
> +	}
> +	*crc = crc32c(0, (const unsigned char *)buf, nblks * EROFS_BLKSIZ);
> +	free(buf);
> +	return 0;
> +}
> +
> +void erofs_write_sb_checksum()
> +{
> +	struct erofs_super_block *sb;
> +	char buf[EROFS_BLKSIZ];
> +	int ret = 0;
> +	u32 crc;
> +
> +	ret = erofs_calc_blk_checksum(EROFS_CKSUM_BLOCKS, &crc);
> +	if (ret) {
> +		return;
> +	}
> +	ret = blk_read(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +
> +	sb = (struct erofs_super_block *)((u8 *)buf + EROFS_SUPER_OFFSET);
> +	if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
> +		return;
> +	}
> +	sb->checksum = cpu_to_le32(crc);
> +	ret = blk_write(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +}
> +
>  int main(int argc, char **argv)
>  {
>  	int err = 0;
> @@ -217,6 +284,7 @@ int main(int argc, char **argv)
>  
>  	cfg.c_legacy_compress = false;
>  	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
> +	sbi.feature = EROFS_FEATURE_SB_CHKSUM;
>  
>  	err = mkfs_parse_options_cfg(argc, argv);
>  	if (err) {
> @@ -301,6 +369,8 @@ int main(int argc, char **argv)
>  		err = -EIO;
>  	else
>  		err = dev_resize(nblocks);
> +	if (sbi.feature & EROFS_FEATURE_SB_CHKSUM)
> +		erofs_write_sb_checksum();
>  exit:
>  	z_erofs_compress_exit();
>  	dev_close();
> -- 
> 2.9.3
> 

^ permalink raw reply	[flat|nested] 4+ messages in thread

* Re: [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks.
  2019-10-15  5:18 [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks Pratik Shinde
  2019-10-15  6:16 ` Gao Xiang
@ 2019-10-15 15:11 ` Li Guifu
  2019-10-15 15:18   ` Gao Xiang via Linux-erofs
  1 sibling, 1 reply; 4+ messages in thread
From: Li Guifu @ 2019-10-15 15:11 UTC (permalink / raw)
  To: Pratik Shinde, linux-erofs, bluce.liguifu, miaoxie, fangwei1

Hi Pratik, Gao Xiang

Can sbi.fearture merge into sbi.feature_incompat ?
If not, it will has two feature parameter

It has the blocks field in the super block, chksum_blocks duplicates it.
and chksum may be used to check whole erofs image file.

> Added code for calculating crc of erofs blocks (4K size).for now it calculates
> checksum of first block. but can modified to calculate crc for any no. of blocks
> 
> 1) Added 'chksum_blocks' field to erofs_super_block
> 2) removed unnecessary prints
> 3) moved EROFS_FEATURE_SB_CHKSUM include/erofs_fs.h
> 
> Signed-off-by: Pratik Shinde <pratikshinde320@gmail.com>
> ---
>   include/erofs/internal.h |  1 +
>   include/erofs/io.h       |  8 ++++++
>   include/erofs_fs.h       |  3 ++-
>   lib/io.c                 | 27 +++++++++++++++++++
>   mkfs/main.c              | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
>   5 files changed, 108 insertions(+), 1 deletion(-)
> 
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 5384946..53335bc 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -55,6 +55,7 @@ struct erofs_sb_info {
>   	u32 feature_incompat;
>   	u64 build_time;
>   	u32 build_time_nsec;
> +	u32 feature;
>   };
>   
>   /* global sbi */
> diff --git a/include/erofs/io.h b/include/erofs/io.h
> index 9775047..e0ca8d9 100644
> --- a/include/erofs/io.h
> +++ b/include/erofs/io.h
> @@ -19,6 +19,7 @@
>   int dev_open(const char *devname);
>   void dev_close(void);
>   int dev_write(const void *buf, u64 offset, size_t len);
> +int dev_read(void *buf, u64 offset, size_t len);
>   int dev_fillzero(u64 offset, size_t len, bool padding);
>   int dev_fsync(void);
>   int dev_resize(erofs_blk_t nblocks);
> @@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
>   			 blknr_to_addr(nblocks));
>   }
>   
> +static inline int blk_read(void *buf, erofs_blk_t start,
> +			    u32 nblocks)
> +{
> +	return dev_read(buf, blknr_to_addr(start),
> +			 blknr_to_addr(nblocks));
> +}
> +
>   #endif
>   
> diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> index f29aa25..8bd29d6 100644
> --- a/include/erofs_fs.h
> +++ b/include/erofs_fs.h
> @@ -19,6 +19,7 @@
>    */
>   #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
>   #define EROFS_ALL_FEATURE_INCOMPAT		EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> +#define EROFS_FEATURE_SB_CHKSUM	0x0001
>   
>   /* 128-byte erofs on-disk super block */
>   struct erofs_super_block {
> @@ -39,7 +40,7 @@ struct erofs_super_block {
>   	__u8 uuid[16];          /* 128-bit uuid for volume */
>   	__u8 volume_name[16];   /* volume name */
>   	__le32 feature_incompat;
> -
> +	__le32 chksum_blocks;	/* number of blocks used for checksum */
>   	__u8 reserved2[44];
>   };
>   
> diff --git a/lib/io.c b/lib/io.c
> index 7f5f94d..52f9424 100644
> --- a/lib/io.c
> +++ b/lib/io.c
> @@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
>   	return dev_fillzero(st.st_size, length, true);
>   }
>   
> +int dev_read(void *buf, u64 offset, size_t len)
> +{
> +	int ret;
> +
> +	if (cfg.c_dry_run)
> +		return 0;
> +
> +	if (!buf) {
> +		erofs_err("buf is NULL");
> +		return -EINVAL;
> +	}
> +	if (offset >= erofs_devsz || len > erofs_devsz ||
> +	    offset > erofs_devsz - len) {
> +		erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
> +			  "the end of device(%" PRIu64 ").",
> +			  offset, len, erofs_devsz);
> +		return -EINVAL;
> +	}
> +
> +	ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
> +	if (ret != (int)len) {
> +		erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
> +			  erofs_devname, offset, len);
> +		return -errno;
> +	}
> +	return 0;
> +}
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 91a018f..0633e83 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -22,6 +22,9 @@
>   
>   #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
>   
> +/* number of blocks for calculating checksum */
> +#define EROFS_CKSUM_BLOCKS	1
> +
>   static void usage(void)
>   {
>   	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
> @@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
>   				return -EINVAL;
>   			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
>   		}
> +
> +		if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
> +			sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
> +		}
>   	}
>   	return 0;
>   }
> @@ -180,6 +187,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>   		.meta_blkaddr  = sbi.meta_blkaddr,
>   		.xattr_blkaddr = 0,
>   		.feature_incompat = cpu_to_le32(sbi.feature_incompat),
> +		.checksum = 0,
> +		.chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
>   	};
>   	const unsigned int sb_blksize =
>   		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
> @@ -202,6 +211,64 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
>   	return 0;
>   }
>   
> +#define CRCPOLY	0x82F63B78
> +static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
> +{
> +	int i;
> +	u32 crc = seed;
> +
> +	while (len--) {
> +		crc ^= *in++;
> +		for (i = 0; i < 8; i++) {
> +			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
> +		}
> +	}
> +	return crc;
> +}
> +
> +/* calculate checksum for first n blocks */
> +u32 erofs_calc_blk_checksum(erofs_blk_t nblks, u32 *crc)
> +{
> +	char *buf;
> +	int err = 0;
> +
> +	buf = malloc(nblks * EROFS_BLKSIZ);
> +	err = blk_read(buf, 0, nblks);
> +	if (err) {
> +		return err;
> +	}
> +	*crc = crc32c(0, (const unsigned char *)buf, nblks * EROFS_BLKSIZ);
> +	free(buf);
> +	return 0;
> +}
> +
> +void erofs_write_sb_checksum()
> +{
> +	struct erofs_super_block *sb;
> +	char buf[EROFS_BLKSIZ];
> +	int ret = 0;
> +	u32 crc;
> +
> +	ret = erofs_calc_blk_checksum(EROFS_CKSUM_BLOCKS, &crc);
> +	if (ret) {
> +		return;
> +	}
> +	ret = blk_read(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +
> +	sb = (struct erofs_super_block *)((u8 *)buf + EROFS_SUPER_OFFSET);
> +	if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
> +		return;
> +	}
> +	sb->checksum = cpu_to_le32(crc);
> +	ret = blk_write(buf, 0, 1);
> +	if (ret) {
> +		return;
> +	}
> +}
> +
>   int main(int argc, char **argv)
>   {
>   	int err = 0;
> @@ -217,6 +284,7 @@ int main(int argc, char **argv)
>   
>   	cfg.c_legacy_compress = false;
>   	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
> +	sbi.feature = EROFS_FEATURE_SB_CHKSUM;
>   
>   	err = mkfs_parse_options_cfg(argc, argv);
>   	if (err) {
> @@ -301,6 +369,8 @@ int main(int argc, char **argv)
>   		err = -EIO;
>   	else
>   		err = dev_resize(nblocks);
> +	if (sbi.feature & EROFS_FEATURE_SB_CHKSUM)
> +		erofs_write_sb_checksum();
>   exit:
>   	z_erofs_compress_exit();
>   	dev_close();
> 

^ permalink raw reply	[flat|nested] 4+ messages in thread

* Re: [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks.
  2019-10-15 15:11 ` Li Guifu
@ 2019-10-15 15:18   ` Gao Xiang via Linux-erofs
  0 siblings, 0 replies; 4+ messages in thread
From: Gao Xiang via Linux-erofs @ 2019-10-15 15:18 UTC (permalink / raw)
  To: Li Guifu; +Cc: miaoxie, linux-erofs

Hi Guifu,

On Tue, Oct 15, 2019 at 11:11:43PM +0800, Li Guifu wrote:
> Hi Pratik, Gao Xiang
> 
> Can sbi.fearture merge into sbi.feature_incompat ?
> If not, it will has two feature parameter

Nope, please refer to
https://lore.kernel.org/r/20190822142142.GB2730@mit.edu/

> 
> It has the blocks field in the super block, chksum_blocks duplicates it.
> and chksum may be used to check whole erofs image file.

Nope, chksum_blocks is used for checksuming given blocks, blocks field is
designed to be used for statfs.

Thanks,
Gao Xiang

> 
> > Added code for calculating crc of erofs blocks (4K size).for now it calculates
> > checksum of first block. but can modified to calculate crc for any no. of blocks
> > 
> > 1) Added 'chksum_blocks' field to erofs_super_block
> > 2) removed unnecessary prints
> > 3) moved EROFS_FEATURE_SB_CHKSUM include/erofs_fs.h
> > 
> > Signed-off-by: Pratik Shinde <pratikshinde320@gmail.com>
> > ---
> >   include/erofs/internal.h |  1 +
> >   include/erofs/io.h       |  8 ++++++
> >   include/erofs_fs.h       |  3 ++-
> >   lib/io.c                 | 27 +++++++++++++++++++
> >   mkfs/main.c              | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
> >   5 files changed, 108 insertions(+), 1 deletion(-)
> > 
> > diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> > index 5384946..53335bc 100644
> > --- a/include/erofs/internal.h
> > +++ b/include/erofs/internal.h
> > @@ -55,6 +55,7 @@ struct erofs_sb_info {
> >   	u32 feature_incompat;
> >   	u64 build_time;
> >   	u32 build_time_nsec;
> > +	u32 feature;
> >   };
> >   /* global sbi */
> > diff --git a/include/erofs/io.h b/include/erofs/io.h
> > index 9775047..e0ca8d9 100644
> > --- a/include/erofs/io.h
> > +++ b/include/erofs/io.h
> > @@ -19,6 +19,7 @@
> >   int dev_open(const char *devname);
> >   void dev_close(void);
> >   int dev_write(const void *buf, u64 offset, size_t len);
> > +int dev_read(void *buf, u64 offset, size_t len);
> >   int dev_fillzero(u64 offset, size_t len, bool padding);
> >   int dev_fsync(void);
> >   int dev_resize(erofs_blk_t nblocks);
> > @@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
> >   			 blknr_to_addr(nblocks));
> >   }
> > +static inline int blk_read(void *buf, erofs_blk_t start,
> > +			    u32 nblocks)
> > +{
> > +	return dev_read(buf, blknr_to_addr(start),
> > +			 blknr_to_addr(nblocks));
> > +}
> > +
> >   #endif
> > diff --git a/include/erofs_fs.h b/include/erofs_fs.h
> > index f29aa25..8bd29d6 100644
> > --- a/include/erofs_fs.h
> > +++ b/include/erofs_fs.h
> > @@ -19,6 +19,7 @@
> >    */
> >   #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING	0x00000001
> >   #define EROFS_ALL_FEATURE_INCOMPAT		EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
> > +#define EROFS_FEATURE_SB_CHKSUM	0x0001
> >   /* 128-byte erofs on-disk super block */
> >   struct erofs_super_block {
> > @@ -39,7 +40,7 @@ struct erofs_super_block {
> >   	__u8 uuid[16];          /* 128-bit uuid for volume */
> >   	__u8 volume_name[16];   /* volume name */
> >   	__le32 feature_incompat;
> > -
> > +	__le32 chksum_blocks;	/* number of blocks used for checksum */
> >   	__u8 reserved2[44];
> >   };
> > diff --git a/lib/io.c b/lib/io.c
> > index 7f5f94d..52f9424 100644
> > --- a/lib/io.c
> > +++ b/lib/io.c
> > @@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
> >   	return dev_fillzero(st.st_size, length, true);
> >   }
> > +int dev_read(void *buf, u64 offset, size_t len)
> > +{
> > +	int ret;
> > +
> > +	if (cfg.c_dry_run)
> > +		return 0;
> > +
> > +	if (!buf) {
> > +		erofs_err("buf is NULL");
> > +		return -EINVAL;
> > +	}
> > +	if (offset >= erofs_devsz || len > erofs_devsz ||
> > +	    offset > erofs_devsz - len) {
> > +		erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
> > +			  "the end of device(%" PRIu64 ").",
> > +			  offset, len, erofs_devsz);
> > +		return -EINVAL;
> > +	}
> > +
> > +	ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
> > +	if (ret != (int)len) {
> > +		erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
> > +			  erofs_devname, offset, len);
> > +		return -errno;
> > +	}
> > +	return 0;
> > +}
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index 91a018f..0633e83 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -22,6 +22,9 @@
> >   #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
> > +/* number of blocks for calculating checksum */
> > +#define EROFS_CKSUM_BLOCKS	1
> > +
> >   static void usage(void)
> >   {
> >   	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
> > @@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
> >   				return -EINVAL;
> >   			cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
> >   		}
> > +
> > +		if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
> > +			sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
> > +		}
> >   	}
> >   	return 0;
> >   }
> > @@ -180,6 +187,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
> >   		.meta_blkaddr  = sbi.meta_blkaddr,
> >   		.xattr_blkaddr = 0,
> >   		.feature_incompat = cpu_to_le32(sbi.feature_incompat),
> > +		.checksum = 0,
> > +		.chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
> >   	};
> >   	const unsigned int sb_blksize =
> >   		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
> > @@ -202,6 +211,64 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
> >   	return 0;
> >   }
> > +#define CRCPOLY	0x82F63B78
> > +static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
> > +{
> > +	int i;
> > +	u32 crc = seed;
> > +
> > +	while (len--) {
> > +		crc ^= *in++;
> > +		for (i = 0; i < 8; i++) {
> > +			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
> > +		}
> > +	}
> > +	return crc;
> > +}
> > +
> > +/* calculate checksum for first n blocks */
> > +u32 erofs_calc_blk_checksum(erofs_blk_t nblks, u32 *crc)
> > +{
> > +	char *buf;
> > +	int err = 0;
> > +
> > +	buf = malloc(nblks * EROFS_BLKSIZ);
> > +	err = blk_read(buf, 0, nblks);
> > +	if (err) {
> > +		return err;
> > +	}
> > +	*crc = crc32c(0, (const unsigned char *)buf, nblks * EROFS_BLKSIZ);
> > +	free(buf);
> > +	return 0;
> > +}
> > +
> > +void erofs_write_sb_checksum()
> > +{
> > +	struct erofs_super_block *sb;
> > +	char buf[EROFS_BLKSIZ];
> > +	int ret = 0;
> > +	u32 crc;
> > +
> > +	ret = erofs_calc_blk_checksum(EROFS_CKSUM_BLOCKS, &crc);
> > +	if (ret) {
> > +		return;
> > +	}
> > +	ret = blk_read(buf, 0, 1);
> > +	if (ret) {
> > +		return;
> > +	}
> > +
> > +	sb = (struct erofs_super_block *)((u8 *)buf + EROFS_SUPER_OFFSET);
> > +	if (le32_to_cpu(sb->magic) != EROFS_SUPER_MAGIC_V1) {
> > +		return;
> > +	}
> > +	sb->checksum = cpu_to_le32(crc);
> > +	ret = blk_write(buf, 0, 1);
> > +	if (ret) {
> > +		return;
> > +	}
> > +}
> > +
> >   int main(int argc, char **argv)
> >   {
> >   	int err = 0;
> > @@ -217,6 +284,7 @@ int main(int argc, char **argv)
> >   	cfg.c_legacy_compress = false;
> >   	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
> > +	sbi.feature = EROFS_FEATURE_SB_CHKSUM;
> >   	err = mkfs_parse_options_cfg(argc, argv);
> >   	if (err) {
> > @@ -301,6 +369,8 @@ int main(int argc, char **argv)
> >   		err = -EIO;
> >   	else
> >   		err = dev_resize(nblocks);
> > +	if (sbi.feature & EROFS_FEATURE_SB_CHKSUM)
> > +		erofs_write_sb_checksum();
> >   exit:
> >   	z_erofs_compress_exit();
> >   	dev_close();
> > 

^ permalink raw reply	[flat|nested] 4+ messages in thread

end of thread, back to index

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-10-15  5:18 [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks Pratik Shinde
2019-10-15  6:16 ` Gao Xiang
2019-10-15 15:11 ` Li Guifu
2019-10-15 15:18   ` Gao Xiang via Linux-erofs

Linux-EROFS Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-erofs/0 linux-erofs/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-erofs linux-erofs/ https://lore.kernel.org/linux-erofs \
		linux-erofs@lists.ozlabs.org linux-erofs@ozlabs.org
	public-inbox-index linux-erofs

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.ozlabs.lists.linux-erofs


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git