From: David Sterba <dsterba@suse.cz>
To: Johannes Thumshirn <jthumshirn@suse.de>
Cc: David Sterba <dsterba@suse.com>,
Linux BTRFS Mailinglist <linux-btrfs@vger.kernel.org>
Subject: Re: [PATCH v4 10/12] btrfs-progs: add xxhash64 as checksum algorithm
Date: Tue, 24 Sep 2019 16:23:15 +0200 [thread overview]
Message-ID: <20190924142315.GR2751@twin.jikos.cz> (raw)
In-Reply-To: <20190903150046.14926-11-jthumshirn@suse.de>
On Tue, Sep 03, 2019 at 05:00:44PM +0200, Johannes Thumshirn wrote:
> From: David Sterba <dsterba@suse.com>
>
> Add xxhash64 as another checksumming algorithm.
>
> Signed-off-by: David Sterba <dsterba@suse.com>
> Signed-off-by: Johannes Thumshirn <jthumshirn@suse.de>
>
> ---
> Changes to v3:
> - Fix usage of is_valid_csum_type() (Nikolay)
> - Remove unrelated whitespace change (Nikolay)
>
> Changes to v2:
> - Integrated comments from Nikolay
> ---
> Makefile | 3 ++-
> cmds/inspect-dump-super.c | 24 +++++++++++++++---------
> convert/common.c | 2 +-
> convert/main.c | 2 +-
> crypto/hash.c | 16 ++++++++++++++++
> crypto/hash.h | 10 ++++++++++
> ctree.h | 14 ++++++++++----
> disk-io.c | 7 +++++--
> image/main.c | 5 +++--
> mkfs/common.c | 14 +++++++-------
> mkfs/main.c | 6 +++++-
> 11 files changed, 75 insertions(+), 28 deletions(-)
> create mode 100644 crypto/hash.c
> create mode 100644 crypto/hash.h
>
> diff --git a/Makefile b/Makefile
> index 370e0c37ff65..45530749e2b9 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -151,7 +151,8 @@ cmds_objects = cmds/subvolume.o cmds/filesystem.o cmds/device.o cmds/scrub.o \
> mkfs/common.o check/mode-common.o check/mode-lowmem.o
> libbtrfs_objects = send-stream.o send-utils.o kernel-lib/rbtree.o btrfs-list.o \
> kernel-lib/crc32c.o common/messages.o \
> - uuid-tree.o utils-lib.o common/rbtree-utils.o
> + uuid-tree.o utils-lib.o common/rbtree-utils.o \
> + crypto/hash.o crypto/xxhash.o
> libbtrfs_headers = send-stream.h send-utils.h send.h kernel-lib/rbtree.h btrfs-list.h \
> kernel-lib/crc32c.h kernel-lib/list.h kerncompat.h \
> kernel-lib/radix-tree.h kernel-lib/sizes.h kernel-lib/raid56.h \
> diff --git a/cmds/inspect-dump-super.c b/cmds/inspect-dump-super.c
> index 58bf82b0bbd3..f9f38751f429 100644
> --- a/cmds/inspect-dump-super.c
> +++ b/cmds/inspect-dump-super.c
> @@ -311,6 +311,17 @@ static void print_readable_super_flag(u64 flag)
> super_flags_num, BTRFS_SUPER_FLAG_SUPP);
> }
>
> +static bool is_valid_csum_type(u16 csum_type)
Please split generic infrastructure updates to a separate patch, and add
the next hash on top of that.
> +{
> + switch (csum_type) {
> + case BTRFS_CSUM_TYPE_CRC32:
> + case BTRFS_CSUM_TYPE_XXHASH:
> + return true;
> + default:
> + return false;
> + }
> +}
> +
> static void dump_superblock(struct btrfs_super_block *sb, int full)
> {
> int i;
> @@ -326,15 +337,11 @@ static void dump_superblock(struct btrfs_super_block *sb, int full)
> csum_type = btrfs_super_csum_type(sb);
> csum_size = BTRFS_CSUM_SIZE;
> printf("csum_type\t\t%hu (", csum_type);
> - if (csum_type >= ARRAY_SIZE(btrfs_csum_sizes)) {
> + if (!is_valid_csum_type(csum_type)) {
> printf("INVALID");
> } else {
> - if (csum_type == BTRFS_CSUM_TYPE_CRC32) {
> - printf("crc32c");
> - csum_size = btrfs_csum_sizes[csum_type];
> - } else {
> - printf("unknown");
> - }
> + printf("%s", btrfs_csums[csum_type].name);
> + csum_size = btrfs_csums[csum_type].size;
> }
> printf(")\n");
> printf("csum_size\t\t%llu\n", (unsigned long long)csum_size);
> @@ -342,8 +349,7 @@ static void dump_superblock(struct btrfs_super_block *sb, int full)
> printf("csum\t\t\t0x");
> for (i = 0, p = sb->csum; i < csum_size; i++)
> printf("%02x", p[i]);
> - if (csum_type != BTRFS_CSUM_TYPE_CRC32 ||
> - csum_size != btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32])
> + if (!is_valid_csum_type(csum_type))
> printf(" [UNKNOWN CSUM TYPE OR SIZE]");
> else if (check_csum_sblock(sb, csum_size, csum_type))
> printf(" [match]");
> diff --git a/convert/common.c b/convert/common.c
> index 2e2318a5863e..5dd1a2644bf6 100644
> --- a/convert/common.c
> +++ b/convert/common.c
> @@ -224,7 +224,7 @@ static inline int write_temp_extent_buffer(int fd, struct extent_buffer *buf,
> {
> int ret;
>
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
>
> /* Temporary extent buffer is always mapped 1:1 on disk */
> diff --git a/convert/main.c b/convert/main.c
> index 5e6b12431f59..5eb2a59fb68a 100644
> --- a/convert/main.c
> +++ b/convert/main.c
> @@ -1058,7 +1058,7 @@ static int migrate_super_block(int fd, u64 old_bytenr)
> BUG_ON(btrfs_super_bytenr(super) != old_bytenr);
> btrfs_set_super_bytenr(super, BTRFS_SUPER_INFO_OFFSET);
>
> - csum_tree_block_size(buf, btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32], 0,
> + csum_tree_block_size(buf, btrfs_csums[BTRFS_CSUM_TYPE_CRC32].size, 0,
> btrfs_super_csum_type(super));
> ret = pwrite(fd, buf->data, BTRFS_SUPER_INFO_SIZE,
> BTRFS_SUPER_INFO_OFFSET);
> diff --git a/crypto/hash.c b/crypto/hash.c
> new file mode 100644
> index 000000000000..fda7fc4e9f23
> --- /dev/null
> +++ b/crypto/hash.c
> @@ -0,0 +1,16 @@
> +#include "crypto/hash.h"
> +#include "crypto/xxhash.h"
> +
> +int hash_xxhash(const u8 *buf, size_t length, u8 *out)
> +{
> + XXH64_hash_t hash;
> +
> + hash = XXH64(buf, length, 0);
> + /* NOTE: we're not taking the canonical form here but the plain hash to
> + * be compatible with the kernel implementation!
> + */
/*
* Comment
*/
> + memcpy(out, &hash, 8);
> +
> + return 0;
> +}
> +
> diff --git a/crypto/hash.h b/crypto/hash.h
> new file mode 100644
> index 000000000000..45c1ef17bc57
> --- /dev/null
> +++ b/crypto/hash.h
> @@ -0,0 +1,10 @@
> +#ifndef CRYPTO_HASH_H
> +#define CRYPTO_HASH_H
> +
> +#include "../kerncompat.h"
> +
> +#define CRYPTO_HASH_SIZE_MAX 32
> +
> +int hash_xxhash(const u8 *buf, size_t length, u8 *out);
> +
> +#endif
> diff --git a/ctree.h b/ctree.h
> index 870d9f4948de..4ded8161d149 100644
> --- a/ctree.h
> +++ b/ctree.h
> @@ -167,10 +167,16 @@ struct btrfs_free_space_ctl;
> /* csum types */
> enum btrfs_csum_type {
> BTRFS_CSUM_TYPE_CRC32 = 0,
> + BTRFS_CSUM_TYPE_XXHASH = 1,
> };
>
> -/* four bytes for CRC32 */
> -static int btrfs_csum_sizes[] = { 4 };
> +static struct btrfs_csum {
> + u16 size;
> + const char *name;
> +} btrfs_csums[] = {
> + [BTRFS_CSUM_TYPE_CRC32] = { 4, "crc32c" },
> + [BTRFS_CSUM_TYPE_XXHASH] = { 8, "xxhash64" },
> +};
This should probably go to .c, same as in kernel so we don't have the
defintion copied in all object files that inclde ctree.h.
> #define BTRFS_EMPTY_DIR_SIZE 0
>
> @@ -2266,8 +2272,8 @@ BTRFS_SETGET_STACK_FUNCS(super_magic, struct btrfs_super_block, magic, 64);
> static inline int btrfs_super_csum_size(struct btrfs_super_block *s)
> {
> int t = btrfs_super_csum_type(s);
> - BUG_ON(t >= ARRAY_SIZE(btrfs_csum_sizes));
> - return btrfs_csum_sizes[t];
> + BUG_ON(t >= ARRAY_SIZE(btrfs_csums));
> + return btrfs_csums[t].size;
> }
>
> static inline unsigned long btrfs_leaf_data(struct extent_buffer *l)
> diff --git a/disk-io.c b/disk-io.c
> index 810c2e14294a..ce0b746f4db9 100644
> --- a/disk-io.c
> +++ b/disk-io.c
> @@ -34,6 +34,7 @@
> #include "print-tree.h"
> #include "common/rbtree-utils.h"
> #include "common/device-scan.h"
> +#include "crypto/hash.h"
>
> /* specified errno for check_tree_block */
> #define BTRFS_BAD_BYTENR (-1)
> @@ -148,6 +149,8 @@ int btrfs_csum_data(u16 csum_type, const u8 *data, u8 *out, size_t len)
> crc = crc32c(crc, data, len);
> put_unaligned_le32(~crc, out);
> return 0;
> + case BTRFS_CSUM_TYPE_XXHASH:
> + return hash_xxhash(data, len, out);
> default:
> fprintf(stderr, "ERROR: unknown csum type: %d\n", csum_type);
> ASSERT(0);
> @@ -1376,11 +1379,11 @@ int btrfs_check_super(struct btrfs_super_block *sb, unsigned sbflags)
> }
>
> csum_type = btrfs_super_csum_type(sb);
> - if (csum_type >= ARRAY_SIZE(btrfs_csum_sizes)) {
> + if (csum_type >= ARRAY_SIZE(btrfs_csums)) {
> error("unsupported checksum algorithm %u", csum_type);
> return -EIO;
> }
> - csum_size = btrfs_csum_sizes[csum_type];
> + csum_size = btrfs_csums[csum_type].size;
>
> btrfs_csum_data(csum_type, (u8 *)sb + BTRFS_CSUM_SIZE,
> result, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
> diff --git a/image/main.c b/image/main.c
> index 0c8ffede56f5..1265152cf524 100644
> --- a/image/main.c
> +++ b/image/main.c
> @@ -121,11 +121,12 @@ static struct extent_buffer *alloc_dummy_eb(u64 bytenr, u32 size);
>
> static void csum_block(u8 *buf, size_t len)
> {
> - u8 result[btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32]];
> + u16 csum_size = btrfs_csums[BTRFS_CSUM_TYPE_CRC32].size;
> + u8 result[csum_size];
> u32 crc = ~(u32)0;
> crc = crc32c(crc, buf + BTRFS_CSUM_SIZE, len - BTRFS_CSUM_SIZE);
> put_unaligned_le32(~crc, result);
> - memcpy(buf, result, btrfs_csum_sizes[BTRFS_CSUM_TYPE_CRC32]);
> + memcpy(buf, result, csum_size);
> }
>
> static int has_name(struct btrfs_key *key)
> diff --git a/mkfs/common.c b/mkfs/common.c
> index 4a417bd7a306..939be5eb2dc2 100644
> --- a/mkfs/common.c
> +++ b/mkfs/common.c
> @@ -101,7 +101,7 @@ static int btrfs_create_tree_root(int fd, struct btrfs_mkfs_config *cfg,
> }
>
> /* generate checksum */
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
>
> /* write back root tree */
> @@ -293,7 +293,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> btrfs_set_header_bytenr(buf, cfg->blocks[MKFS_EXTENT_TREE]);
> btrfs_set_header_owner(buf, BTRFS_EXTENT_TREE_OBJECTID);
> btrfs_set_header_nritems(buf, nritems);
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, cfg->nodesize, cfg->blocks[MKFS_EXTENT_TREE]);
> if (ret != cfg->nodesize) {
> @@ -382,7 +382,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> btrfs_set_header_bytenr(buf, cfg->blocks[MKFS_CHUNK_TREE]);
> btrfs_set_header_owner(buf, BTRFS_CHUNK_TREE_OBJECTID);
> btrfs_set_header_nritems(buf, nritems);
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, cfg->nodesize, cfg->blocks[MKFS_CHUNK_TREE]);
> if (ret != cfg->nodesize) {
> @@ -423,7 +423,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> btrfs_set_header_bytenr(buf, cfg->blocks[MKFS_DEV_TREE]);
> btrfs_set_header_owner(buf, BTRFS_DEV_TREE_OBJECTID);
> btrfs_set_header_nritems(buf, nritems);
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, cfg->nodesize, cfg->blocks[MKFS_DEV_TREE]);
> if (ret != cfg->nodesize) {
> @@ -437,7 +437,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> btrfs_set_header_bytenr(buf, cfg->blocks[MKFS_FS_TREE]);
> btrfs_set_header_owner(buf, BTRFS_FS_TREE_OBJECTID);
> btrfs_set_header_nritems(buf, 0);
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, cfg->nodesize, cfg->blocks[MKFS_FS_TREE]);
> if (ret != cfg->nodesize) {
> @@ -450,7 +450,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> btrfs_set_header_bytenr(buf, cfg->blocks[MKFS_CSUM_TREE]);
> btrfs_set_header_owner(buf, BTRFS_CSUM_TREE_OBJECTID);
> btrfs_set_header_nritems(buf, 0);
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, cfg->nodesize, cfg->blocks[MKFS_CSUM_TREE]);
> if (ret != cfg->nodesize) {
> @@ -462,7 +462,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
> memset(buf->data, 0, BTRFS_SUPER_INFO_SIZE);
> memcpy(buf->data, &super, sizeof(super));
> buf->len = BTRFS_SUPER_INFO_SIZE;
> - csum_tree_block_size(buf, btrfs_csum_sizes[cfg->csum_type], 0,
> + csum_tree_block_size(buf, btrfs_csums[cfg->csum_type].size, 0,
> cfg->csum_type);
> ret = pwrite(fd, buf->data, BTRFS_SUPER_INFO_SIZE,
> cfg->blocks[MKFS_SUPER_BLOCK]);
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e96cbc5399a2..64806dac7706 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -391,6 +391,9 @@ static enum btrfs_csum_type parse_csum_type(const char *s)
> {
> if (strcasecmp(s, "crc32c") == 0) {
> return BTRFS_CSUM_TYPE_CRC32;
> + } else if (strcasecmp(s, "xxhash64") == 0 ||
> + strcasecmp(s, "xxhash") == 0) {
> + return BTRFS_CSUM_TYPE_XXHASH;
> } else {
> error("unknown csum type %s", s);
> exit(1);
> @@ -1376,7 +1379,8 @@ raid_groups:
> pretty_size(allocation.system));
> printf("SSD detected: %s\n", ssd ? "yes" : "no");
> btrfs_parse_features_to_string(features_buf, features);
> - printf("Incompat features: %s", features_buf);
> + printf("Incompat features: %s\n", features_buf);
> + printf("Checksum: %s", btrfs_csums[csum_type].name);
> printf("\n");
>
> list_all_devices(root);
> --
> 2.16.4
next prev parent reply other threads:[~2019-09-24 14:23 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-09-03 15:00 [PATCH v4 00/12] btrfs-progs: support xxhash64 checksums Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 01/12] btrfs-progs: don't blindly assume crc32c in csum_tree_block_size() Johannes Thumshirn
2019-09-24 12:56 ` David Sterba
2019-09-24 13:06 ` David Sterba
2019-09-03 15:00 ` [PATCH v4 02/12] btrfs-progs: cache csum_type in recover_control Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 03/12] btrfs-progs: add checksum type to checksumming functions Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 04/12] btrfs-progs: don't assume checksums are always 4 bytes Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 05/12] btrfs-progs: pass checksum type to btrfs_csum_data()/btrfs_csum_final() Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 06/12] btrfs-progs: simplify update_block_csum() in btrfs-sb-mod.c Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 07/12] btrfs-progs: update checksumming api Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 08/12] btrfs-progs: add option for checksum type to mkfs Johannes Thumshirn
2019-09-24 14:26 ` David Sterba
2019-09-24 15:34 ` Adam Borowski
2019-09-24 15:52 ` David Sterba
2019-09-03 15:00 ` [PATCH v4 09/12] btrfs-progs: add xxhash sources Johannes Thumshirn
2019-09-03 15:00 ` [PATCH v4 10/12] btrfs-progs: add xxhash64 as checksum algorithm Johannes Thumshirn
2019-09-04 8:31 ` Nikolay Borisov
2019-09-24 14:23 ` David Sterba [this message]
2019-09-03 15:00 ` [PATCH v4 11/12] btrfs-progs: move crc32c implementation to crypto/ Johannes Thumshirn
2019-09-04 8:32 ` Nikolay Borisov
2019-09-03 15:00 ` [PATCH v4 12/12] btrfs-progs: add test-case for mkfs with xxhash64 Johannes Thumshirn
2019-09-24 14:26 ` David Sterba
2019-09-24 14:34 ` [PATCH v4 00/12] btrfs-progs: support xxhash64 checksums David Sterba
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190924142315.GR2751@twin.jikos.cz \
--to=dsterba@suse.cz \
--cc=dsterba@suse.com \
--cc=jthumshirn@suse.de \
--cc=linux-btrfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).