[PATCH v4 0/8] btrfs-progs: Support in-band de-duplication

[PATCH v4 0/8] btrfs-progs: Support in-band de-duplication

[Qu Wenruo]

Preparation patchset for in-coming (aimed for 4.6) kernel in-band
de-duplication patchset.

New kernel dedup will has 2 different dedup backends and a ioctl
interface to enable/disable dedup.

The ioctl interface and on-disk format (mostly) is determined, so submit
this patchset first before de-duplication first.

The patchset can also be fetched from github:
https://github.com/adam900710/btrfs-progs.git dedup

v2:
  Better objectid/offset format for dedup hash items
  Output hash for dedup hash items
v3:
  Keep dedup tree objectid and ro_flag the same with kernal,
  since the tree objectid 11 and ro_flag 1 are all taken by space cache
  tree v2.
  Add dedup property support.
v4:
  Better debug tree output for dedup tree
  Update document to keep consist with new default values.

Qu Wenruo (7):
  btrfs-progs: Basic framework for dedup command group
  btrfs-progs: dedup: Add enable command for dedup command group
  btrfs-progs: dedup: Add disable support for inband deduplication
  btrfs-progs: dedup: Add status subcommand
  btrfs-progs: Add dedup feature for mkfs and convert
  btrfs-progs: Add show-super support for new DEDUP flag
  btrfs-progs: debug-tree: Add dedup tree support

Wang Xiaoguang (1):
  btrfs-progs: property: add a dedup property

 Documentation/Makefile.in             |   1 +
 Documentation/btrfs-dedup.asciidoc    | 124 ++++++++++++++
 Documentation/btrfs-property.asciidoc |   2 +
 Documentation/btrfs.asciidoc          |   4 +
 Documentation/mkfs.btrfs.asciidoc     |   9 +
 Makefile.in                           |   2 +-
 btrfs-convert.c                       |  19 ++-
 btrfs-debug-tree.c                    |   4 +
 btrfs-show-super.c                    |  17 ++
 btrfs.c                               |   1 +
 cmds-dedup.c                          | 304 ++++++++++++++++++++++++++++++++++
 commands.h                            |   2 +
 ctree.h                               |  42 ++++-
 dedup.h                               |  42 +++++
 ioctl.h                               |  23 +++
 mkfs.c                                |   8 +-
 print-tree.c                          |  94 +++++++++++
 props.c                               |  73 ++++++++
 utils.c                               |  47 ++++--
 utils.h                               |   7 +-
 20 files changed, 800 insertions(+), 25 deletions(-)
 create mode 100644 Documentation/btrfs-dedup.asciidoc
 create mode 100644 cmds-dedup.c
 create mode 100644 dedup.h

-- 
2.7.0

[PATCH v4 1/8] btrfs-progs: Basic framework for dedup command group

[Qu Wenruo]

Add basic ioctl header and command group framework for later use.
Alone with basic man page doc.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/Makefile.in          |  1 +
 Documentation/btrfs-dedup.asciidoc | 39 +++++++++++++++++++++++++++++++
 Documentation/btrfs.asciidoc       |  4 ++++
 Makefile.in                        |  2 +-
 btrfs.c                            |  1 +
 cmds-dedup.c                       | 48 ++++++++++++++++++++++++++++++++++++++
 commands.h                         |  2 ++
 ctree.h                            | 35 ++++++++++++++++++++++++++-
 dedup.h                            | 42 +++++++++++++++++++++++++++++++++
 ioctl.h                            | 21 +++++++++++++++++
 10 files changed, 193 insertions(+), 2 deletions(-)
 create mode 100644 Documentation/btrfs-dedup.asciidoc
 create mode 100644 cmds-dedup.c
 create mode 100644 dedup.h

diff --git a/Documentation/Makefile.in b/Documentation/Makefile.in
index f046abd..9756570 100644
--- a/Documentation/Makefile.in
+++ b/Documentation/Makefile.in
@@ -30,6 +30,7 @@ MAN8_TXT += btrfs-qgroup.asciidoc
 MAN8_TXT += btrfs-replace.asciidoc
 MAN8_TXT += btrfs-restore.asciidoc
 MAN8_TXT += btrfs-property.asciidoc
+MAN8_TXT += btrfs-dedup.asciidoc
 
 # Category 5 manual page
 MAN5_TXT += btrfs-man5.asciidoc
diff --git a/Documentation/btrfs-dedup.asciidoc b/Documentation/btrfs-dedup.asciidoc
new file mode 100644
index 0000000..917977b
--- /dev/null
+++ b/Documentation/btrfs-dedup.asciidoc
@@ -0,0 +1,39 @@
+btrfs-dedup(8)
+==============
+
+NAME
+----
+btrfs-dedup - manage in-band (write time) de-duplication of a btrfs filesystem
+
+SYNOPSIS
+--------
+*btrfs dedup* <subcommand> <args>
+
+DESCRIPTION
+-----------
+*btrfs dedup* is used to enable/disable or show current in-band de-duplication
+status of a btrfs filesystem.
+
+Kernel support for in-band de-duplication starts from 4.6.
+
+WARNING: In-band de-duplication is still an experimental feautre of btrfs,
+use with caution.
+
+SUBCOMMAND
+----------
+Nothing yet
+
+EXIT STATUS
+-----------
+*btrfs dedup* returns a zero exit status if it succeeds. Non zero is
+returned in case of failure.
+
+AVAILABILITY
+------------
+*btrfs* is part of btrfs-progs.
+Please refer to the btrfs wiki http://btrfs.wiki.kernel.org for
+further details.
+
+SEE ALSO
+--------
+`mkfs.btrfs`(8),
diff --git a/Documentation/btrfs.asciidoc b/Documentation/btrfs.asciidoc
index abf1ff8..27c8883 100644
--- a/Documentation/btrfs.asciidoc
+++ b/Documentation/btrfs.asciidoc
@@ -43,6 +43,10 @@ COMMANDS
 	Do off-line check on a btrfs filesystem. +
 	See `btrfs-check`(8) for details.
 
+*dedup*::
+	Control btrfs in-band(write time) de-duplication. +
+	See `btrfs-dedup`(8) for details.
+
 *device*::
 	Manage devices managed by btrfs, including add/delete/scan and so
 	on. +
diff --git a/Makefile.in b/Makefile.in
index 19697ff..474a67a 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -75,7 +75,7 @@ cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
 	       cmds-inspect.o cmds-balance.o cmds-send.o cmds-receive.o \
 	       cmds-quota.o cmds-qgroup.o cmds-replace.o cmds-check.o \
 	       cmds-restore.o cmds-rescue.o chunk-recover.o super-recover.o \
-	       cmds-property.o cmds-fi-usage.o
+	       cmds-property.o cmds-fi-usage.o cmds-dedup.o
 libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
 		   uuid-tree.o utils-lib.o rbtree-utils.o
 libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
diff --git a/btrfs.c b/btrfs.c
index cc70515..3775dfa 100644
--- a/btrfs.c
+++ b/btrfs.c
@@ -199,6 +199,7 @@ static const struct cmd_group btrfs_cmd_group = {
 		{ "receive", cmd_receive, cmd_receive_usage, NULL, 0 },
 		{ "quota", cmd_quota, NULL, &quota_cmd_group, 0 },
 		{ "qgroup", cmd_qgroup, NULL, &qgroup_cmd_group, 0 },
+		{ "dedup", cmd_dedup, NULL, &dedup_cmd_group, 0 },
 		{ "replace", cmd_replace, NULL, &replace_cmd_group, 0 },
 		{ "help", cmd_help, cmd_help_usage, NULL, 0 },
 		{ "version", cmd_version, cmd_version_usage, NULL, 0 },
diff --git a/cmds-dedup.c b/cmds-dedup.c
new file mode 100644
index 0000000..800df34
--- /dev/null
+++ b/cmds-dedup.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Fujitsu.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "ctree.h"
+#include "ioctl.h"
+
+#include "commands.h"
+#include "utils.h"
+#include "kerncompat.h"
+#include "dedup.h"
+
+static const char * const dedup_cmd_group_usage[] = {
+	"btrfs dedup <command> [options] <path>",
+	NULL
+};
+
+static const char dedup_cmd_group_info[] =
+"manage inband(write time) de-duplication";
+
+const struct cmd_group dedup_cmd_group = {
+	dedup_cmd_group_usage, dedup_cmd_group_info, {
+		NULL_CMD_STRUCT
+	}
+};
+
+int cmd_dedup(int argc, char **argv)
+{
+	return handle_command_group(&dedup_cmd_group, argc, argv);
+}
diff --git a/commands.h b/commands.h
index 2da093b..cff8104 100644
--- a/commands.h
+++ b/commands.h
@@ -93,6 +93,7 @@ extern const struct cmd_group inspect_cmd_group;
 extern const struct cmd_group property_cmd_group;
 extern const struct cmd_group quota_cmd_group;
 extern const struct cmd_group qgroup_cmd_group;
+extern const struct cmd_group dedup_cmd_group;
 extern const struct cmd_group replace_cmd_group;
 extern const struct cmd_group rescue_cmd_group;
 
@@ -118,6 +119,7 @@ int cmd_send(int argc, char **argv);
 int cmd_receive(int argc, char **argv);
 int cmd_quota(int argc, char **argv);
 int cmd_qgroup(int argc, char **argv);
+int cmd_dedup(int argc, char **argv);
 int cmd_replace(int argc, char **argv);
 int cmd_restore(int argc, char **argv);
 int cmd_select_super(int argc, char **argv);
diff --git a/ctree.h b/ctree.h
index 2e1e631..bbffd1f 100644
--- a/ctree.h
+++ b/ctree.h
@@ -457,6 +457,7 @@ struct btrfs_super_block {
  * ones specified below then we will fail to mount
  */
 #define BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE	(1ULL << 0)
+#define BTRFS_FEATURE_COMPAT_RO_DEDUP		(1ULL << 1)
 
 #define BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF	(1ULL << 0)
 #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL	(1ULL << 1)
@@ -483,7 +484,9 @@ struct btrfs_super_block {
 
 #define BTRFS_FEATURE_COMPAT_SUPP		0ULL
 
-#define BTRFS_FEATURE_COMPAT_RO_SUPP		0ULL
+#define BTRFS_FEATURE_COMPAT_RO_SUPP			\
+	(BTRFS_FEATURE_COMPAT_RO_DEDUP |		\
+	 BTRFS_FEATURE_COMPAT_RO_DEDUP)
 
 #define BTRFS_FEATURE_INCOMPAT_SUPP			\
 	(BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF |		\
@@ -834,6 +837,23 @@ struct btrfs_csum_item {
 	u8 csum;
 } __attribute__ ((__packed__));
 
+struct btrfs_dedup_status_item {
+	__le64 blocksize;
+	__le64 limit_nr;
+	__le16 hash_type;
+	__le16 backend;
+} __attribute__ ((__packed__));
+
+struct btrfs_dedup_hash_item {
+	/* on disk length of dedup range */
+	__le64 len;
+
+	/* Spare space */
+	u8 __unused[16];
+
+	/* Hash follows */
+} __attribute__ ((__packed__));
+
 /*
  * We don't want to overwrite 1M at the beginning of device, even though
  * there is our 1st superblock at 64k. Some possible reasons:
@@ -2072,6 +2092,19 @@ static inline unsigned long btrfs_leaf_data(struct extent_buffer *l)
 	return offsetof(struct btrfs_leaf, items);
 }
 
+/* btrfs_dedup_status */
+BTRFS_SETGET_FUNCS(dedup_status_blocksize, struct btrfs_dedup_status_item,
+		   blocksize, 64);
+BTRFS_SETGET_FUNCS(dedup_status_limit, struct btrfs_dedup_status_item,
+		   limit_nr, 64);
+BTRFS_SETGET_FUNCS(dedup_status_hash_type, struct btrfs_dedup_status_item,
+		   hash_type, 16);
+BTRFS_SETGET_FUNCS(dedup_status_backend, struct btrfs_dedup_status_item,
+		   backend, 16);
+
+/* btrfs_dedup_hash_item */
+BTRFS_SETGET_FUNCS(dedup_hash_len, struct btrfs_dedup_hash_item, len, 64);
+
 /* struct btrfs_file_extent_item */
 BTRFS_SETGET_FUNCS(file_extent_type, struct btrfs_file_extent_item, type, 8);
 BTRFS_SETGET_STACK_FUNCS(stack_file_extent_type, struct btrfs_file_extent_item, type, 8);
diff --git a/dedup.h b/dedup.h
new file mode 100644
index 0000000..58f57f7
--- /dev/null
+++ b/dedup.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2015 Fujitsu.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#ifndef __BTRFS_DEDUP__
+#define __BTRFS_DEDUP__
+
+/*
+ * Dedup storage backend
+ * On disk is persist storage but overhead is large
+ * In memory is fast but will lose all its hash on umount
+ */
+#define BTRFS_DEDUP_BACKEND_INMEMORY		0
+#define BTRFS_DEDUP_BACKEND_ONDISK		1
+#define BTRFS_DEDUP_BACKEND_LAST		2
+
+/* Dedup block size limit and default value */
+#define BTRFS_DEDUP_BLOCKSIZE_MAX	(8 * 1024 * 1024)
+#define BTRFS_DEDUP_BLOCKSIZE_MIN	(16 * 1024)
+#define BTRFS_DEDUP_BLOCKSIZE_DEFAULT	(128 * 1024)
+
+/* Default dedup limit on number of hash */
+#define BTRFS_DEDUP_LIMIT_NR_DEFAULT	(32 * 1024)
+
+/* Hash algorithm, only support SHA256 yet */
+#define BTRFS_DEDUP_HASH_SHA256		0
+
+#endif
diff --git a/ioctl.h b/ioctl.h
index 771da23..b19d4e6 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -492,6 +492,27 @@ struct btrfs_ioctl_get_dev_stats {
 	__u64 unused[128 - 2 - BTRFS_DEV_STAT_VALUES_MAX]; /* pad to 1k */
 };
 
+/*
+ * de-duplication control modes
+ * For re-config, enable will handle it
+ */
+#define BTRFS_DEDUP_CTL_ENABLE	1
+#define BTRFS_DEDUP_CTL_DISABLE 2
+#define BTRFS_DEDUP_CTL_STATUS	3
+#define BTRFS_DEDUP_CTL_LAST	4
+struct btrfs_ioctl_dedup_args {
+	__u16 cmd;		/* In: command(see above macro) */
+	__u64 blocksize;	/* In/Out: For enable/status */
+	__u64 limit_nr;		/* In/Out: For enable/status */
+	__u64 limit_mem;	/* In/Out: For enable/status */
+	__u64 current_nr;	/* Out: For status output */
+	__u16 backend;		/* In/Out: For enable/status */
+	__u16 hash_type;	/* In/Out: For enable/status */
+	u8 status;		/* Out: For status output */
+	/* pad to 512 bytes */
+	u8 __unused[473];
+};
+
 /* BTRFS_IOC_SNAP_CREATE is no longer used by the btrfs command */
 #define BTRFS_QUOTA_CTL_ENABLE	1
 #define BTRFS_QUOTA_CTL_DISABLE	2
-- 
2.7.0

[PATCH v4 2/8] btrfs-progs: dedup: Add enable command for dedup command group

[Qu Wenruo]

Add enable subcommand for dedup commmand group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedup.asciidoc |  79 ++++++++++++++++++++-
 cmds-dedup.c                       | 138 +++++++++++++++++++++++++++++++++++++
 ioctl.h                            |   2 +
 3 files changed, 218 insertions(+), 1 deletion(-)

diff --git a/Documentation/btrfs-dedup.asciidoc b/Documentation/btrfs-dedup.asciidoc
index 917977b..1956436 100644
--- a/Documentation/btrfs-dedup.asciidoc
+++ b/Documentation/btrfs-dedup.asciidoc
@@ -21,7 +21,84 @@ use with caution.
 
 SUBCOMMAND
 ----------
-Nothing yet
+*enable* [options] <path>::
+Enable in-band de-duplication for a filesystem.
++
+`Options`
++
+-s|--storage-backend <BACKEND>::::
+Specify de-duplication hash storage backend.
+Supported backends are 'ondisk' and 'inmemory'.
+If not specified, default value is 'inmemory'.
++
+Refer to *BACKENDS* sector for more information.
+
+-b|--blocksize <BLOCKSIZE>::::
+Specify dedup block size.
+Supported values are power of 2 from '16K' to '8M'.
+Default value is '128K'.
++
+Refer to *BLOCKSIZE* sector for more information.
+
+-a|--hash-algorithm <HASH>::::
+Specify hash algorithm.
+Only 'sha256' is supported yet.
+
+-l|--limit-hash <LIMIT>::::
+Specify maximum number of hashes stored in memory.
+Only works for 'inmemory' backend.
+Default value is '32K' if using 'inmemory' backend.
+
+-m|--limit-memory <LIMIT>::::
+Specify maximum memory used for hashes.
+Only works for 'inmemory' backend.
+No default value.
+
+BACKENDS
+--------
+Btrfs in-band de-duplication support two different backends with their own
+features.
+
+In-memory backend::
+This backend can be used on old btrfs(without '-O dedup' mkfs option) with
+newer kernel(4.6+).
+When used on old btrfs, this backend must be re-enabled after umount.
++
+Designed for speed, in-memory backend will keep all dedup hash in memory.
+And only keeps a limit of number of hash in memory.
+Hashes over the limit will be dropped following Last-Recent-Use behavior.
+So this backend has a consistent overhead for given limit but can't ensure
+any all duplicated blocks will be de-duplicated.
++
+After umount and mount, in-memory backend need to refill its hash table.
+
+On-disk backend::
+This backend needs '-O dedup' mkfs option to enable.
++
+Designed for de-duplication rate, on-disk backend will keep dedup hash on disk.
+This behavior may cause extra disk IO for de-duplication under high memory
+pressure, but will have a much higher dedup rate.
++
+After umount and mount, on-disk backend still has its hash on disk, no need to
+refill its dedup hash table.
+
+BLOCKSIZE
+---------
+Block in-band de-duplication is done at block size unit.
+Any data smaller than dedup block size won't go through the dedup backends.
+
+Smaller block size will cause more fragments and lower performance, but a
+higher dedup rate.
+
+Larger block size will cause less fragments and higher performance, but a
+lower dedup rate.
+
+In-band de-duplication rate is highly related to the workload pattern.
+So it's highly recommended to align de-duplication blocksize to the workload
+blocksize to make full use of de-duplication.
+
+And blocksize larger than 128K will cause compression unavailable, as
+compression only support maximum extent size of 128K.
 
 EXIT STATUS
 -----------
diff --git a/cmds-dedup.c b/cmds-dedup.c
index 800df34..1da416f 100644
--- a/cmds-dedup.c
+++ b/cmds-dedup.c
@@ -19,6 +19,7 @@
 #include <getopt.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <sys/ioctl.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -36,8 +37,145 @@ static const char * const dedup_cmd_group_usage[] = {
 static const char dedup_cmd_group_info[] =
 "manage inband(write time) de-duplication";
 
+static const char * const cmd_dedup_enable_usage[] = {
+	"btrfs dedup enable [options] <path>",
+	"Enable in-band(write time) de-duplication of a btrfs.",
+	"",
+	"-s|--storage-backend <BACKEND>",
+	"           specify dedup hash storage backend",
+	"           supported backend: 'ondisk', 'inmemory'",
+	"           inmemory is the default backend",
+	"-b|--blocksize <BLOCKSIZE>",
+	"           specify dedup block size",
+	"           default value is 128K",
+	"-a|--hash-algorithm <HASH>",
+	"           specify hash algorithm",
+	"           only 'sha256' is supported yet",
+	"-l|--limit-hash <LIMIT>",
+	"           specify maximum number of hashes stored in memory",
+	"           only for 'inmemory' backend",
+	"           default value is 32K if using 'inmemory' backend",
+	"-m|--limit-mem <LIMIT>",
+	"           specify maximum memory used for hashes",
+	"           only for 'inmemory' backend",
+	"           only one of '-m' and '-l' is allowed",
+	NULL
+};
+
+static int cmd_dedup_enable(int argc, char **argv)
+{
+	int ret;
+	int fd;
+	char *path;
+	u64 blocksize = BTRFS_DEDUP_BLOCKSIZE_DEFAULT;
+	u16 hash_type = BTRFS_DEDUP_HASH_SHA256;
+	u16 backend = BTRFS_DEDUP_BACKEND_INMEMORY;
+	u64 limit_nr = 0;
+	u64 limit_mem = 0;
+	struct btrfs_ioctl_dedup_args dargs;
+	DIR *dirstream;
+
+	while (1) {
+		int c;
+		static const struct option long_options[] = {
+			{ "storage-backend", required_argument, NULL, 's'},
+			{ "blocksize", required_argument, NULL, 'b'},
+			{ "hash-algorithm", required_argument, NULL, 'a'},
+			{ "limit-hash", required_argument, NULL, 'l'},
+			{ "limit-memory", required_argument, NULL, 'm'},
+			{ NULL, 0, NULL, 0}
+		};
+
+		c = getopt_long(argc, argv, "s:b:a:l:m:", long_options, NULL);
+		if (c < 0)
+			break;
+		switch (c) {
+		case 's':
+			if (!strcmp("ondisk", optarg))
+				backend = BTRFS_DEDUP_BACKEND_ONDISK;
+			else if (!strcmp("inmemory", optarg))
+				backend = BTRFS_DEDUP_BACKEND_INMEMORY;
+			else {
+				error("unsupported dedup backend: %s", optarg);
+				exit(1);
+			}
+			break;
+		case 'b':
+			blocksize = parse_size(optarg);
+			break;
+		case 'a':
+			if (strcmp("sha256", optarg)) {
+				error("unsupported dedup hash algorithm: %s",
+				      optarg);
+				return 1;
+			}
+			break;
+		case 'l':
+			limit_nr = parse_size(optarg);
+			break;
+		case 'm':
+			limit_mem = parse_size(optarg);
+			break;
+		}
+	}
+
+	path = argv[optind];
+	if (check_argc_exact(argc - optind, 1))
+		usage(cmd_dedup_enable_usage);
+
+	/* Validation check */
+	if (!is_power_of_2(blocksize) ||
+	    blocksize > BTRFS_DEDUP_BLOCKSIZE_MAX ||
+	    blocksize < BTRFS_DEDUP_BLOCKSIZE_MIN) {
+		error("invalid dedup blocksize: %llu, not in range [%u,%u] or power of 2",
+		      blocksize, BTRFS_DEDUP_BLOCKSIZE_MIN,
+		      BTRFS_DEDUP_BLOCKSIZE_MAX);
+		return 1;
+	}
+	if ((limit_nr || limit_mem) && backend == BTRFS_DEDUP_BACKEND_ONDISK) {
+		error("limit is only valid for 'inmemory' backend");
+		return 1;
+	}
+	if (limit_nr && limit_mem) {
+		error("limit-memory and limit-hash can't be given at the same time");
+		return 1;
+	}
+
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		return 1;
+	}
+	memset(&dargs, 0, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUP_CTL_ENABLE;
+	dargs.blocksize = blocksize;
+	dargs.hash_type = hash_type;
+	dargs.limit_nr = limit_nr;
+	dargs.limit_mem = limit_mem;
+	dargs.backend = backend;
+
+	ret = ioctl(fd, BTRFS_IOC_DEDUP_CTL, &dargs);
+	if (ret < 0) {
+		char *error_message = NULL;
+		/* Special case, provide better error message */
+		if (backend == BTRFS_DEDUP_BACKEND_ONDISK &&
+		    errno == -EOPNOTSUPP)
+			error_message = "Need 'dedup' mkfs feature to enable ondisk backend";
+		error("failed to enable inband deduplication: %s",
+		      error_message ? error_message : strerror(errno));
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+
+out:
+	close_file_or_dir(fd, dirstream);
+	return ret;
+}
+
 const struct cmd_group dedup_cmd_group = {
 	dedup_cmd_group_usage, dedup_cmd_group_info, {
+		{ "enable", cmd_dedup_enable, cmd_dedup_enable_usage, NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
diff --git a/ioctl.h b/ioctl.h
index b19d4e6..4de92d5 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -724,6 +724,8 @@ static inline char *btrfs_err_str(enum btrfs_err_code err_code)
 				    struct btrfs_ioctl_dev_replace_args)
 #define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
 					 struct btrfs_ioctl_same_args)
+#define BTRFS_IOC_DEDUP_CTL	_IOWR(BTRFS_IOCTL_MAGIC, 55, \
+				      struct btrfs_ioctl_dedup_args)
 #define BTRFS_IOC_GET_FEATURES _IOR(BTRFS_IOCTL_MAGIC, 57, \
                                   struct btrfs_ioctl_feature_flags)
 #define BTRFS_IOC_SET_FEATURES _IOW(BTRFS_IOCTL_MAGIC, 57, \
-- 
2.7.0

[PATCH v4 3/8] btrfs-progs: dedup: Add disable support for inband deduplication

[Qu Wenruo]

Add disable subcommand for dedup command group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedup.asciidoc |  5 +++++
 cmds-dedup.c                       | 42 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/Documentation/btrfs-dedup.asciidoc b/Documentation/btrfs-dedup.asciidoc
index 1956436..99141cd 100644
--- a/Documentation/btrfs-dedup.asciidoc
+++ b/Documentation/btrfs-dedup.asciidoc
@@ -21,6 +21,11 @@ use with caution.
 
 SUBCOMMAND
 ----------
+*disable* <path>::
+Disable in-band de-duplication for a filesystem.
++
+This will trash all stored dedup hash.
++
 *enable* [options] <path>::
 Enable in-band de-duplication for a filesystem.
 +
diff --git a/cmds-dedup.c b/cmds-dedup.c
index 1da416f..c85bb5b 100644
--- a/cmds-dedup.c
+++ b/cmds-dedup.c
@@ -173,9 +173,51 @@ out:
 	return ret;
 }
 
+static const char * const cmd_dedup_disable_usage[] = {
+	"btrfs dedup disable <path>",
+	"Disable in-band(write time) de-duplication of a btrfs.",
+	NULL
+};
+
+static int cmd_dedup_disable(int argc, char **argv)
+{
+	struct btrfs_ioctl_dedup_args dargs;
+	DIR *dirstream;
+	char *path;
+	int fd;
+	int ret;
+
+	if (check_argc_exact(argc, 2))
+		usage(cmd_dedup_disable_usage);
+
+	path = argv[1];
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		return 1;
+	}
+	memset(&dargs, 0, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUP_CTL_DISABLE;
+
+	ret = ioctl(fd, BTRFS_IOC_DEDUP_CTL, &dargs);
+	if (ret < 0) {
+		error("failed to disable inband deduplication: %s",
+		      strerror(errno));
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+
+out:
+	close_file_or_dir(fd, dirstream);
+	return 0;
+}
+
 const struct cmd_group dedup_cmd_group = {
 	dedup_cmd_group_usage, dedup_cmd_group_info, {
 		{ "enable", cmd_dedup_enable, cmd_dedup_enable_usage, NULL, 0},
+		{ "disable", cmd_dedup_disable, cmd_dedup_disable_usage,
+		  NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
-- 
2.7.0

[PATCH v4 4/8] btrfs-progs: dedup: Add status subcommand

[Qu Wenruo]

Add status subcommand for dedup command group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedup.asciidoc |  3 ++
 cmds-dedup.c                       | 76 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/Documentation/btrfs-dedup.asciidoc b/Documentation/btrfs-dedup.asciidoc
index 99141cd..fd0550f 100644
--- a/Documentation/btrfs-dedup.asciidoc
+++ b/Documentation/btrfs-dedup.asciidoc
@@ -59,6 +59,9 @@ Specify maximum memory used for hashes.
 Only works for 'inmemory' backend.
 No default value.
 
+*status* <path>::
+Show current in-band de-duplication status of a filesystem.
+
 BACKENDS
 --------
 Btrfs in-band de-duplication support two different backends with their own
diff --git a/cmds-dedup.c b/cmds-dedup.c
index c85bb5b..64f898b 100644
--- a/cmds-dedup.c
+++ b/cmds-dedup.c
@@ -213,11 +213,87 @@ out:
 	return 0;
 }
 
+static const char * const cmd_dedup_status_usage[] = {
+	"btrfs dedup status <path>",
+	"Show current in-band(write time) de-duplication status of a btrfs.",
+	NULL
+};
+
+static int cmd_dedup_status(int argc, char **argv)
+{
+	struct btrfs_ioctl_dedup_args dargs;
+	DIR *dirstream;
+	char *path;
+	int fd;
+	int ret;
+	int print_limit = 1;
+
+	if (check_argc_exact(argc, 2))
+		usage(cmd_dedup_status_usage);
+
+	path = argv[1];
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		ret = 1;
+		goto out;
+	}
+	memset(&dargs, 0, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUP_CTL_STATUS;
+
+	ret = ioctl(fd, BTRFS_IOC_DEDUP_CTL, &dargs);
+	if (ret < 0) {
+		error("failed to get inband deduplication status: %s",
+		      strerror(errno));
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+	if (dargs.status == 0) {
+		printf("Status: \t\t\tDisabled\n");
+		goto out;
+	}
+	printf("Status:\t\t\tEnabled\n");
+
+	if (dargs.hash_type == BTRFS_DEDUP_HASH_SHA256)
+		printf("Hash algorithm:\t\tSHA-256\n");
+	else
+		printf("Hash algorithm:\t\tUnrecognized(%x)\n",
+			dargs.hash_type);
+
+	if (dargs.backend == BTRFS_DEDUP_BACKEND_INMEMORY) {
+		printf("Backend:\t\tIn-memory\n");
+		print_limit = 1;
+	} else if (dargs.backend == BTRFS_DEDUP_BACKEND_ONDISK) {
+		printf("Backend:\t\tOn-disk\n");
+		print_limit = 0;
+	} else  {
+		printf("Backend:\t\tUnrecognized(%x)\n",
+			dargs.backend);
+	}
+
+	printf("Dedup Blocksize:\t%llu\n", dargs.blocksize);
+
+	if (print_limit) {
+		printf("Number of hash: \t[%llu/%llu]\n", dargs.current_nr,
+			dargs.limit_nr);
+		printf("Memory usage: \t\t[%s/%s]\n",
+			pretty_size(dargs.current_nr *
+				(dargs.limit_mem / dargs.limit_nr)),
+			pretty_size(dargs.limit_mem));
+	}
+out:
+	close_file_or_dir(fd, dirstream);
+	return ret;
+}
+
 const struct cmd_group dedup_cmd_group = {
 	dedup_cmd_group_usage, dedup_cmd_group_info, {
 		{ "enable", cmd_dedup_enable, cmd_dedup_enable_usage, NULL, 0},
 		{ "disable", cmd_dedup_disable, cmd_dedup_disable_usage,
 		  NULL, 0},
+		{ "status", cmd_dedup_status, cmd_dedup_status_usage,
+		  NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
-- 
2.7.0

[PATCH v4 5/8] btrfs-progs: Add dedup feature for mkfs and convert

[Qu Wenruo]

Add new DEDUP ro compat flag and corresponding mkfs/convert flag
'dedup'.

Since dedup tree is completely isolated from fs tree, so even old kernel
could do read mount.
So add it to RO compat flag instead of common incompat flags

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/mkfs.btrfs.asciidoc |  9 ++++++++
 btrfs-convert.c                   | 19 +++++++++++-----
 mkfs.c                            |  8 +++++--
 utils.c                           | 47 +++++++++++++++++++++++++++++----------
 utils.h                           |  7 +++---
 5 files changed, 67 insertions(+), 23 deletions(-)

diff --git a/Documentation/mkfs.btrfs.asciidoc b/Documentation/mkfs.btrfs.asciidoc
index 0b145c7..f52690d 100644
--- a/Documentation/mkfs.btrfs.asciidoc
+++ b/Documentation/mkfs.btrfs.asciidoc
@@ -208,6 +208,15 @@ reduced-size metadata for extent references, saves a few percent of metadata
 improved representation of file extents where holes are not explicitly
 stored as an extent, saves a few percent of metadata if sparse files are used
 
+*dedup*::
+allow btrfs to use new on-disk format designed for in-band(write time)
+de-duplication.
++
+on-disk storage backend and persist de-duplication status needs this feature.
++
+this feature is RO compat feature, means old kernel can still mount it
+read-only.
+
 BLOCK GROUPS, CHUNKS, RAID
 --------------------------
 
diff --git a/btrfs-convert.c b/btrfs-convert.c
index 4baa68e..ad25065 100644
--- a/btrfs-convert.c
+++ b/btrfs-convert.c
@@ -2453,7 +2453,7 @@ static int convert_open_fs(const char *devname,
 
 static int do_convert(const char *devname, int datacsum, int packing, int noxattr,
 		u32 nodesize, int copylabel, const char *fslabel, int progress,
-		u64 features)
+		u64 features, u64 ro_features)
 {
 	int i, ret, blocks_per_node;
 	int fd = -1;
@@ -2504,8 +2504,9 @@ static int do_convert(const char *devname, int datacsum, int packing, int noxatt
 		fprintf(stderr, "unable to open %s\n", devname);
 		goto fail;
 	}
-	btrfs_parse_features_to_string(features_buf, features);
-	if (features == BTRFS_MKFS_DEFAULT_FEATURES)
+	btrfs_parse_features_to_string(features_buf, features, ro_features);
+	if (features == BTRFS_MKFS_DEFAULT_FEATURES &&
+	    ro_features == 0)
 		strcat(features_buf, " (default)");
 
 	printf("create btrfs filesystem:\n");
@@ -2521,6 +2522,7 @@ static int do_convert(const char *devname, int datacsum, int packing, int noxatt
 	mkfs_cfg.sectorsize = blocksize;
 	mkfs_cfg.stripesize = blocksize;
 	mkfs_cfg.features = features;
+	mkfs_cfg.ro_features = ro_features;
 
 	ret = make_btrfs(fd, &mkfs_cfg);
 	if (ret) {
@@ -3071,6 +3073,7 @@ int main(int argc, char *argv[])
 	char *file;
 	char fslabel[BTRFS_LABEL_SIZE];
 	u64 features = BTRFS_MKFS_DEFAULT_FEATURES;
+	u64 ro_features = 0;
 
 	while(1) {
 		enum { GETOPT_VAL_NO_PROGRESS = 256 };
@@ -3129,7 +3132,8 @@ int main(int argc, char *argv[])
 				char *orig = strdup(optarg);
 				char *tmp = orig;
 
-				tmp = btrfs_parse_fs_features(tmp, &features);
+				tmp = btrfs_parse_fs_features(tmp, &features,
+							      &ro_features);
 				if (tmp) {
 					fprintf(stderr,
 						"Unrecognized filesystem feature '%s'\n",
@@ -3147,7 +3151,9 @@ int main(int argc, char *argv[])
 					char buf[64];
 
 					btrfs_parse_features_to_string(buf,
-						features & ~BTRFS_CONVERT_ALLOWED_FEATURES);
+						features &
+						~BTRFS_CONVERT_ALLOWED_FEATURES,
+						ro_features);
 					fprintf(stderr,
 						"ERROR: features not allowed for convert: %s\n",
 						buf);
@@ -3198,7 +3204,8 @@ int main(int argc, char *argv[])
 		ret = do_rollback(file);
 	} else {
 		ret = do_convert(file, datacsum, packing, noxattr, nodesize,
-				copylabel, fslabel, progress, features);
+				copylabel, fslabel, progress, features,
+				ro_features);
 	}
 	if (ret)
 		return 1;
diff --git a/mkfs.c b/mkfs.c
index ea58404..184b9d2 100644
--- a/mkfs.c
+++ b/mkfs.c
@@ -1369,6 +1369,7 @@ int main(int ac, char **av)
 	int saved_optind;
 	char fs_uuid[BTRFS_UUID_UNPARSED_SIZE] = { 0 };
 	u64 features = BTRFS_MKFS_DEFAULT_FEATURES;
+	u64 ro_features = 0;
 	struct mkfs_allocation allocation = { 0 };
 	struct btrfs_mkfs_config mkfs_cfg;
 
@@ -1431,7 +1432,8 @@ int main(int ac, char **av)
 				char *orig = strdup(optarg);
 				char *tmp = orig;
 
-				tmp = btrfs_parse_fs_features(tmp, &features);
+				tmp = btrfs_parse_fs_features(tmp, &features,
+							      &ro_features);
 				if (tmp) {
 					fprintf(stderr,
 						"Unrecognized filesystem feature '%s'\n",
@@ -1674,6 +1676,7 @@ int main(int ac, char **av)
 	mkfs_cfg.sectorsize = sectorsize;
 	mkfs_cfg.stripesize = stripesize;
 	mkfs_cfg.features = features;
+	mkfs_cfg.ro_features = ro_features;
 
 	ret = make_btrfs(fd, &mkfs_cfg);
 	if (ret) {
@@ -1828,7 +1831,8 @@ raid_groups:
 			btrfs_group_profile_str(metadata_profile),
 			pretty_size(allocation.system));
 		printf("SSD detected:       %s\n", ssd ? "yes" : "no");
-		btrfs_parse_features_to_string(features_buf, features);
+		btrfs_parse_features_to_string(features_buf, features,
+					       ro_features);
 		printf("Incompat features:  %s", features_buf);
 		printf("\n");
 
diff --git a/utils.c b/utils.c
index 3df8b42..a1af682 100644
--- a/utils.c
+++ b/utils.c
@@ -252,6 +252,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg)
 	btrfs_set_super_chunk_root_generation(&super, 1);
 	btrfs_set_super_cache_generation(&super, -1);
 	btrfs_set_super_incompat_flags(&super, cfg->features);
+	btrfs_set_super_compat_ro_flags(&super, cfg->ro_features);
 	if (cfg->label)
 		strncpy(super.label, cfg->label, BTRFS_LABEL_SIZE - 1);
 
@@ -574,23 +575,26 @@ out:
 static const struct btrfs_fs_feature {
 	const char *name;
 	u64 flag;
+	u64 ro_flag;
 	const char *desc;
 } mkfs_features[] = {
-	{ "mixed-bg", BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS,
+	{ "mixed-bg", BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS, 0,
 		"mixed data and metadata block groups" },
-	{ "extref", BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF,
+	{ "extref", BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF, 0,
 		"increased hardlink limit per file to 65536" },
-	{ "raid56", BTRFS_FEATURE_INCOMPAT_RAID56,
+	{ "raid56", BTRFS_FEATURE_INCOMPAT_RAID56, 0,
 		"raid56 extended format" },
-	{ "skinny-metadata", BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA,
+	{ "skinny-metadata", BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA, 0,
 		"reduced-size metadata extent refs" },
-	{ "no-holes", BTRFS_FEATURE_INCOMPAT_NO_HOLES,
+	{ "no-holes", BTRFS_FEATURE_INCOMPAT_NO_HOLES, 0,
 		"no explicit hole extents for files" },
+	{ "dedup", 0, BTRFS_FEATURE_COMPAT_RO_DEDUP,
+		"support on-disk dedup backend and persist dedup status" },
 	/* Keep this one last */
-	{ "list-all", BTRFS_FEATURE_LIST_ALL, NULL }
+	{ "list-all", BTRFS_FEATURE_LIST_ALL, 0, NULL }
 };
 
-static int parse_one_fs_feature(const char *name, u64 *flags)
+static int parse_one_fs_feature(const char *name, u64 *flags, u64 *ro_flags)
 {
 	int i;
 	int found = 0;
@@ -599,9 +603,11 @@ static int parse_one_fs_feature(const char *name, u64 *flags)
 		if (name[0] == '^' &&
 			!strcmp(mkfs_features[i].name, name + 1)) {
 			*flags &= ~ mkfs_features[i].flag;
+			*ro_flags &= ~mkfs_features[i].ro_flag;
 			found = 1;
 		} else if (!strcmp(mkfs_features[i].name, name)) {
 			*flags |= mkfs_features[i].flag;
+			*ro_flags |= mkfs_features[i].ro_flag;
 			found = 1;
 		}
 	}
@@ -609,7 +615,7 @@ static int parse_one_fs_feature(const char *name, u64 *flags)
 	return !found;
 }
 
-void btrfs_parse_features_to_string(char *buf, u64 flags)
+void btrfs_parse_features_to_string(char *buf, u64 flags, u64 ro_flags)
 {
 	int i;
 
@@ -621,10 +627,15 @@ void btrfs_parse_features_to_string(char *buf, u64 flags)
 				strcat(buf, ", ");
 			strcat(buf, mkfs_features[i].name);
 		}
+		if (ro_flags & mkfs_features[i].ro_flag) {
+			if (*buf)
+				strcat(buf, ", ");
+			strcat(buf, mkfs_features[i].name);
+		}
 	}
 }
 
-void btrfs_process_fs_features(u64 flags)
+void btrfs_process_fs_features(u64 flags, u64 ro_flags)
 {
 	int i;
 
@@ -634,6 +645,11 @@ void btrfs_process_fs_features(u64 flags)
 				mkfs_features[i].name,
 				mkfs_features[i].desc);
 		}
+		if (ro_flags & mkfs_features[i].ro_flag) {
+			printf("Turning ON RO compat features '%s': %s\n",
+				mkfs_features[i].name,
+				mkfs_features[i].desc);
+		}
 	}
 }
 
@@ -649,11 +665,18 @@ void btrfs_list_all_fs_features(u64 mask_disallowed)
 			continue;
 		if (mkfs_features[i].flag & BTRFS_MKFS_DEFAULT_FEATURES)
 			is_default = ", default";
-		fprintf(stderr, "%-20s- %s (0x%llx%s)\n",
+		if (mkfs_features[i].flag)
+			printf("%-20s- %s (incompat flag: 0x%llx%s)\n",
 				mkfs_features[i].name,
 				mkfs_features[i].desc,
 				mkfs_features[i].flag,
 				is_default);
+		else
+			printf("%-20s- %s (ro compat flag: 0x%llx%s)\n",
+				mkfs_features[i].name,
+				mkfs_features[i].desc,
+				mkfs_features[i].ro_flag,
+				is_default);
 	}
 }
 
@@ -661,7 +684,7 @@ void btrfs_list_all_fs_features(u64 mask_disallowed)
  * Return NULL if all features were parsed fine, otherwise return the name of
  * the first unparsed.
  */
-char* btrfs_parse_fs_features(char *namelist, u64 *flags)
+char *btrfs_parse_fs_features(char *namelist, u64 *flags, u64 *ro_flags)
 {
 	char *this_char;
 	char *save_ptr = NULL; /* Satisfy static checkers */
@@ -669,7 +692,7 @@ char* btrfs_parse_fs_features(char *namelist, u64 *flags)
 	for (this_char = strtok_r(namelist, ",", &save_ptr);
 	     this_char != NULL;
 	     this_char = strtok_r(NULL, ",", &save_ptr)) {
-		if (parse_one_fs_feature(this_char, flags))
+		if (parse_one_fs_feature(this_char, flags, ro_flags))
 			return this_char;
 	}
 
diff --git a/utils.h b/utils.h
index d53357a..4a14d25 100644
--- a/utils.h
+++ b/utils.h
@@ -102,9 +102,9 @@ void units_set_mode(unsigned *units, unsigned mode);
 void units_set_base(unsigned *units, unsigned base);
 
 void btrfs_list_all_fs_features(u64 mask_disallowed);
-char* btrfs_parse_fs_features(char *namelist, u64 *flags);
-void btrfs_process_fs_features(u64 flags);
-void btrfs_parse_features_to_string(char *buf, u64 flags);
+char *btrfs_parse_fs_features(char *namelist, u64 *flags, u64 *ro_flags);
+void btrfs_process_fs_features(u64 flags, u64 ro_flags);
+void btrfs_parse_features_to_string(char *buf, u64 flags, u64 ro_flags);
 
 struct btrfs_mkfs_config {
 	char *label;
@@ -119,6 +119,7 @@ struct btrfs_mkfs_config {
 
 	/* Super bytenr after make_btrfs */
 	u64 super_bytenr;
+	u64 ro_features;
 };
 
 int make_btrfs(int fd, struct btrfs_mkfs_config *cfg);
-- 
2.7.0

[PATCH v4 6/8] btrfs-progs: Add show-super support for new DEDUP flag

[Qu Wenruo]

Now btrfs-show-super can handle DEDUP ro compat flag.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 btrfs-show-super.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/btrfs-show-super.c b/btrfs-show-super.c
index 051bd11..0bc0b1f 100644
--- a/btrfs-show-super.c
+++ b/btrfs-show-super.c
@@ -328,6 +328,15 @@ struct readable_flag_entry {
 	u64 bit;
 	char *output;
 };
+#define DEF_RO_COMPAT_FLAG_ENTRY(bit_name)		\
+	{BTRFS_FEATURE_COMPAT_RO_##bit_name, #bit_name}
+
+struct readable_flag_entry ro_compat_flags_array[] = {
+	DEF_RO_COMPAT_FLAG_ENTRY(DEDUP)
+};
+
+static const int ro_compat_flags_num = sizeof(ro_compat_flags_array) /
+				       sizeof(struct readable_flag_entry);
 
 #define DEF_INCOMPAT_FLAG_ENTRY(bit_name)		\
 	{BTRFS_FEATURE_INCOMPAT_##bit_name, #bit_name}
@@ -400,6 +409,13 @@ static void __print_readable_flag(u64 flag, struct readable_flag_entry *array,
 	printf(")\n");
 }
 
+static void print_readable_ro_compat_flag(u64 ro_flag)
+{
+	return __print_readable_flag(ro_flag, ro_compat_flags_array,
+				     ro_compat_flags_num,
+				     BTRFS_FEATURE_COMPAT_RO_SUPP);
+}
+
 static void print_readable_incompat_flag(u64 flag)
 {
 	return __print_readable_flag(flag, incompat_flags_array,
@@ -491,6 +507,7 @@ static void dump_superblock(struct btrfs_super_block *sb, int full)
 	       (unsigned long long)btrfs_super_compat_flags(sb));
 	printf("compat_ro_flags\t\t0x%llx\n",
 	       (unsigned long long)btrfs_super_compat_ro_flags(sb));
+	print_readable_ro_compat_flag(btrfs_super_compat_ro_flags(sb));
 	printf("incompat_flags\t\t0x%llx\n",
 	       (unsigned long long)btrfs_super_incompat_flags(sb));
 	print_readable_incompat_flag(btrfs_super_incompat_flags(sb));
-- 
2.7.0

[PATCH v4 7/8] btrfs-progs: debug-tree: Add dedup tree support

[Qu Wenruo]

Add dedup tree support for btrfs-debug-tree.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
v2:
  Add support to print hex objectid/offset for dedup hash.
  Add support to print hex hash.
---
 btrfs-debug-tree.c |  4 +++
 ctree.h            |  7 ++++
 print-tree.c       | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 105 insertions(+)

diff --git a/btrfs-debug-tree.c b/btrfs-debug-tree.c
index 266176f..ce8f76c 100644
--- a/btrfs-debug-tree.c
+++ b/btrfs-debug-tree.c
@@ -408,6 +408,10 @@ again:
 					printf("multiple");
 				}
 				break;
+			case BTRFS_DEDUP_TREE_OBJECTID:
+				if (!skip)
+					printf("dedup");
+				break;
 			default:
 				if (!skip) {
 					printf("file");
diff --git a/ctree.h b/ctree.h
index bbffd1f..1ffe084 100644
--- a/ctree.h
+++ b/ctree.h
@@ -79,6 +79,9 @@ struct btrfs_free_space_ctl;
 /* tracks free space in block groups. */
 #define BTRFS_FREE_SPACE_TREE_OBJECTID 10ULL
 
+/* on-disk dedup tree (EXPERIMENTAL) */
+#define BTRFS_DEDUP_TREE_OBJECTID 11ULL
+
 /* for storing balance parameters in the root tree */
 #define BTRFS_BALANCE_OBJECTID -4ULL
 
@@ -1216,6 +1219,10 @@ struct btrfs_root {
 #define BTRFS_DEV_ITEM_KEY	216
 #define BTRFS_CHUNK_ITEM_KEY	228
 
+#define BTRFS_DEDUP_STATUS_ITEM_KEY	230
+#define BTRFS_DEDUP_HASH_ITEM_KEY	231
+#define BTRFS_DEDUP_BYTENR_ITEM_KEY	232
+
 #define BTRFS_BALANCE_ITEM_KEY	248
 
 /*
diff --git a/print-tree.c b/print-tree.c
index 6704ff6..53a6813 100644
--- a/print-tree.c
+++ b/print-tree.c
@@ -25,6 +25,7 @@
 #include "disk-io.h"
 #include "print-tree.h"
 #include "utils.h"
+#include "dedup.h"
 
 
 static void print_dir_item_type(struct extent_buffer *eb,
@@ -667,11 +668,31 @@ static void print_key_type(u64 objectid, u8 type)
 	case BTRFS_UUID_KEY_RECEIVED_SUBVOL:
 		printf("UUID_KEY_RECEIVED_SUBVOL");
 		break;
+	case BTRFS_DEDUP_STATUS_ITEM_KEY:
+		printf("DEDUP_STATUS_ITEM");
+		break;
+	case BTRFS_DEDUP_HASH_ITEM_KEY:
+		printf("DEDUP_HASH_ITEM");
+		break;
+	case BTRFS_DEDUP_BYTENR_ITEM_KEY:
+		printf("DEDUP_BYTENR_ITEM");
+		break;
 	default:
 		printf("UNKNOWN.%d", type);
 	};
 }
 
+static void print_64bit_hash(u64 hash)
+{
+	int i;
+	unsigned char buf[8];
+
+	memcpy(buf, &hash, 8);
+	printf("0x");
+	for (i = 0; i < 8; i++)
+		printf("%02x", buf[i]);
+}
+
 static void print_objectid(u64 objectid, u8 type)
 {
 	switch (type) {
@@ -686,6 +707,9 @@ static void print_objectid(u64 objectid, u8 type)
 	case BTRFS_UUID_KEY_RECEIVED_SUBVOL:
 		printf("0x%016llx", (unsigned long long)objectid);
 		return;
+	case BTRFS_DEDUP_HASH_ITEM_KEY:
+		print_64bit_hash(objectid);
+		return;
 	}
 
 	switch (objectid) {
@@ -752,6 +776,9 @@ static void print_objectid(u64 objectid, u8 type)
 	case BTRFS_MULTIPLE_OBJECTIDS:
 		printf("MULTIPLE");
 		break;
+	case BTRFS_DEDUP_TREE_OBJECTID:
+		printf("DEDUP_TREE");
+		break;
 	case (u64)-1:
 		printf("-1");
 		break;
@@ -787,6 +814,9 @@ void btrfs_print_key(struct btrfs_disk_key *disk_key)
 	case BTRFS_UUID_KEY_RECEIVED_SUBVOL:
 		printf(" 0x%016llx)", (unsigned long long)offset);
 		break;
+	case BTRFS_DEDUP_BYTENR_ITEM_KEY:
+		print_64bit_hash(offset);
+		break;
 	default:
 		if (offset == (u64)-1)
 			printf(" -1)");
@@ -815,6 +845,54 @@ static void print_uuid_item(struct extent_buffer *l, unsigned long offset,
 	}
 }
 
+static void print_dedup_status(struct extent_buffer *node, int slot)
+{
+	struct btrfs_dedup_status_item *status_item;
+	u64 blocksize;
+	u64 limit;
+	u16 hash_type;
+	u16 backend;
+
+	status_item = btrfs_item_ptr(node, slot,
+			struct btrfs_dedup_status_item);
+	blocksize = btrfs_dedup_status_blocksize(node, status_item);
+	limit = btrfs_dedup_status_limit(node, status_item);
+	hash_type = btrfs_dedup_status_hash_type(node, status_item);
+	backend = btrfs_dedup_status_backend(node, status_item);
+
+	printf("\t\tdedup status item ");
+	if (backend == BTRFS_DEDUP_BACKEND_INMEMORY)
+		printf("backend: inmemory\n");
+	else if (backend == BTRFS_DEDUP_BACKEND_ONDISK)
+		printf("backend: ondisk\n");
+	else
+		printf("backend: Unrecognized(%u)\n", backend);
+
+	if (hash_type == BTRFS_DEDUP_HASH_SHA256)
+		printf("\t\thash algorithm: SHA-256 ");
+	else
+		printf("\t\thash algorithm: Unrecognized(%u) ", hash_type);
+
+	printf("blocksize: %llu limit: %llu\n", blocksize, limit);
+}
+
+static void print_dedup_hash(struct extent_buffer *eb, unsigned long offset)
+{
+	u8 buf[32];
+	int i;
+
+	printf("\t\thash: ");
+	read_extent_buffer(eb, buf, offset, 32);
+	for (i = 0; i < 32; i++) {
+		if (i == 16)
+			printf("\n\t\t      ");
+		if (i == 8 || i == 24)
+			printf("-");
+		printf("%02x", buf[i]);
+	}
+	printf("\n");
+}
+
 void btrfs_print_leaf(struct btrfs_root *root, struct extent_buffer *l)
 {
 	int i;
@@ -836,6 +914,8 @@ void btrfs_print_leaf(struct btrfs_root *root, struct extent_buffer *l)
 	struct btrfs_qgroup_info_item *qg_info;
 	struct btrfs_qgroup_limit_item *qg_limit;
 	struct btrfs_qgroup_status_item *qg_status;
+	struct btrfs_dedup_hash_item *hash_item;
+
 	u32 nr = btrfs_header_nritems(l);
 	u64 objectid;
 	u32 type;
@@ -1070,6 +1150,20 @@ void btrfs_print_leaf(struct btrfs_root *root, struct extent_buffer *l)
 		case BTRFS_DEV_STATS_KEY:
 			printf("\t\tdevice stats\n");
 			break;
+		case BTRFS_DEDUP_STATUS_ITEM_KEY:
+			print_dedup_status(l, i);
+			break;
+		case BTRFS_DEDUP_HASH_ITEM_KEY:
+			hash_item = btrfs_item_ptr(l, i,
+					struct btrfs_dedup_hash_item);
+
+			printf("\t\tdedup hash item num_bytes: %llu\n",
+				btrfs_dedup_hash_len(l, hash_item));
+			print_dedup_hash(l, (unsigned long)(hash_item + 1));
+			break;
+		case BTRFS_DEDUP_BYTENR_ITEM_KEY:
+			print_dedup_hash(l, btrfs_item_ptr_offset(l, i));
+			break;
 		};
 		fflush(stdout);
 	}
-- 
2.7.0

[PATCH v4 8/8] btrfs-progs: property: add a dedup property

[Qu Wenruo]

From: Wang Xiaoguang <wangxg.fnst@cn.fujitsu.com>

Normally if we enable online dedup for a fs, it's filesystem wide
de-duplication. With this property, we can explicitly disable data
de-duplication for specified files.

Signed-off-by: Wang Xiaoguang <wangxg.fnst@cn.fujitsu.com>
---
 Documentation/btrfs-property.asciidoc |  2 +
 props.c                               | 73 +++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)

diff --git a/Documentation/btrfs-property.asciidoc b/Documentation/btrfs-property.asciidoc
index 8b9b7f0..2fbecf6 100644
--- a/Documentation/btrfs-property.asciidoc
+++ b/Documentation/btrfs-property.asciidoc
@@ -44,6 +44,8 @@ label::::
 label of device
 compression::::
 compression setting for an inode: lzo, zlib, or "" (empty string)
+dedup::::
+online dedup setting for an inode: disable or "" (empty string)
 
 *list* [-t <type>] <object>::
 Lists available properties with their descriptions for the given object.
diff --git a/props.c b/props.c
index 5b74932..f27eb8b 100644
--- a/props.c
+++ b/props.c
@@ -187,6 +187,77 @@ out:
 	return ret;
 }
 
+static int prop_dedup(enum prop_object_type type, const char *object,
+	const char *name, const char *value)
+{
+	int ret;
+	ssize_t sret;
+	int fd = -1;
+	DIR *dirstream = NULL;
+	char *buf = NULL;
+	char *xattr_name = NULL;
+	int open_flags = value ? O_RDWR : O_RDONLY;
+
+	fd = open_file_or_dir3(object, &dirstream, open_flags);
+	if (fd == -1) {
+		ret = -errno;
+		fprintf(stderr, "ERROR: open %s failed. %s\n",
+			object, strerror(-ret));
+		goto out;
+	}
+
+	xattr_name = malloc(XATTR_BTRFS_PREFIX_LEN + strlen(name) + 1);
+	if (!xattr_name) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	memcpy(xattr_name, XATTR_BTRFS_PREFIX, XATTR_BTRFS_PREFIX_LEN);
+	memcpy(xattr_name + XATTR_BTRFS_PREFIX_LEN, name, strlen(name));
+	xattr_name[XATTR_BTRFS_PREFIX_LEN + strlen(name)] = '\0';
+
+	if (value)
+		sret = fsetxattr(fd, xattr_name, value, strlen(value), 0);
+	else
+		sret = fgetxattr(fd, xattr_name, NULL, 0);
+	if (sret < 0) {
+		ret = -errno;
+		if (ret != -ENOATTR)
+			fprintf(stderr,
+				"ERROR: failed to %s dedup for %s. %s\n",
+				value ? "set" : "get", object, strerror(-ret));
+		else
+			ret = 0;
+		goto out;
+	}
+	if (!value) {
+		size_t len = sret;
+
+		buf = malloc(len);
+		if (!buf) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		sret = fgetxattr(fd, xattr_name, buf, len);
+		if (sret < 0) {
+			ret = -errno;
+			fprintf(stderr,
+				"ERROR: failed to get dedup for %s. %s\n",
+				object, strerror(-ret));
+			goto out;
+		}
+		fprintf(stdout, "dedup=%.*s\n", (int)len, buf);
+	}
+
+	ret = 0;
+out:
+	free(xattr_name);
+	free(buf);
+	if (fd >= 0)
+		close_file_or_dir(fd, dirstream);
+
+	return ret;
+}
+
 const struct prop_handler prop_handlers[] = {
 	{"ro", "Set/get read-only flag of subvolume.", 0, prop_object_subvol,
 	 prop_read_only},
@@ -194,5 +265,7 @@ const struct prop_handler prop_handlers[] = {
 	 prop_object_dev | prop_object_root, prop_label},
 	{"compression", "Set/get compression for a file or directory", 0,
 	 prop_object_inode, prop_compression},
+	{"dedup", "Set/get dedup for a file or directory", 0,
+	 prop_object_inode, prop_dedup},
 	{NULL, NULL, 0, 0, NULL}
 };
-- 
2.7.0