[PATCH V9 0/5] In-band de-duplication for btrfs-progs

[PATCH V9 0/5] In-band de-duplication for btrfs-progs

[Qu Wenruo]

Patchset can be fetched from github:
https://github.com/adam900710/btrfs-progs.git dedupe_20160630

Inband dedupe(in-memory backend only) ioctl support for btrfs-progs.

User/reviewer/tester can still use previous btrfs-progs patchset to test,
this update is just cleanuping unsupported functions, like on-disk
backend and any on-disk format change.

v7 changes:
   Update ctree.h to follow kernel structure change
   Update print-tree to follow kernel structure change
V8 changes:
   Move dedup props and on-disk backend support out of the patchset
   Change command group name to "dedupe-inband", to avoid confusion with
   possible out-of-band dedupe. Suggested by Mark.
   Rebase to latest devel branch.
V9 changes:
   Follow kernels ioctl change to support FORCE flag, new reconf ioctl,
   and more precious error reporting.

Qu Wenruo (5):
  btrfs-progs: Basic framework for dedupe-inband command group
  btrfs-progs: dedupe: Add enable command for dedupe command group
  btrfs-progs: dedupe: Add disable support for inband dedupelication
  btrfs-progs: dedupe: Add status subcommand
  btrfs-progs: dedupe: introduce reconfigure subcommand

 Documentation/Makefile.in                  |   1 +
 Documentation/btrfs-dedupe-inband.asciidoc | 167 +++++++++++
 Documentation/btrfs.asciidoc               |   4 +
 Makefile.in                                |   3 +-
 btrfs-completion                           |   6 +-
 btrfs.c                                    |   2 +
 cmds-dedupe-ib.c                           | 435 +++++++++++++++++++++++++++++
 commands.h                                 |   2 +
 dedupe-ib.h                                |  41 +++
 ioctl.h                                    |  37 +++
 10 files changed, 696 insertions(+), 2 deletions(-)
 create mode 100644 Documentation/btrfs-dedupe-inband.asciidoc
 create mode 100644 cmds-dedupe-ib.c
 create mode 100644 dedupe-ib.h

-- 
2.9.0

[PATCH v9 1/5] btrfs-progs: Basic framework for dedupe-inband command group

[Qu Wenruo]

Add basic ioctl header and command group framework for later use.
Alone with basic man page doc.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/Makefile.in                  |  1 +
 Documentation/btrfs-dedupe-inband.asciidoc | 40 +++++++++++++++++++++++++
 Documentation/btrfs.asciidoc               |  4 +++
 Makefile.in                                |  3 +-
 btrfs.c                                    |  2 ++
 cmds-dedupe-ib.c                           | 48 ++++++++++++++++++++++++++++++
 commands.h                                 |  2 ++
 dedupe-ib.h                                | 41 +++++++++++++++++++++++++
 ioctl.h                                    | 35 ++++++++++++++++++++++
 9 files changed, 175 insertions(+), 1 deletion(-)
 create mode 100644 Documentation/btrfs-dedupe-inband.asciidoc
 create mode 100644 cmds-dedupe-ib.c
 create mode 100644 dedupe-ib.h

diff --git a/Documentation/Makefile.in b/Documentation/Makefile.in
index aea2cb4..7db4164 100644
--- a/Documentation/Makefile.in
+++ b/Documentation/Makefile.in
@@ -28,6 +28,7 @@ MAN8_TXT += btrfs-qgroup.asciidoc
 MAN8_TXT += btrfs-replace.asciidoc
 MAN8_TXT += btrfs-restore.asciidoc
 MAN8_TXT += btrfs-property.asciidoc
+MAN8_TXT += btrfs-dedupe-inband.asciidoc
 
 # Category 5 manual page
 MAN5_TXT += btrfs-man5.asciidoc
diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
new file mode 100644
index 0000000..9ee2bc7
--- /dev/null
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -0,0 +1,40 @@
+btrfs-dedupe(8)
+==============
+
+NAME
+----
+btrfs-dedupe-inband - manage in-band (write time) de-duplication of a btrfs
+filesystem
+
+SYNOPSIS
+--------
+*btrfs dedupe-inband* <subcommand> <args>
+
+DESCRIPTION
+-----------
+*btrfs dedupe-inband* is used to enable/disable or show current in-band de-duplication
+status of a btrfs filesystem.
+
+Kernel support for in-band de-duplication starts from 4.8.
+
+WARNING: In-band de-duplication is still an experimental feautre of btrfs,
+use with caution.
+
+SUBCOMMAND
+----------
+Nothing yet
+
+EXIT STATUS
+-----------
+*btrfs dedupe-inband* returns a zero exit status if it succeeds. Non zero is
+returned in case of failure.
+
+AVAILABILITY
+------------
+*btrfs* is part of btrfs-progs.
+Please refer to the btrfs wiki http://btrfs.wiki.kernel.org for
+further details.
+
+SEE ALSO
+--------
+`mkfs.btrfs`(8),
diff --git a/Documentation/btrfs.asciidoc b/Documentation/btrfs.asciidoc
index 6a77a85..8ded842 100644
--- a/Documentation/btrfs.asciidoc
+++ b/Documentation/btrfs.asciidoc
@@ -43,6 +43,10 @@ COMMANDS
 	Do off-line check on a btrfs filesystem. +
 	See `btrfs-check`(8) for details.
 
+*dedupe*::
+	Control btrfs in-band(write time) de-duplication. +
+	See `btrfs-dedupe`(8) for details.
+
 *device*::
 	Manage devices managed by btrfs, including add/delete/scan and so
 	on. +
diff --git a/Makefile.in b/Makefile.in
index ac6b353..5cf0fc2 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -86,7 +86,8 @@ cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
 	       cmds-quota.o cmds-qgroup.o cmds-replace.o cmds-check.o \
 	       cmds-restore.o cmds-rescue.o chunk-recover.o super-recover.o \
 	       cmds-property.o cmds-fi-usage.o cmds-inspect-dump-tree.o \
-	       cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o
+	       cmds-inspect-dump-super.o cmds-inspect-tree-stats.o cmds-fi-du.o \
+	       cmds-dedupe-ib.o
 libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
 		   uuid-tree.o utils-lib.o rbtree-utils.o
 libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
diff --git a/btrfs.c b/btrfs.c
index cc70515..06c211b 100644
--- a/btrfs.c
+++ b/btrfs.c
@@ -199,6 +199,8 @@ static const struct cmd_group btrfs_cmd_group = {
 		{ "receive", cmd_receive, cmd_receive_usage, NULL, 0 },
 		{ "quota", cmd_quota, NULL, &quota_cmd_group, 0 },
 		{ "qgroup", cmd_qgroup, NULL, &qgroup_cmd_group, 0 },
+		{ "dedupe-inband", cmd_dedupe_ib, NULL, &dedupe_ib_cmd_group,
+			0 },
 		{ "replace", cmd_replace, NULL, &replace_cmd_group, 0 },
 		{ "help", cmd_help, cmd_help_usage, NULL, 0 },
 		{ "version", cmd_version, cmd_version_usage, NULL, 0 },
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
new file mode 100644
index 0000000..160c8ec
--- /dev/null
+++ b/cmds-dedupe-ib.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Fujitsu.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "ctree.h"
+#include "ioctl.h"
+
+#include "commands.h"
+#include "utils.h"
+#include "kerncompat.h"
+#include "dedupe-ib.h"
+
+static const char * const dedupe_ib_cmd_group_usage[] = {
+	"btrfs dedupe-inband <command> [options] <path>",
+	NULL
+};
+
+static const char dedupe_ib_cmd_group_info[] =
+"manage inband(write time) de-duplication";
+
+const struct cmd_group dedupe_ib_cmd_group = {
+	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
+		NULL_CMD_STRUCT
+	}
+};
+
+int cmd_dedupe_ib(int argc, char **argv)
+{
+	return handle_command_group(&dedupe_ib_cmd_group, argc, argv);
+}
diff --git a/commands.h b/commands.h
index 94229c1..2002c45 100644
--- a/commands.h
+++ b/commands.h
@@ -93,6 +93,7 @@ extern const struct cmd_group inspect_cmd_group;
 extern const struct cmd_group property_cmd_group;
 extern const struct cmd_group quota_cmd_group;
 extern const struct cmd_group qgroup_cmd_group;
+extern const struct cmd_group dedupe_ib_cmd_group;
 extern const struct cmd_group replace_cmd_group;
 extern const struct cmd_group rescue_cmd_group;
 
@@ -118,6 +119,7 @@ int cmd_send(int argc, char **argv);
 int cmd_receive(int argc, char **argv);
 int cmd_quota(int argc, char **argv);
 int cmd_qgroup(int argc, char **argv);
+int cmd_dedupe_ib(int argc, char **argv);
 int cmd_replace(int argc, char **argv);
 int cmd_restore(int argc, char **argv);
 int cmd_select_super(int argc, char **argv);
diff --git a/dedupe-ib.h b/dedupe-ib.h
new file mode 100644
index 0000000..02c5f8f
--- /dev/null
+++ b/dedupe-ib.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2015 Fujitsu.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#ifndef __BTRFS_DEDUPE__
+#define __BTRFS_DEDUPE__
+
+/*
+ * Dedup storage backend
+ * On disk is persist storage but overhead is large
+ * In memory is fast but will lose all its hash on umount
+ */
+#define BTRFS_DEDUPE_BACKEND_INMEMORY		0
+#define BTRFS_DEDUPE_BACKEND_LAST		1
+
+/* Dedup block size limit and default value */
+#define BTRFS_DEDUPE_BLOCKSIZE_MAX	(8 * 1024 * 1024)
+#define BTRFS_DEDUPE_BLOCKSIZE_MIN	(16 * 1024)
+#define BTRFS_DEDUPE_BLOCKSIZE_DEFAULT	(128 * 1024)
+
+/* Default dedupe limit on number of hash */
+#define BTRFS_DEDUPE_LIMIT_NR_DEFAULT	(32 * 1024)
+
+/* Hash algorithm, only support SHA256 yet */
+#define BTRFS_DEDUPE_HASH_SHA256		0
+
+#endif
diff --git a/ioctl.h b/ioctl.h
index 5f18bcb..e698ef7 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -503,6 +503,41 @@ struct btrfs_ioctl_get_dev_stats {
 	__u64 unused[128 - 2 - BTRFS_DEV_STAT_VALUES_MAX]; /* pad to 1k */
 };
 
+/*
+ * de-duplication control modes
+ * For re-config, enable will handle it
+ */
+#define BTRFS_DEDUPE_CTL_ENABLE	1
+#define BTRFS_DEDUPE_CTL_DISABLE 2
+#define BTRFS_DEDUPE_CTL_STATUS	3
+#define BTRFS_DEDUPE_CTL_RECONF	4
+#define BTRFS_DEDUPE_CTL_LAST	5
+
+/*
+ * Allow enable command to be executed on dedupe enabled fs.
+ * Make dedupe_enable ioctl to be stateless.
+ *
+ * Or only dedup_reconf ioctl can be executed on dedupe enabled fs
+ */
+#define BTRFS_DEDUPE_FLAG_FORCE		(1 << 0)
+/*
+ * This structure is used for dedupe enable/disable/configure
+ * and status ioctl.
+ * Reserved range should be set to 0xff.
+ */
+struct btrfs_ioctl_dedupe_args {
+	__u16 cmd;		/* In: command */
+	__u64 blocksize;	/* In/Out: blocksize */
+	__u64 limit_nr;		/* In/Out: limit nr for inmem backend */
+	__u64 limit_mem;	/* In/Out: limit mem for inmem backend */
+	__u64 current_nr;	/* Out: current hash nr */
+	__u16 backend;		/* In/Out: current backend */
+	__u16 hash_algo;	/* In/Out: hash algorithm */
+	u8 status;		/* Out: enabled or disabled */
+	u8 flags;		/* In: special flags for ioctl */
+	u8 __unused[472];	/* Pad to 512 bytes */
+};
+
 /* BTRFS_IOC_SNAP_CREATE is no longer used by the btrfs command */
 #define BTRFS_QUOTA_CTL_ENABLE	1
 #define BTRFS_QUOTA_CTL_DISABLE	2
-- 
2.9.0

[PATCH v9 2/5] btrfs-progs: dedupe: Add enable command for dedupe command group

[Qu Wenruo]

Add enable subcommand for dedupe commmand group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedupe-inband.asciidoc | 114 ++++++++++++++-
 btrfs-completion                           |   6 +-
 cmds-dedupe-ib.c                           | 222 +++++++++++++++++++++++++++++
 ioctl.h                                    |   2 +
 4 files changed, 342 insertions(+), 2 deletions(-)

diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index 9ee2bc7..82f970a 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -22,7 +22,119 @@ use with caution.
 
 SUBCOMMAND
 ----------
-Nothing yet
+*enable* [options] <path>::
+Enable in-band de-duplication for a filesystem.
++
+`Options`
++
+-f|--force::::
+Force 'enable' command to be exected.
+Will skip memory limit check and allow 'enable' to be executed even in-band
+de-duplication is already enabled.
++
+NOTE: If re-enable dedupe with '-f' option, any unspecified parameter will be
+reset to its default value.
+
+-s|--storage-backend <BACKEND>::::
+Specify de-duplication hash storage backend.
+Only 'inmemory' backend is supported yet.
+If not specified, default value is 'inmemory'.
++
+Refer to *BACKENDS* sector for more information.
+
+-b|--blocksize <BLOCKSIZE>::::
+Specify dedupe block size.
+Supported values are power of 2 from '16K' to '8M'.
+Default value is '128K'.
++
+Refer to *BLOCKSIZE* sector for more information.
+
+-a|--hash-algorithm <HASH>::::
+Specify hash algorithm.
+Only 'sha256' is supported yet.
+
+-l|--limit-hash <LIMIT>::::
+Specify maximum number of hashes stored in memory.
+Only works for 'inmemory' backend.
+Conflicts with '-m' option.
++
+Only positive values are valid.
+Default value is '32K'.
+
+-m|--limit-memory <LIMIT>::::
+Specify maximum memory used for hashes.
+Only works for 'inmemory' backend.
+Conflicts with '-l' option.
++
+Only value larger than or equal to '1024' is valid.
+No default value.
++
+NOTE: Memory limit will be rounded down to kernel internal hash size,
+so the memory limit shown in 'btrfs dedupe status' may be different
+from the <LIMIT>.
+
+WARNING: Too large value for '-l' or '-m' will easily trigger OOM.
+Please use with caution according to system memory.
+
+NOTE: In-band de-duplication is not compactible with compression yet.
+And compression has higher priority than in-band de-duplication, means if
+compression and de-duplication is enabled at the same time, only compression
+will work.
+
+BACKENDS
+--------
+Btrfs in-band de-duplication will support different storage backends, with
+different use case and features.
+
+In-memory backend::
+This backend provides backward-compatibility, and more fine-tuning options.
+But hash pool is non-persistent and may exhaust kernel memory if not setup
+properly.
++
+This backend can be used on old btrfs(without '-O dedupe' mkfs option).
+When used on old btrfs, this backend needs to be enabled manually after mount.
++
+Designed for fast hash search speed, in-memory backend will keep all dedupe
+hashes in memory. (Although overall performance is still much the same with
+'ondisk' backend if all 'ondisk' hash can be cached in memory)
++
+And only keeps limited number of hash in memory to avoid exhausting memory.
+Hashes over the limit will be dropped following Last-Recent-Use behavior.
+So this backend has a consistent overhead for given limit but can\'t ensure
+all duplicated blocks will be de-duplicated.
++
+After umount and mount, in-memory backend need to refill its hash pool.
+
+On-disk backend::
+This backend provides persistent hash pool, with more smart memory management
+for hash pool.
+But it\'s not backward-compatible, meaning it must be used with '-O dedupe' mkfs
+option and older kernel can\'t mount it read-write.
++
+Designed for de-duplication rate, hash pool is stored as btrfs B+ tree on disk.
+This behavior may cause extra disk IO for hash search under high memory
+pressure.
++
+After umount and mount, on-disk backend still has its hash on disk, no need to
+refill its dedupe hash pool.
+
+Currently, only 'inmemory' backend is supported in btrfs-progs.
+
+DEDUPE BLOCK SIZE
+----------------
+In-band de-duplication is done at dedupe block size.
+Any data smaller than dedupe block size won\'t go through in-band
+de-duplication.
+
+And dedupe block size affects dedupe rate and fragmentation heavily.
+
+Smaller block size will cause more fragments, but higher dedupe rate.
+
+Larger block size will cause less fragments, but lower dedupe rate.
+
+In-band de-duplication rate is highly related to the workload pattern.
+So it\'s highly recommended to align dedupe block size to the workload
+block size to make full use of de-duplication.
 
 EXIT STATUS
 -----------
diff --git a/btrfs-completion b/btrfs-completion
index 3ede77b..50f7ea2 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -29,7 +29,7 @@ _btrfs()
 
 	local cmd=${words[1]}
 
-    commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup replace help version'
+    commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup dedupe replace help version'
     commands_subvolume='create delete list snapshot find-new get-default set-default show sync'
     commands_filesystem='defragment sync resize show df label usage'
     commands_balance='start pause cancel resume status'
@@ -40,6 +40,7 @@ _btrfs()
     commands_property='get set list'
     commands_quota='enable disable rescan'
     commands_qgroup='assign remove create destroy show limit'
+    commands_dedupe='enable'
     commands_replace='start status cancel'
 
 	if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
@@ -94,6 +95,9 @@ _btrfs()
             qgroup)
                 opts="$commands_qgroup"
                 ;;
+            dedupe)
+                opts="$commands_dedupe"
+                ;;
             replace)
                 opts="$commands_replace"
                 ;;
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index 160c8ec..a62b9cb 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -19,6 +19,7 @@
 #include <getopt.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <sys/sysinfo.h>
 
 #include "ctree.h"
 #include "ioctl.h"
@@ -36,8 +37,229 @@ static const char * const dedupe_ib_cmd_group_usage[] = {
 static const char dedupe_ib_cmd_group_info[] =
 "manage inband(write time) de-duplication";
 
+static const char * const cmd_dedupe_ib_enable_usage[] = {
+	"btrfs dedupe-inband enable [options] <path>",
+	"Enable in-band(write time) de-duplication of a btrfs.",
+	"",
+	"-s|--storage-backend <BACKEND>",
+	"           specify dedupe hash storage backend",
+	"           supported backend: 'inmemory'",
+	"-b|--blocksize <BLOCKSIZE>",
+	"           specify dedupe block size",
+	"           default value is 128K",
+	"-a|--hash-algorithm <HASH>",
+	"           specify hash algorithm",
+	"           only 'sha256' is supported yet",
+	"-l|--limit-hash <LIMIT>",
+	"           specify maximum number of hashes stored in memory",
+	"           only for 'inmemory' backend",
+	"           positive value is valid, default value is 32K",
+	"-m|--limit-mem <LIMIT>",
+	"           specify maximum memory used for hashes",
+	"           only for 'inmemory' backend",
+	"           value larger than or equal to 1024 is valid, no default",
+	"           only one of '-m' and '-l' is allowed",
+	"-f|--force",
+	"           force enable command to be executed",
+	"           will skip some memory limit check",
+	"           also without this flag enable command is not allowed to be",
+	"           executed if dedupe is already enabled",
+	"           note: unspecified parameter will be reset to default value",
+	NULL
+};
+
+
+#define report_fatal_parameter(dargs, old, member, type, err_val, fmt)	\
+if (dargs->member != old->member && dargs->member == (type)(err_val)) {	\
+	error("unsupported dedupe "#member": %"#fmt"", old->member);	\
+	return;								\
+}
+
+#define report_option_parameter(dargs, old, member, type, err_val, fmt)	\
+if (dargs->member != old->member && dargs->member == (type)(err_val))	\
+	warning("unsupported optional "#member": %"#fmt", continue",	\
+		old->member);
+static void report_parameter_error(struct btrfs_ioctl_dedupe_args *dargs,
+				   struct btrfs_ioctl_dedupe_args *old)
+{
+	if (dargs->flags == (u8)-1) {
+		if (dargs->status == 1 &&
+		    old->cmd == BTRFS_DEDUPE_CTL_ENABLE &&
+		    !(old->flags & BTRFS_DEDUPE_FLAG_FORCE)) {
+			error("can't re-enable dedupe without --foce");
+			return;
+		}
+		report_option_parameter(dargs, old, flags, u8, -1, x);
+	}
+	report_fatal_parameter(dargs, old, cmd, u16, -1, u);
+	report_fatal_parameter(dargs, old, blocksize, u64, -1, llu);
+	report_fatal_parameter(dargs, old, backend, u16, -1, u);
+	report_fatal_parameter(dargs, old, hash_algo, u16, -1, u);
+	if (dargs->limit_nr == 0 && dargs->limit_mem == 0)
+		error("unsupported dedupe limit combination: nr: %llu, mem: %llu",
+				old->limit_nr, old->limit_mem);
+	return;
+}
+
+static int cmd_dedupe_ib_enable(int argc, char **argv)
+{
+	int ret;
+	int fd = -1;
+	char *path;
+	u64 blocksize = BTRFS_DEDUPE_BLOCKSIZE_DEFAULT;
+	u16 hash_algo = BTRFS_DEDUPE_HASH_SHA256;
+	u16 backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
+	u64 limit_nr = 0;
+	u64 limit_mem = 0;
+	u64 sys_mem = 0;
+	int force = 0;
+	struct btrfs_ioctl_dedupe_args dargs;
+	struct btrfs_ioctl_dedupe_args backup;
+	struct sysinfo info;
+	DIR *dirstream = NULL;
+
+	while (1) {
+		int c;
+		static const struct option long_options[] = {
+			{ "storage-backend", required_argument, NULL, 's'},
+			{ "blocksize", required_argument, NULL, 'b'},
+			{ "hash-algorithm", required_argument, NULL, 'a'},
+			{ "limit-hash", required_argument, NULL, 'l'},
+			{ "limit-memory", required_argument, NULL, 'm'},
+			{ "force", required_argument, NULL, 'f'},
+			{ NULL, 0, NULL, 0}
+		};
+
+		c = getopt_long(argc, argv, "s:b:a:l:m:", long_options, NULL);
+		if (c < 0)
+			break;
+		switch (c) {
+		case 's':
+			if (!strcasecmp("inmemory", optarg))
+				backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
+			else {
+				error("unsupported dedupe backend: %s", optarg);
+				exit(1);
+			}
+			break;
+		case 'b':
+			blocksize = parse_size(optarg);
+			break;
+		case 'a':
+			if (strcmp("sha256", optarg)) {
+				error("unsupported dedupe hash algorithm: %s",
+				      optarg);
+				return 1;
+			}
+			break;
+		case 'l':
+			limit_nr = parse_size(optarg);
+			if (limit_nr == 0) {
+				error("limit should be larger than 0");
+				return 1;
+			}
+			break;
+		case 'm':
+			limit_mem = parse_size(optarg);
+			/*
+			 * Make sure at least one hash is allocated
+			 * 1024 should be good enough though.
+			 */
+			if (limit_mem < 1024) {
+				error("memory limit should be larger than or equal to 1024");
+				return 1;
+			}
+			break;
+		case 'f':
+			force = 1;
+			break;
+		default:
+			usage(cmd_dedupe_ib_enable_usage);
+			return 1;
+		}
+	}
+
+	path = argv[optind];
+	if (check_argc_exact(argc - optind, 1))
+		usage(cmd_dedupe_ib_enable_usage);
+
+	/* Validation check */
+	if (!is_power_of_2(blocksize) ||
+	    blocksize > BTRFS_DEDUPE_BLOCKSIZE_MAX ||
+	    blocksize < BTRFS_DEDUPE_BLOCKSIZE_MIN) {
+		error("invalid dedupe blocksize: %llu, not in range [%u,%u] or power of 2",
+		      blocksize, BTRFS_DEDUPE_BLOCKSIZE_MIN,
+		      BTRFS_DEDUPE_BLOCKSIZE_MAX);
+		return 1;
+	}
+	if ((limit_nr || limit_mem) && backend != BTRFS_DEDUPE_BACKEND_INMEMORY) {
+		error("limit is only valid for 'inmemory' backend");
+		return 1;
+	}
+	if (limit_nr && limit_mem) {
+		error("limit-memory and limit-hash can't be given at the same time");
+		return 1;
+	}
+
+	ret = sysinfo(&info);
+	if (ret < 0)
+		warning("failed to determine system total ram size: %s",
+			strerror(errno));
+	else
+		sys_mem = info.totalram;
+
+	/*
+	 * TODO: Add check for limit_nr against current system
+	 * memory to avoid wrongly set limit.
+	 */
+	if (!force && limit_mem && sys_mem && sys_mem < limit_mem * 4)
+		goto mem_check;
+
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		return 1;
+	}
+	memset(&dargs, -1, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUPE_CTL_ENABLE;
+	dargs.blocksize = blocksize;
+	dargs.hash_algo = hash_algo;
+	dargs.limit_nr = limit_nr;
+	dargs.limit_mem = limit_mem;
+	dargs.backend = backend;
+	if (force)
+		dargs.flags |= BTRFS_DEDUPE_FLAG_FORCE;
+	else
+		dargs.flags = 0;
+
+	memcpy(&backup, &dargs, sizeof(dargs));
+	ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+	if (ret < 0) {
+		error("failed to enable inband deduplication: %s",
+		      strerror(errno));
+		report_parameter_error(&dargs, &backup);
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+mem_check:
+	if (!force && dargs.limit_mem > sys_mem / 4) {
+		ret = 1;
+		error("memory limit %llu is too large compared to system memory: %llu",
+		      limit_mem, sys_mem);
+		error("recommened memory limit is no more than %llu",
+		      sys_mem / 4);
+		error("use --force option if you know what you are doing");
+	}
+out:
+	close_file_or_dir(fd, dirstream);
+	return ret;
+}
+
 const struct cmd_group dedupe_ib_cmd_group = {
 	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
+		{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
+		  NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
diff --git a/ioctl.h b/ioctl.h
index e698ef7..adf4754 100644
--- a/ioctl.h
+++ b/ioctl.h
@@ -749,6 +749,8 @@ static inline char *btrfs_err_str(enum btrfs_err_code err_code)
 				    struct btrfs_ioctl_dev_replace_args)
 #define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
 					 struct btrfs_ioctl_same_args)
+#define BTRFS_IOC_DEDUPE_CTL	_IOWR(BTRFS_IOCTL_MAGIC, 55, \
+				      struct btrfs_ioctl_dedupe_args)
 #define BTRFS_IOC_GET_FEATURES _IOR(BTRFS_IOCTL_MAGIC, 57, \
                                   struct btrfs_ioctl_feature_flags)
 #define BTRFS_IOC_SET_FEATURES _IOW(BTRFS_IOCTL_MAGIC, 57, \
-- 
2.9.0

[PATCH v9 3/5] btrfs-progs: dedupe: Add disable support for inband dedupelication

[Qu Wenruo]

Add disable subcommand for dedupe command group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedupe-inband.asciidoc |  5 ++++
 btrfs-completion                           |  2 +-
 cmds-dedupe-ib.c                           | 42 ++++++++++++++++++++++++++++++
 3 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index 82f970a..de32eb9 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -22,6 +22,11 @@ use with caution.
 
 SUBCOMMAND
 ----------
+*disable* <path>::
+Disable in-band de-duplication for a filesystem.
++
+This will trash all stored dedupe hash.
++
 *enable* [options] <path>::
 Enable in-band de-duplication for a filesystem.
 +
diff --git a/btrfs-completion b/btrfs-completion
index 50f7ea2..9a6c73b 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -40,7 +40,7 @@ _btrfs()
     commands_property='get set list'
     commands_quota='enable disable rescan'
     commands_qgroup='assign remove create destroy show limit'
-    commands_dedupe='enable'
+    commands_dedupe='enable disable'
     commands_replace='start status cancel'
 
 	if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index a62b9cb..ca377f1 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -256,10 +256,52 @@ out:
 	return ret;
 }
 
+static const char * const cmd_dedupe_ib_disable_usage[] = {
+	"btrfs dedupe disable <path>",
+	"Disable in-band(write time) de-duplication of a btrfs.",
+	NULL
+};
+
+static int cmd_dedupe_ib_disable(int argc, char **argv)
+{
+	struct btrfs_ioctl_dedupe_args dargs;
+	DIR *dirstream;
+	char *path;
+	int fd;
+	int ret;
+
+	if (check_argc_exact(argc, 2))
+		usage(cmd_dedupe_ib_disable_usage);
+
+	path = argv[1];
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		return 1;
+	}
+	memset(&dargs, 0, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUPE_CTL_DISABLE;
+
+	ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+	if (ret < 0) {
+		error("failed to disable inband deduplication: %s",
+		      strerror(errno));
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+
+out:
+	close_file_or_dir(fd, dirstream);
+	return 0;
+}
+
 const struct cmd_group dedupe_ib_cmd_group = {
 	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
 		{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
 		  NULL, 0},
+		{ "disable", cmd_dedupe_ib_disable, cmd_dedupe_ib_disable_usage,
+		  NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
-- 
2.9.0

[PATCH v9 4/5] btrfs-progs: dedupe: Add status subcommand

[Qu Wenruo]

Add status subcommand for dedupe command group.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedupe-inband.asciidoc |  3 ++
 btrfs-completion                           |  2 +-
 cmds-dedupe-ib.c                           | 81 ++++++++++++++++++++++++++++++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index de32eb9..df068c3 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -86,6 +86,9 @@ And compression has higher priority than in-band de-duplication, means if
 compression and de-duplication is enabled at the same time, only compression
 will work.
 
+*status* <path>::
+Show current in-band de-duplication status of a filesystem.
+
 BACKENDS
 --------
 Btrfs in-band de-duplication will support different storage backends, with
diff --git a/btrfs-completion b/btrfs-completion
index 9a6c73b..fbaae0c 100644
--- a/btrfs-completion
+++ b/btrfs-completion
@@ -40,7 +40,7 @@ _btrfs()
     commands_property='get set list'
     commands_quota='enable disable rescan'
     commands_qgroup='assign remove create destroy show limit'
-    commands_dedupe='enable disable'
+    commands_dedupe='enable disable status'
     commands_replace='start status cancel'
 
 	if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index ca377f1..c9085ac 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -296,12 +296,93 @@ out:
 	return 0;
 }
 
+static const char * const cmd_dedupe_ib_status_usage[] = {
+	"btrfs dedupe status <path>",
+	"Show current in-band(write time) de-duplication status of a btrfs.",
+	NULL
+};
+
+static int cmd_dedupe_ib_status(int argc, char **argv)
+{
+	struct btrfs_ioctl_dedupe_args dargs;
+	DIR *dirstream;
+	char *path;
+	int fd;
+	int ret;
+	int print_limit = 1;
+
+	if (check_argc_exact(argc, 2))
+		usage(cmd_dedupe_ib_status_usage);
+
+	path = argv[1];
+	fd = open_file_or_dir(path, &dirstream);
+	if (fd < 0) {
+		error("failed to open file or directory: %s", path);
+		ret = 1;
+		goto out;
+	}
+	memset(&dargs, 0, sizeof(dargs));
+	dargs.cmd = BTRFS_DEDUPE_CTL_STATUS;
+
+	ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
+	if (ret < 0) {
+		error("failed to get inband deduplication status: %s",
+		      strerror(errno));
+		ret = 1;
+		goto out;
+	}
+	ret = 0;
+	if (dargs.status == 0) {
+		printf("Status: \t\t\tDisabled\n");
+		goto out;
+	}
+	printf("Status:\t\t\tEnabled\n");
+
+	if (dargs.hash_algo == BTRFS_DEDUPE_HASH_SHA256)
+		printf("Hash algorithm:\t\tSHA-256\n");
+	else
+		printf("Hash algorithm:\t\tUnrecognized(%x)\n",
+			dargs.hash_algo);
+
+	if (dargs.backend == BTRFS_DEDUPE_BACKEND_INMEMORY) {
+		printf("Backend:\t\tIn-memory\n");
+		print_limit = 1;
+	} else  {
+		printf("Backend:\t\tUnrecognized(%x)\n",
+			dargs.backend);
+	}
+
+	printf("Dedup Blocksize:\t%llu\n", dargs.blocksize);
+
+	if (print_limit) {
+		u64 cur_mem;
+
+		/* Limit nr may be 0 */
+		if (dargs.limit_nr)
+			cur_mem = dargs.current_nr * (dargs.limit_mem /
+					dargs.limit_nr);
+		else
+			cur_mem = 0;
+
+		printf("Number of hash: \t[%llu/%llu]\n", dargs.current_nr,
+			dargs.limit_nr);
+		printf("Memory usage: \t\t[%s/%s]\n",
+			pretty_size(cur_mem),
+			pretty_size(dargs.limit_mem));
+	}
+out:
+	close_file_or_dir(fd, dirstream);
+	return ret;
+}
+
 const struct cmd_group dedupe_ib_cmd_group = {
 	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
 		{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
 		  NULL, 0},
 		{ "disable", cmd_dedupe_ib_disable, cmd_dedupe_ib_disable_usage,
 		  NULL, 0},
+		{ "status", cmd_dedupe_ib_status, cmd_dedupe_ib_status_usage,
+		  NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
-- 
2.9.0

[PATCH v9 5/5] btrfs-progs: dedupe: introduce reconfigure subcommand

[Qu Wenruo]

Introduce reconfigure subcommand to co-operate with new kernel ioctl
modification.

Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
---
 Documentation/btrfs-dedupe-inband.asciidoc |  7 +++
 cmds-dedupe-ib.c                           | 74 +++++++++++++++++++++++-------
 2 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
index df068c3..5fc4bb0 100644
--- a/Documentation/btrfs-dedupe-inband.asciidoc
+++ b/Documentation/btrfs-dedupe-inband.asciidoc
@@ -86,6 +86,13 @@ And compression has higher priority than in-band de-duplication, means if
 compression and de-duplication is enabled at the same time, only compression
 will work.
 
+*reconfigure* [options] <path>::
+Re-configure in-band de-duplication parameters of a filesystem.
++
+In-band de-duplication must be enbaled first before re-configuration.
++
+[Options] are the same with 'btrfs dedupe-inband enable'.
+
 *status* <path>::
 Show current in-band de-duplication status of a filesystem.
 
diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
index c9085ac..0c3ef4b 100644
--- a/cmds-dedupe-ib.c
+++ b/cmds-dedupe-ib.c
@@ -68,7 +68,6 @@ static const char * const cmd_dedupe_ib_enable_usage[] = {
 	NULL
 };
 
-
 #define report_fatal_parameter(dargs, old, member, type, err_val, fmt)	\
 if (dargs->member != old->member && dargs->member == (type)(err_val)) {	\
 	error("unsupported dedupe "#member": %"#fmt"", old->member);	\
@@ -91,6 +90,10 @@ static void report_parameter_error(struct btrfs_ioctl_dedupe_args *dargs,
 		}
 		report_option_parameter(dargs, old, flags, u8, -1, x);
 	}
+	if (dargs->status == 0 && old->cmd == BTRFS_DEDUPE_CTL_RECONF) {
+		error("must enable dedupe before reconfiguration");
+		return;
+	}
 	report_fatal_parameter(dargs, old, cmd, u16, -1, u);
 	report_fatal_parameter(dargs, old, blocksize, u64, -1, llu);
 	report_fatal_parameter(dargs, old, backend, u16, -1, u);
@@ -101,14 +104,17 @@ static void report_parameter_error(struct btrfs_ioctl_dedupe_args *dargs,
 	return;
 }
 
-static int cmd_dedupe_ib_enable(int argc, char **argv)
+static int enable_reconfig_dedupe(int argc, char **argv, int reconf)
 {
 	int ret;
 	int fd = -1;
 	char *path;
 	u64 blocksize = BTRFS_DEDUPE_BLOCKSIZE_DEFAULT;
+	int blocksize_set = 0;
 	u16 hash_algo = BTRFS_DEDUPE_HASH_SHA256;
+	int hash_algo_set = 0;
 	u16 backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
+	int backend_set = 0;
 	u64 limit_nr = 0;
 	u64 limit_mem = 0;
 	u64 sys_mem = 0;
@@ -130,20 +136,22 @@ static int cmd_dedupe_ib_enable(int argc, char **argv)
 			{ NULL, 0, NULL, 0}
 		};
 
-		c = getopt_long(argc, argv, "s:b:a:l:m:", long_options, NULL);
+		c = getopt_long(argc, argv, "s:b:a:l:m:f", long_options, NULL);
 		if (c < 0)
 			break;
 		switch (c) {
 		case 's':
-			if (!strcasecmp("inmemory", optarg))
+			if (!strcasecmp("inmemory", optarg)) {
 				backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
-			else {
+				backend_set = 1;
+			} else {
 				error("unsupported dedupe backend: %s", optarg);
 				exit(1);
 			}
 			break;
 		case 'b':
 			blocksize = parse_size(optarg);
+			blocksize_set = 1;
 			break;
 		case 'a':
 			if (strcmp("sha256", optarg)) {
@@ -151,6 +159,7 @@ static int cmd_dedupe_ib_enable(int argc, char **argv)
 				      optarg);
 				return 1;
 			}
+			hash_algo = 1;
 			break;
 		case 'l':
 			limit_nr = parse_size(optarg);
@@ -221,26 +230,40 @@ static int cmd_dedupe_ib_enable(int argc, char **argv)
 		return 1;
 	}
 	memset(&dargs, -1, sizeof(dargs));
-	dargs.cmd = BTRFS_DEDUPE_CTL_ENABLE;
-	dargs.blocksize = blocksize;
-	dargs.hash_algo = hash_algo;
-	dargs.limit_nr = limit_nr;
-	dargs.limit_mem = limit_mem;
-	dargs.backend = backend;
-	if (force)
-		dargs.flags |= BTRFS_DEDUPE_FLAG_FORCE;
-	else
-		dargs.flags = 0;
+	if (reconf) {
+		dargs.cmd = BTRFS_DEDUPE_CTL_RECONF;
+		if (blocksize_set)
+			dargs.blocksize = blocksize;
+		if (hash_algo_set)
+			dargs.hash_algo = hash_algo;
+		if (backend_set)
+			dargs.backend = backend;
+		dargs.limit_nr = limit_nr;
+		dargs.limit_mem = limit_mem;
+	} else {
+		dargs.cmd = BTRFS_DEDUPE_CTL_ENABLE;
+		dargs.blocksize = blocksize;
+		dargs.hash_algo = hash_algo;
+		dargs.limit_nr = limit_nr;
+		dargs.limit_mem = limit_mem;
+		dargs.backend = backend;
+		if (force)
+			dargs.flags |= BTRFS_DEDUPE_FLAG_FORCE;
+		else
+			dargs.flags = 0;
+	}
 
 	memcpy(&backup, &dargs, sizeof(dargs));
 	ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
 	if (ret < 0) {
-		error("failed to enable inband deduplication: %s",
+		error("failed to (%s) inband deduplication: %s",
+		      reconf ? "reconfigure" : "enable",
 		      strerror(errno));
 		report_parameter_error(&dargs, &backup);
 		ret = 1;
 		goto out;
 	}
+	report_parameter_error(&dargs, &backup);
 	ret = 0;
 mem_check:
 	if (!force && dargs.limit_mem > sys_mem / 4) {
@@ -255,6 +278,10 @@ out:
 	close_file_or_dir(fd, dirstream);
 	return ret;
 }
+static int cmd_dedupe_ib_enable(int argc, char **argv)
+{
+	return enable_reconfig_dedupe(argc, argv, 0);
+}
 
 static const char * const cmd_dedupe_ib_disable_usage[] = {
 	"btrfs dedupe disable <path>",
@@ -375,6 +402,19 @@ out:
 	return ret;
 }
 
+static int cmd_dedupe_ib_reconf(int argc, char **argv)
+{
+	return enable_reconfig_dedupe(argc, argv, 1);
+}
+
+static const char * const cmd_dedupe_ib_reconf_usage[] = {
+	"btrfs dedupe-inband reconfigure [options] <path>",
+	"Reconfigure in-band(write time) de-duplication of a btrfs.",
+	"",
+	"Options are the same as 'btrfs dedupe-inband enable'",
+	NULL
+};
+
 const struct cmd_group dedupe_ib_cmd_group = {
 	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
 		{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
@@ -383,6 +423,8 @@ const struct cmd_group dedupe_ib_cmd_group = {
 		  NULL, 0},
 		{ "status", cmd_dedupe_ib_status, cmd_dedupe_ib_status_usage,
 		  NULL, 0},
+		{ "reconfigure", cmd_dedupe_ib_reconf,
+		  cmd_dedupe_ib_reconf_usage, NULL, 0},
 		NULL_CMD_STRUCT
 	}
 };
-- 
2.9.0

Re: [PATCH v9 2/5] btrfs-progs: dedupe: Add enable command for dedupe command group

[Qu Wenruo]

At 06/30/2016 05:24 PM, Qu Wenruo wrote:
> Add enable subcommand for dedupe commmand group.
>
> Signed-off-by: Qu Wenruo <quwenruo@cn.fujitsu.com>
> ---
>  Documentation/btrfs-dedupe-inband.asciidoc | 114 ++++++++++++++-
>  btrfs-completion                           |   6 +-
>  cmds-dedupe-ib.c                           | 222 +++++++++++++++++++++++++++++
>  ioctl.h                                    |   2 +
>  4 files changed, 342 insertions(+), 2 deletions(-)
>
> diff --git a/Documentation/btrfs-dedupe-inband.asciidoc b/Documentation/btrfs-dedupe-inband.asciidoc
> index 9ee2bc7..82f970a 100644
> --- a/Documentation/btrfs-dedupe-inband.asciidoc
> +++ b/Documentation/btrfs-dedupe-inband.asciidoc
> @@ -22,7 +22,119 @@ use with caution.
>
>  SUBCOMMAND
>  ----------
> -Nothing yet
> +*enable* [options] <path>::
> +Enable in-band de-duplication for a filesystem.
> ++
> +`Options`
> ++
> +-f|--force::::
> +Force 'enable' command to be exected.
> +Will skip memory limit check and allow 'enable' to be executed even in-band
> +de-duplication is already enabled.
> ++
> +NOTE: If re-enable dedupe with '-f' option, any unspecified parameter will be
> +reset to its default value.
> +
> +-s|--storage-backend <BACKEND>::::
> +Specify de-duplication hash storage backend.
> +Only 'inmemory' backend is supported yet.
> +If not specified, default value is 'inmemory'.
> ++
> +Refer to *BACKENDS* sector for more information.
> +
> +-b|--blocksize <BLOCKSIZE>::::
> +Specify dedupe block size.
> +Supported values are power of 2 from '16K' to '8M'.
> +Default value is '128K'.
> ++
> +Refer to *BLOCKSIZE* sector for more information.
> +
> +-a|--hash-algorithm <HASH>::::
> +Specify hash algorithm.
> +Only 'sha256' is supported yet.
> +
> +-l|--limit-hash <LIMIT>::::
> +Specify maximum number of hashes stored in memory.
> +Only works for 'inmemory' backend.
> +Conflicts with '-m' option.
> ++
> +Only positive values are valid.
> +Default value is '32K'.
> +
> +-m|--limit-memory <LIMIT>::::
> +Specify maximum memory used for hashes.
> +Only works for 'inmemory' backend.
> +Conflicts with '-l' option.
> ++
> +Only value larger than or equal to '1024' is valid.
> +No default value.
> ++
> +NOTE: Memory limit will be rounded down to kernel internal hash size,
> +so the memory limit shown in 'btrfs dedupe status' may be different
> +from the <LIMIT>.
> +
> +WARNING: Too large value for '-l' or '-m' will easily trigger OOM.
> +Please use with caution according to system memory.
> +
> +NOTE: In-band de-duplication is not compactible with compression yet.
> +And compression has higher priority than in-band de-duplication, means if
> +compression and de-duplication is enabled at the same time, only compression
> +will work.
> +
> +BACKENDS
> +--------
> +Btrfs in-band de-duplication will support different storage backends, with
> +different use case and features.
> +
> +In-memory backend::
> +This backend provides backward-compatibility, and more fine-tuning options.
> +But hash pool is non-persistent and may exhaust kernel memory if not setup
> +properly.
> ++
> +This backend can be used on old btrfs(without '-O dedupe' mkfs option).
> +When used on old btrfs, this backend needs to be enabled manually after mount.
> ++
> +Designed for fast hash search speed, in-memory backend will keep all dedupe
> +hashes in memory. (Although overall performance is still much the same with
> +'ondisk' backend if all 'ondisk' hash can be cached in memory)
> ++
> +And only keeps limited number of hash in memory to avoid exhausting memory.
> +Hashes over the limit will be dropped following Last-Recent-Use behavior.
> +So this backend has a consistent overhead for given limit but can\'t ensure
> +all duplicated blocks will be de-duplicated.
> ++
> +After umount and mount, in-memory backend need to refill its hash pool.
> +
> +On-disk backend::
> +This backend provides persistent hash pool, with more smart memory management
> +for hash pool.
> +But it\'s not backward-compatible, meaning it must be used with '-O dedupe' mkfs
> +option and older kernel can\'t mount it read-write.
> ++
> +Designed for de-duplication rate, hash pool is stored as btrfs B+ tree on disk.
> +This behavior may cause extra disk IO for hash search under high memory
> +pressure.
> ++
> +After umount and mount, on-disk backend still has its hash on disk, no need to
> +refill its dedupe hash pool.
> +
> +Currently, only 'inmemory' backend is supported in btrfs-progs.
> +
> +DEDUPE BLOCK SIZE
> +----------------
> +In-band de-duplication is done at dedupe block size.
> +Any data smaller than dedupe block size won\'t go through in-band
> +de-duplication.
> +
> +And dedupe block size affects dedupe rate and fragmentation heavily.
> +
> +Smaller block size will cause more fragments, but higher dedupe rate.
> +
> +Larger block size will cause less fragments, but lower dedupe rate.
> +
> +In-band de-duplication rate is highly related to the workload pattern.
> +So it\'s highly recommended to align dedupe block size to the workload
> +block size to make full use of de-duplication.
>
>  EXIT STATUS
>  -----------
> diff --git a/btrfs-completion b/btrfs-completion
> index 3ede77b..50f7ea2 100644
> --- a/btrfs-completion
> +++ b/btrfs-completion
> @@ -29,7 +29,7 @@ _btrfs()
>
>  	local cmd=${words[1]}
>
> -    commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup replace help version'
> +    commands='subvolume filesystem balance device scrub check rescue restore inspect-internal property send receive quota qgroup dedupe replace help version'
>      commands_subvolume='create delete list snapshot find-new get-default set-default show sync'
>      commands_filesystem='defragment sync resize show df label usage'
>      commands_balance='start pause cancel resume status'
> @@ -40,6 +40,7 @@ _btrfs()
>      commands_property='get set list'
>      commands_quota='enable disable rescan'
>      commands_qgroup='assign remove create destroy show limit'
> +    commands_dedupe='enable'
>      commands_replace='start status cancel'
>
>  	if [[ "$cur" == -* && $cword -le 3 && "$cmd" != "help" ]]; then
> @@ -94,6 +95,9 @@ _btrfs()
>              qgroup)
>                  opts="$commands_qgroup"
>                  ;;
> +            dedupe)
> +                opts="$commands_dedupe"
> +                ;;
>              replace)
>                  opts="$commands_replace"
>                  ;;
> diff --git a/cmds-dedupe-ib.c b/cmds-dedupe-ib.c
> index 160c8ec..a62b9cb 100644
> --- a/cmds-dedupe-ib.c
> +++ b/cmds-dedupe-ib.c
> @@ -19,6 +19,7 @@
>  #include <getopt.h>
>  #include <unistd.h>
>  #include <sys/ioctl.h>
> +#include <sys/sysinfo.h>
>
>  #include "ctree.h"
>  #include "ioctl.h"
> @@ -36,8 +37,229 @@ static const char * const dedupe_ib_cmd_group_usage[] = {
>  static const char dedupe_ib_cmd_group_info[] =
>  "manage inband(write time) de-duplication";
>
> +static const char * const cmd_dedupe_ib_enable_usage[] = {
> +	"btrfs dedupe-inband enable [options] <path>",
> +	"Enable in-band(write time) de-duplication of a btrfs.",
> +	"",
> +	"-s|--storage-backend <BACKEND>",
> +	"           specify dedupe hash storage backend",
> +	"           supported backend: 'inmemory'",
> +	"-b|--blocksize <BLOCKSIZE>",
> +	"           specify dedupe block size",
> +	"           default value is 128K",
> +	"-a|--hash-algorithm <HASH>",
> +	"           specify hash algorithm",
> +	"           only 'sha256' is supported yet",
> +	"-l|--limit-hash <LIMIT>",
> +	"           specify maximum number of hashes stored in memory",
> +	"           only for 'inmemory' backend",
> +	"           positive value is valid, default value is 32K",
> +	"-m|--limit-mem <LIMIT>",
> +	"           specify maximum memory used for hashes",
> +	"           only for 'inmemory' backend",
> +	"           value larger than or equal to 1024 is valid, no default",
> +	"           only one of '-m' and '-l' is allowed",
> +	"-f|--force",
> +	"           force enable command to be executed",
> +	"           will skip some memory limit check",
> +	"           also without this flag enable command is not allowed to be",
> +	"           executed if dedupe is already enabled",
> +	"           note: unspecified parameter will be reset to default value",
> +	NULL
> +};
> +
> +
> +#define report_fatal_parameter(dargs, old, member, type, err_val, fmt)	\
> +if (dargs->member != old->member && dargs->member == (type)(err_val)) {	\
> +	error("unsupported dedupe "#member": %"#fmt"", old->member);	\
> +	return;								\
> +}
> +
> +#define report_option_parameter(dargs, old, member, type, err_val, fmt)	\
> +if (dargs->member != old->member && dargs->member == (type)(err_val))	\
> +	warning("unsupported optional "#member": %"#fmt", continue",	\
> +		old->member);
> +static void report_parameter_error(struct btrfs_ioctl_dedupe_args *dargs,
> +				   struct btrfs_ioctl_dedupe_args *old)
> +{
> +	if (dargs->flags == (u8)-1) {
> +		if (dargs->status == 1 &&
> +		    old->cmd == BTRFS_DEDUPE_CTL_ENABLE &&
> +		    !(old->flags & BTRFS_DEDUPE_FLAG_FORCE)) {
> +			error("can't re-enable dedupe without --foce");

There is a typo, thanks Tsutomu Itoh for pointing this out.

The typo is fixed in my github tree, dedupe_20160630 branch in place.

Thanks,
Qu

> +			return;
> +		}
> +		report_option_parameter(dargs, old, flags, u8, -1, x);
> +	}
> +	report_fatal_parameter(dargs, old, cmd, u16, -1, u);
> +	report_fatal_parameter(dargs, old, blocksize, u64, -1, llu);
> +	report_fatal_parameter(dargs, old, backend, u16, -1, u);
> +	report_fatal_parameter(dargs, old, hash_algo, u16, -1, u);
> +	if (dargs->limit_nr == 0 && dargs->limit_mem == 0)
> +		error("unsupported dedupe limit combination: nr: %llu, mem: %llu",
> +				old->limit_nr, old->limit_mem);
> +	return;
> +}
> +
> +static int cmd_dedupe_ib_enable(int argc, char **argv)
> +{
> +	int ret;
> +	int fd = -1;
> +	char *path;
> +	u64 blocksize = BTRFS_DEDUPE_BLOCKSIZE_DEFAULT;
> +	u16 hash_algo = BTRFS_DEDUPE_HASH_SHA256;
> +	u16 backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
> +	u64 limit_nr = 0;
> +	u64 limit_mem = 0;
> +	u64 sys_mem = 0;
> +	int force = 0;
> +	struct btrfs_ioctl_dedupe_args dargs;
> +	struct btrfs_ioctl_dedupe_args backup;
> +	struct sysinfo info;
> +	DIR *dirstream = NULL;
> +
> +	while (1) {
> +		int c;
> +		static const struct option long_options[] = {
> +			{ "storage-backend", required_argument, NULL, 's'},
> +			{ "blocksize", required_argument, NULL, 'b'},
> +			{ "hash-algorithm", required_argument, NULL, 'a'},
> +			{ "limit-hash", required_argument, NULL, 'l'},
> +			{ "limit-memory", required_argument, NULL, 'm'},
> +			{ "force", required_argument, NULL, 'f'},
> +			{ NULL, 0, NULL, 0}
> +		};
> +
> +		c = getopt_long(argc, argv, "s:b:a:l:m:", long_options, NULL);
> +		if (c < 0)
> +			break;
> +		switch (c) {
> +		case 's':
> +			if (!strcasecmp("inmemory", optarg))
> +				backend = BTRFS_DEDUPE_BACKEND_INMEMORY;
> +			else {
> +				error("unsupported dedupe backend: %s", optarg);
> +				exit(1);
> +			}
> +			break;
> +		case 'b':
> +			blocksize = parse_size(optarg);
> +			break;
> +		case 'a':
> +			if (strcmp("sha256", optarg)) {
> +				error("unsupported dedupe hash algorithm: %s",
> +				      optarg);
> +				return 1;
> +			}
> +			break;
> +		case 'l':
> +			limit_nr = parse_size(optarg);
> +			if (limit_nr == 0) {
> +				error("limit should be larger than 0");
> +				return 1;
> +			}
> +			break;
> +		case 'm':
> +			limit_mem = parse_size(optarg);
> +			/*
> +			 * Make sure at least one hash is allocated
> +			 * 1024 should be good enough though.
> +			 */
> +			if (limit_mem < 1024) {
> +				error("memory limit should be larger than or equal to 1024");
> +				return 1;
> +			}
> +			break;
> +		case 'f':
> +			force = 1;
> +			break;
> +		default:
> +			usage(cmd_dedupe_ib_enable_usage);
> +			return 1;
> +		}
> +	}
> +
> +	path = argv[optind];
> +	if (check_argc_exact(argc - optind, 1))
> +		usage(cmd_dedupe_ib_enable_usage);
> +
> +	/* Validation check */
> +	if (!is_power_of_2(blocksize) ||
> +	    blocksize > BTRFS_DEDUPE_BLOCKSIZE_MAX ||
> +	    blocksize < BTRFS_DEDUPE_BLOCKSIZE_MIN) {
> +		error("invalid dedupe blocksize: %llu, not in range [%u,%u] or power of 2",
> +		      blocksize, BTRFS_DEDUPE_BLOCKSIZE_MIN,
> +		      BTRFS_DEDUPE_BLOCKSIZE_MAX);
> +		return 1;
> +	}
> +	if ((limit_nr || limit_mem) && backend != BTRFS_DEDUPE_BACKEND_INMEMORY) {
> +		error("limit is only valid for 'inmemory' backend");
> +		return 1;
> +	}
> +	if (limit_nr && limit_mem) {
> +		error("limit-memory and limit-hash can't be given at the same time");
> +		return 1;
> +	}
> +
> +	ret = sysinfo(&info);
> +	if (ret < 0)
> +		warning("failed to determine system total ram size: %s",
> +			strerror(errno));
> +	else
> +		sys_mem = info.totalram;
> +
> +	/*
> +	 * TODO: Add check for limit_nr against current system
> +	 * memory to avoid wrongly set limit.
> +	 */
> +	if (!force && limit_mem && sys_mem && sys_mem < limit_mem * 4)
> +		goto mem_check;
> +
> +	fd = open_file_or_dir(path, &dirstream);
> +	if (fd < 0) {
> +		error("failed to open file or directory: %s", path);
> +		return 1;
> +	}
> +	memset(&dargs, -1, sizeof(dargs));
> +	dargs.cmd = BTRFS_DEDUPE_CTL_ENABLE;
> +	dargs.blocksize = blocksize;
> +	dargs.hash_algo = hash_algo;
> +	dargs.limit_nr = limit_nr;
> +	dargs.limit_mem = limit_mem;
> +	dargs.backend = backend;
> +	if (force)
> +		dargs.flags |= BTRFS_DEDUPE_FLAG_FORCE;
> +	else
> +		dargs.flags = 0;
> +
> +	memcpy(&backup, &dargs, sizeof(dargs));
> +	ret = ioctl(fd, BTRFS_IOC_DEDUPE_CTL, &dargs);
> +	if (ret < 0) {
> +		error("failed to enable inband deduplication: %s",
> +		      strerror(errno));
> +		report_parameter_error(&dargs, &backup);
> +		ret = 1;
> +		goto out;
> +	}
> +	ret = 0;
> +mem_check:
> +	if (!force && dargs.limit_mem > sys_mem / 4) {
> +		ret = 1;
> +		error("memory limit %llu is too large compared to system memory: %llu",
> +		      limit_mem, sys_mem);
> +		error("recommened memory limit is no more than %llu",
> +		      sys_mem / 4);
> +		error("use --force option if you know what you are doing");
> +	}
> +out:
> +	close_file_or_dir(fd, dirstream);
> +	return ret;
> +}
> +
>  const struct cmd_group dedupe_ib_cmd_group = {
>  	dedupe_ib_cmd_group_usage, dedupe_ib_cmd_group_info, {
> +		{ "enable", cmd_dedupe_ib_enable, cmd_dedupe_ib_enable_usage,
> +		  NULL, 0},
>  		NULL_CMD_STRUCT
>  	}
>  };
> diff --git a/ioctl.h b/ioctl.h
> index e698ef7..adf4754 100644
> --- a/ioctl.h
> +++ b/ioctl.h
> @@ -749,6 +749,8 @@ static inline char *btrfs_err_str(enum btrfs_err_code err_code)
>  				    struct btrfs_ioctl_dev_replace_args)
>  #define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
>  					 struct btrfs_ioctl_same_args)
> +#define BTRFS_IOC_DEDUPE_CTL	_IOWR(BTRFS_IOCTL_MAGIC, 55, \
> +				      struct btrfs_ioctl_dedupe_args)
>  #define BTRFS_IOC_GET_FEATURES _IOR(BTRFS_IOCTL_MAGIC, 57, \
>                                    struct btrfs_ioctl_feature_flags)
>  #define BTRFS_IOC_SET_FEATURES _IOW(BTRFS_IOCTL_MAGIC, 57, \
>

Re: [PATCH V9 0/5] In-band de-duplication for btrfs-progs

[David Sterba]

On Thu, Jun 30, 2016 at 05:24:26PM +0800, Qu Wenruo wrote:
> Patchset can be fetched from github:
> https://github.com/adam900710/btrfs-progs.git dedupe_20160630
> 
> Inband dedupe(in-memory backend only) ioctl support for btrfs-progs.
> 
> User/reviewer/tester can still use previous btrfs-progs patchset to test,
> this update is just cleanuping unsupported functions, like on-disk
> backend and any on-disk format change.
> 
> v7 changes:
>    Update ctree.h to follow kernel structure change
>    Update print-tree to follow kernel structure change
> V8 changes:
>    Move dedup props and on-disk backend support out of the patchset
>    Change command group name to "dedupe-inband", to avoid confusion with
>    possible out-of-band dedupe. Suggested by Mark.
>    Rebase to latest devel branch.
> V9 changes:
>    Follow kernels ioctl change to support FORCE flag, new reconf ioctl,
>    and more precious error reporting.

FYI, patchset updated in the integration branch.