[PATCH 0/2] btrfs-progs: add lone extent defragging support

[PATCH 0/2] btrfs-progs: add lone extent defragging support

[Qu Wenruo]

This the progs support for the new kernel defrag parameters on lone file
extents.

This adds 2 new fine tunning parameters, --lone-ratio and
--lone-wasted-bytes.

The ratio is between [0, 65536], and wasted bytes is between
[0, U32_MAX], but in reality the value only makes sense below
max file extent size (for both compressed and regular extents).
Any value higher than max file extent size would mostly disable the
wasted bytes check (as it would always be false).

The end user interface is not yet deteremined, especially for the ratio
(RATIO/65536 is too trikcy for most end users).
But it is already enough to address the lone trimed file extent case
mentioned in the kernel patchset.

Qu Wenruo (2):
  btrfs-progs: defrag: sync the new lone extent fine-tunning from kernel
  btrfs-progs: cmds/filesystem: fine-tune lone file extents defragging

 Documentation/btrfs-filesystem.rst | 18 ++++++++++
 cmds/filesystem.c                  | 53 ++++++++++++++++++++++++++++--
 kernel-shared/uapi/btrfs.h         | 26 +++++++++++++--
 3 files changed, 91 insertions(+), 6 deletions(-)

--
2.43.0

[PATCH 1/2] btrfs-progs: defrag: sync the new lone extent fine-tunning from kernel

[Qu Wenruo]

This would add the long extent fine-tunning support for defrag.

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 kernel-shared/uapi/btrfs.h | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/kernel-shared/uapi/btrfs.h b/kernel-shared/uapi/btrfs.h
index 34e84b89edf2..7fea5f1bcbe8 100644
--- a/kernel-shared/uapi/btrfs.h
+++ b/kernel-shared/uapi/btrfs.h
@@ -642,8 +642,14 @@ _static_assert(sizeof(struct btrfs_ioctl_clone_range_args) == 32);
  * Used by:
  * struct btrfs_ioctl_defrag_range_args.flags
  */
-#define BTRFS_DEFRAG_RANGE_COMPRESS 1
-#define BTRFS_DEFRAG_RANGE_START_IO 2
+#define BTRFS_DEFRAG_RANGE_COMPRESS		(1ULL << 0)
+#define BTRFS_DEFRAG_RANGE_START_IO		(1ULL << 1)
+#define BTRFS_DEFRAG_RANGE_LONE_RATIO		(1ULL << 2)
+#define BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES	(1ULL << 3)
+#define BTRFS_DEFRAG_RANGE_FLAGS_SUPP  (BTRFS_DEFRAG_RANGE_COMPRESS |  \
+					BTRFS_DEFRAG_RANGE_START_IO |  \
+					BTRFS_DEFRAG_RANGE_LONE_RATIO |\
+					BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES)
 struct btrfs_ioctl_defrag_range_args {
 	/* start of the defrag operation */
 	__u64 start;
@@ -671,8 +677,22 @@ struct btrfs_ioctl_defrag_range_args {
 	 */
 	__u32 compress_type;
 
+	/*
+	 * If a lone extent (has no adjacent file extent) is utilizing less
+	 * than the ratio (0 means skip this check, while 65536 means 100%) of
+	 * the on-disk space, it would be considered as a defrag target.
+	 */
+	__u32 lone_ratio;
+
+	/*
+	 * If we defrag a lone extent (has no adjacent file extent) can free
+	 * up more space than this value (in bytes), it would be considered
+	 * as a defrag target.
+	 */
+	__u32 lone_wasted_bytes;
+
 	/* spare for later */
-	__u32 unused[4];
+	__u32 unused[2];
 };
 _static_assert(sizeof(struct btrfs_ioctl_defrag_range_args) == 48);
 
-- 
2.43.0

[PATCH 2/2] btrfs-progs: cmds/filesystem: fine-tune lone file extents defragging

[Qu Wenruo]

This allows defrag to fine-tune long file extents (which have no
adjacent file extents) with new "--lone-ratio" and "--lone-wasted-bytes"
options.

The default behavior is, defrag would go with 4096 as the ratio (1/16),
and 16M as the wasted bytes on the first try.

If the kernel doesn't support it, then retry without lone file extents
handling.

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 Documentation/btrfs-filesystem.rst | 18 ++++++++++
 cmds/filesystem.c                  | 53 ++++++++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/Documentation/btrfs-filesystem.rst b/Documentation/btrfs-filesystem.rst
index 54069ce444f6..f5cd34e9062f 100644
--- a/Documentation/btrfs-filesystem.rst
+++ b/Documentation/btrfs-filesystem.rst
@@ -144,6 +144,24 @@ defragment [options] <file>|<dir> [<file>|<dir>...]
                 The range is default (the whole file) or given by *-s* and *-l*, split into
                 the steps or done in one go if the step is larger. Minimum range size is 256KiB.
 
+	--lone-ratio <ratio>
+		For a lone file extent (which has no adjacent file exctent), if its utilizing
+		less than (ratio / 65536) of the on-disk extent size, it would also be defraged
+		for its potential to free up the on-disk extent.
+
+		Valid values are in the range [0, 65536].
+
+	--lone-wasted-bytes <bytes>[kKmMgGgtTpPeE]
+		For a lone file extent (which has no adjacent file exctent), if defragging
+		the file extent can free up more than <bytes> (excluding the file extent
+		size), the file extent would be defragged.
+
+		Valid values are in the range [0, U32_MAX], but any value larger than 128K
+		would make no sense for compressed file extents, and any value larger than
+		128M would make no sense for regular file extents.
+		As the largest file extent supported is 128K for compressed extent and 128M
+		otherwise, thus it's impossible to free up more bytes than those limits.
+
         -v
                 (deprecated) alias for global *-v* option
 
diff --git a/cmds/filesystem.c b/cmds/filesystem.c
index b7ca4c9a2257..3579e7160c96 100644
--- a/cmds/filesystem.c
+++ b/cmds/filesystem.c
@@ -915,6 +915,8 @@ static const char * const cmd_filesystem_defrag_usage[] = {
 	OPTLINE("-l len", "defragment only up to len bytes"),
 	OPTLINE("-t size", "target extent size hint (default: 32M)"),
 	OPTLINE("--step SIZE", "process the range in given steps, flush after each one"),
+	OPTLINE("--lone-ratio RATIO", "defrag lone extents with utilization ratio lower than RATIO/65536"),
+	OPTLINE("--lone-wasted-bytes SIZE", "defrag lone extents which can free up more bytes than SIZE"),
 	OPTLINE("-v", "deprecated, alias for global -v option"),
 	HELPINFO_INSERT_GLOBALS,
 	HELPINFO_INSERT_VERBOSE,
@@ -930,13 +932,32 @@ static struct btrfs_ioctl_defrag_range_args defrag_global_range;
 static int defrag_global_errors;
 static u64 defrag_global_step;
 
+static int defrag_ioctl(int fd, struct btrfs_ioctl_defrag_range_args *range)
+{
+	int ret = 0;
+
+	ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, range);
+	if (ret < 0 && ((errno == EOPNOTSUPP) | (errno == ENOTTY)) &&
+	    range->flags & (BTRFS_DEFRAG_RANGE_LONE_RATIO |
+			    BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES)) {
+		/*
+		 * Older kernel, no lone extent handling support.
+		 * Retry with lone extent fine-tunning disabled.
+		 */
+		range->flags &= ~(BTRFS_DEFRAG_RANGE_LONE_RATIO |
+				  BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES);
+		ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, range);
+	}
+	return ret;
+}
+
 static int defrag_range_in_steps(int fd, const struct stat *st) {
 	int ret = 0;
 	u64 end;
 	struct btrfs_ioctl_defrag_range_args range;
 
 	if (defrag_global_step == 0)
-		return ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, &defrag_global_range);
+		return defrag_ioctl(fd, &defrag_global_range);
 
 	/*
 	 * If start is set but length is not within or beyond the u64 range,
@@ -947,13 +968,17 @@ static int defrag_range_in_steps(int fd, const struct stat *st) {
 
 	range = defrag_global_range;
 	range.flags |= BTRFS_DEFRAG_RANGE_START_IO;
+	range.flags |= (BTRFS_DEFRAG_RANGE_LONE_RATIO |
+			BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES);
+	range.lone_ratio = defrag_global_range.lone_ratio;
+	range.lone_wasted_bytes = defrag_global_range.lone_wasted_bytes;
 	while (range.start < end) {
 		u64 start;
 
 		range.len = defrag_global_step;
 		pr_verbose(LOG_VERBOSE, "defrag range step: start=%llu len=%llu step=%llu\n",
 			   range.start, range.len, defrag_global_step);
-		ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, &range);
+		ret = defrag_ioctl(fd, &range);
 		if (ret < 0)
 			return ret;
 		if (check_add_overflow(range.start, defrag_global_step, &start))
@@ -1014,6 +1039,8 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	bool recursive = false;
 	int ret = 0;
 	int compress_type = BTRFS_COMPRESS_NONE;
+	u32 lone_ratio = SZ_64K / 16;
+	u32 lone_wasted_bytes = SZ_16M;
 	DIR *dirstream;
 
 	/*
@@ -1048,9 +1075,14 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	defrag_global_errors = 0;
 	optind = 0;
 	while(1) {
-		enum { GETOPT_VAL_STEP = GETOPT_VAL_FIRST };
+		enum { GETOPT_VAL_STEP = GETOPT_VAL_FIRST,
+		       GETOPT_VAL_LONE_RATIO, GETOPT_VAL_LONE_WASTED_BYTES };
 		static const struct option long_options[] = {
 			{ "step", required_argument, NULL, GETOPT_VAL_STEP },
+			{ "lone-ratio", required_argument, NULL,
+				GETOPT_VAL_LONE_RATIO },
+			{ "lone-wasted-bytes", required_argument, NULL,
+				GETOPT_VAL_LONE_WASTED_BYTES },
 			{ NULL, 0, NULL, 0 }
 		};
 		int c;
@@ -1100,6 +1132,17 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 				defrag_global_step = SZ_256K;
 			}
 			break;
+		case GETOPT_VAL_LONE_RATIO:
+			lone_ratio = arg_strtou64(optarg);
+			if (lone_ratio > 65536) {
+				error("invalid ratio, has %u expect [0, 65536]",
+					lone_ratio);
+				exit(1);
+			}
+			break;
+		case GETOPT_VAL_LONE_WASTED_BYTES:
+			lone_wasted_bytes = arg_strtou64_with_suffix(optarg);
+			break;
 		default:
 			usage_unknown_option(cmd, argv);
 		}
@@ -1118,6 +1161,10 @@ static int cmd_filesystem_defrag(const struct cmd_struct *cmd,
 	}
 	if (flush)
 		defrag_global_range.flags |= BTRFS_DEFRAG_RANGE_START_IO;
+	defrag_global_range.lone_ratio = lone_ratio;
+	defrag_global_range.lone_wasted_bytes = lone_wasted_bytes;
+	defrag_global_range.flags |= (BTRFS_DEFRAG_RANGE_LONE_RATIO |
+				      BTRFS_DEFRAG_RANGE_LONE_WASTED_BYTES);
 
 	/*
 	 * Look for directory arguments and warn if the recursive mode is not
-- 
2.43.0