[PATCH] generic: ENOSPC regression test in a multi-threaded scenario

[Pratik Rajesh Sampat <psampat@linux.ibm.com>]

Test allocation strategies of the file system and validate space
anomalies as reported by the system versus the allocated by the
program.

The test is motivated by a bug in ext4 systems where-in ENOSPC is
reported by the file system even though enough space for allocations is
available[1].
[1]: https://patchwork.ozlabs.org/patch/1294003

Suggested-by: Ritesh Harjani <riteshh@linux.ibm.com>
Co-authored-by: Sourabh Jain <sourabhjain@linux.ibm.com>

Signed-off-by: Sourabh Jain <sourabhjain@linux.ibm.com>
Signed-off-by: Pratik Rajesh Sampat <psampat@linux.ibm.com>
---
 src/Makefile          |   2 +-
 src/t_enospc.c        | 186 ++++++++++++++++++++++++++++++++++++++++++
 tests/generic/618     | 168 ++++++++++++++++++++++++++++++++++++++
 tests/generic/618.out |   2 +
 tests/generic/group   |   1 +
 5 files changed, 358 insertions(+), 1 deletion(-)
 create mode 100644 src/t_enospc.c
 create mode 100755 tests/generic/618
 create mode 100644 tests/generic/618.out

diff --git a/src/Makefile b/src/Makefile
index 919d77c4..32940142 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -17,7 +17,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \
 	t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \
 	t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \
 	t_ofd_locks t_mmap_collision mmap-write-concurrent \
-	t_get_file_time t_create_short_dirs t_create_long_dirs
+	t_get_file_time t_create_short_dirs t_create_long_dirs t_enospc
 
 LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \
diff --git a/src/t_enospc.c b/src/t_enospc.c
new file mode 100644
index 00000000..a0a8c783
--- /dev/null
+++ b/src/t_enospc.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 IBM.
+ * All Rights Reserved.
+ *
+ * simple mmap write multithreaded test
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <assert.h>
+#include <sys/mman.h>
+#include <pthread.h>
+#include <signal.h>
+
+#define pr_debug(fmt, ...) do { \
+    if (verbose) { \
+        printf (fmt, ##__VA_ARGS__); \
+    } \
+} while (0)
+
+struct thread_s {
+	int id;
+};
+
+static float file_ratio[2] = {0.5, 0.5};
+static unsigned long fsize = 0;
+static char *base_dir = ".";
+static char *ratio = "1";
+
+static bool use_fallocate = false;
+static bool verbose = false;
+
+pthread_barrier_t bar;
+
+void handle_sigbus(int sig)
+{
+	pr_debug("Enospc test failed with SIGBUS\n");
+	exit(7);
+}
+
+void enospc_test(int id)
+{
+	int fd;
+	char fpath[255] = {0};
+	char *addr;
+	unsigned long size = 0;
+
+	if (id % 2 == 0)
+		size = fsize * file_ratio[0];
+	else
+		size = fsize * file_ratio[1];
+
+	pthread_barrier_wait(&bar);
+
+	sprintf(fpath, "%s/mmap-file-%d", base_dir, id);
+	pr_debug("Test write phase starting file %s fsize %lu, id %d\n", fpath, size, id);
+
+	signal(SIGBUS, handle_sigbus);
+
+	fd = open(fpath, O_RDWR | O_CREAT, 0644);
+	if (fd < 0) {
+		pr_debug("Open failed\n");
+		exit(1);
+	}
+
+	if (use_fallocate)
+		assert(fallocate(fd, 0, 0, fsize) == 0);
+	else
+		assert(ftruncate(fd, size) == 0);
+
+	addr = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, 0);
+	assert(addr != MAP_FAILED);
+
+	for (int i = 0; i < size; i++) {
+		addr[i] = 0xAB;
+	}
+
+	assert(munmap(addr, size) != -1);
+	close(fd);
+	pr_debug("Test write phase completed...file %s, fsize %lu, id %d\n", fpath, size, id);
+}
+
+void *spawn_test_thread(void *arg)
+{
+	struct thread_s *thread_info = (struct thread_s *)arg;
+	pthread_t tid;
+
+	tid = pthread_self();
+	pr_debug("Inside thread %lu %d\n", tid, thread_info->id);
+	enospc_test(thread_info->id);
+	return NULL;
+}
+
+void _run_test(int threads)
+{
+	pthread_t tid[threads];
+
+	pthread_barrier_init(&bar, NULL, threads+1);
+	for (int i = 0; i < threads; i++) {
+		struct thread_s *thread_info = (struct thread_s *) malloc(sizeof(struct thread_s));
+		thread_info->id = i;
+		assert(pthread_create(&tid[i], NULL, spawn_test_thread, thread_info) == 0);
+	}
+
+	pthread_barrier_wait(&bar);
+
+	for (int i = 0; i <threads; i++) {
+		assert(pthread_join(tid[i], NULL) == 0);
+	}
+
+	pthread_barrier_destroy(&bar);
+	return;
+}
+
+static void usage(void)
+{
+	printf("\nUsage: t_enospc [options]\n\n"
+	       " -t                    thread count\n"
+	       " -s size               file size\n"
+	       " -p path               set base path\n"
+	       " -f fallocate          use fallocate instead of ftruncate\n"
+	       " -v verbose            print debug information\n"
+	       " -r ratio              ratio of file sizes\n");
+}
+
+int main(int argc, char *argv[])
+{
+	int opt;
+	int threads = 0;
+	char *ratio_ptr;
+
+	while ((opt = getopt(argc, argv, ":r:p:t:s:vf")) != -1) {
+		switch(opt) {
+		case 't':
+			threads = atoi(optarg);
+			pr_debug("Testing with (%d) threads\n", threads);
+			break;
+		case 's':
+			fsize = atoi(optarg);
+			pr_debug("size: %lu Bytes\n", fsize);
+			break;
+		case 'p':
+			base_dir = optarg;
+			break;
+		case 'r':
+			ratio = optarg;
+			ratio_ptr = strtok(ratio, ",");
+			if (ratio_ptr[0] == '1') {
+				file_ratio[0] = 1;
+				file_ratio[1] = 1;
+				break;
+			}
+			if (ratio_ptr != NULL)
+				file_ratio[0] = strtod(ratio_ptr, NULL);
+			ratio_ptr = strtok(NULL, ",");
+			if (ratio_ptr != NULL)
+				file_ratio[1] = strtod(ratio_ptr, NULL);
+			break;
+		case 'f':
+			use_fallocate = true;
+			break;
+		case 'v':
+			verbose = true;
+			break;
+		case '?':
+			usage();
+			exit(1);
+		}
+	}
+
+	/* Sanity test of parameters */
+	assert(threads);
+	assert(fsize);
+	assert(file_ratio[0]);
+	assert(file_ratio[1]);
+
+	pr_debug("Testing with (%d) threads\n", threads);
+	pr_debug("size: %lu Bytes\n", fsize);
+
+	_run_test(threads);
+	return 0;
+}
diff --git a/tests/generic/618 b/tests/generic/618
new file mode 100755
index 00000000..57216b3d
--- /dev/null
+++ b/tests/generic/618
@@ -0,0 +1,168 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2020 IBM Corporation. All Rights Reserved.
+#
+# FS QA Test 618
+# ENOSPC regression test in a multi-threaded scenario. Test allocation
+# strategies of the file system and validate space anomalies as reported by
+# the system versus the allocated by the program.
+#
+# With this we should be able to catch similar regressions as reported on
+# ext4 on 5.7 kernel which was fixed by patch [1]
+# [1]: https://patchwork.ozlabs.org/patch/1294003
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+FS_SIZE=$((240*1024*1024)) # 240MB
+DEBUG=0 # set to 1 to enable debug statements in shell and c-prog
+FACT=0.7
+
+# Disk allocation methods
+FALLOCATE=1
+FTRUNCATE=2
+
+# Helps to build TEST_VECTORS
+SMALL_FILE_SIZE=$((512 * 1024)) # in Bytes
+BIG_FILE_SIZE=$((1536 * 1024))  # in Bytes
+MIX_FILE_SIZE=$((2048 * 1024))  # (BIG + SMALL small file size)
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# remove previous $seqres.full before test
+rm -f $seqres.full
+
+# Modify as appropriate.
+_supported_fs generic
+_require_test
+_require_scratch
+_require_test_program "t_enospc"
+
+debug()
+{
+	if [ $DEBUG -eq 1 ]; then
+		echo >&2 "func->${FUNCNAME[1]}: $1"
+	fi
+}
+
+# Calculate the number of threads needed to fill the disk space
+calc_thread_cnt()
+{
+	local file_ratio_unit=$1
+	local file_ratio=$2
+	local disk_saturation=$3
+	local tot_avail_size
+	local avail_size
+	local thread_cnt
+
+	IFS=',' read -ra fratio <<< $file_ratio
+	file_ratio_cnt=${#fratio[@]}
+
+	tot_avail_size=$(df --block-size=1 $SCRATCH_DEV | awk 'FNR == 2 { print $4 }')
+	avail_size=$(echo $tot_avail_size*$disk_saturation | bc)
+	thread_cnt=$(echo "$file_ratio_cnt*($avail_size/$file_ratio_unit)" | bc)
+
+	debug "Total available size: $tot_avail_size"
+	debug "Available size: $avail_size"
+	debug "Thread count: $thread_cnt"
+
+	echo ${thread_cnt}
+}
+
+run_testcase()
+{
+	IFS=':' read -ra args <<< $1
+	local test_name=${args[0]}
+	local file_ratio_unit=${args[1]}
+	local file_ratio=${args[2]}
+	local disk_saturation=${args[3]}
+	local disk_alloc_method=${args[4]}
+	local test_iteration_cnt=${args[5]}
+	local extra_args=""
+	local thread_cnt
+
+	if [ "$disk_alloc_method" == "$FTRUNCATE" ]; then
+		extra_args="$extra_args -f"
+	fi
+
+	# enable the debug statements in c program
+	if [ "$DEBUG" -eq 1 ]; then
+		extra_args="$extra_args -v"
+	fi
+
+	for i in $(eval echo "{1..$test_iteration_cnt}")
+	do
+		# Setup the device
+		_scratch_mkfs_sized $FS_SIZE >> $seqres.full 2>&1
+		_scratch_mount
+
+		thread_cnt=$(calc_thread_cnt $file_ratio_unit $file_ratio $disk_saturation)
+
+		debug "====== Test details starts ======"
+		debug "Test name: $test_name"
+		debug "File ratio unit: $file_ratio_unit"
+		debug "File ratio: $file_ratio"
+		debug "Disk saturation $disk_saturation"
+		debug "Disk alloc method $disk_alloc_method"
+		debug "Test iteration count: $test_iteration_cnt"
+		debug "Extra arg: $extra_args"
+		debug "Thread count: $thread_cnt"
+		debug "============ end ==============="
+
+		# Start the test
+		$here/src/t_enospc -t $thread_cnt -s $file_ratio_unit -r $file_ratio -p $SCRATCH_MNT $extra_args
+
+		status=$(echo $?)
+		if [ $status -ne 0 ]; then
+			use_per=$(df -h | grep $SCRATCH_MNT | awk '{print substr($5, 1, length($5)-1)}' | bc)
+			alloc_per=$(echo "$FACT * 100" | bc)
+			# We are here since t_nospc failed with an error code.
+			# If the used filesystem space is still < available space - that means
+			# the test failed due to FS wrongly reported ENOSPC.
+			if [ $(echo "$use_per < $alloc_per" | bc) -ne 0 ]; then
+				if [ $status -eq 134 ]; then
+					echo "FAIL: Aborted assertion faliure"
+				elif [ $status -eq 7 ]; then
+					echo "FAIL: ENOSPC BUS faliure"
+				fi
+				echo "$test_name failed at iteration count: $i"
+				echo "$(df -h $SCRATCH_MNT)"
+				echo "Allocated: $alloc_per% Used: $use_per%"
+				exit
+			fi
+		fi
+
+		# Make space for other tests
+		_scratch_unmount
+	done
+}
+
+declare -a TEST_VECTORS=(
+# test-name:file-ratio-unit:file-ratio:disk-saturation:disk-alloc-method:test-iteration-cnt
+"Small-file-test:$SMALL_FILE_SIZE:1:$FACT:$FALLOCATE:3"
+"Big-file-test:$BIG_FILE_SIZE:1:$FACT:$FALLOCATE:3"
+"Mix-file-test:$MIX_FILE_SIZE:0.75,0.25:$FACT:$FALLOCATE:3"
+)
+
+# real QA test starts here
+for i in "${TEST_VECTORS[@]}"; do
+	run_testcase $i
+done
+
+echo "Silence is golden"
+status=0
+exit
diff --git a/tests/generic/618.out b/tests/generic/618.out
new file mode 100644
index 00000000..8940b72f
--- /dev/null
+++ b/tests/generic/618.out
@@ -0,0 +1,2 @@
+QA output created by 618
+Silence is golden
diff --git a/tests/generic/group b/tests/generic/group
index 94e860b8..dc668c06 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -620,3 +620,4 @@
 615 auto rw
 616 auto rw io_uring stress
 617 auto rw io_uring stress
+618 auto rw stress
-- 
2.28.0