[PATCH 0/2] fstests: incore unlinked list

[PATCH 0/2] fstests: incore unlinked list

[Darrick J. Wong]

Hi all,

This series adds regression tests for the new XFS incore unlinked backref
cache feature.

The first patch teaches the XFS debug knob code to _notrun a test if the
kernel supports debug knob discovery (via sysfs) but doesn't advertise
the particular knob.

The second patch adds four new regression tests.  The two generic/ tests
exercise the creation and removal of O_TMPFILE files (or files which
have been created, opened, unlinked, and then closed), which ensures
that the filesystem can demonstrate that behavior without problems.  The
two xfs/ tests examine the same behavior, but they deliberately skip the
backref cache lookups to ensure that the old code still works.

If you're going to start using this mess, you probably ought to just
pull from my git trees, which are linked below.

This is an extraordinary way to destroy everything.  Enjoy!
Comments and questions are, as always, welcome.

--D

kernel git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfs-linux.git/log/?h=incore-unlinked-list

xfsprogs git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfsprogs-dev.git/log/?h=incore-unlinked-list

fstests git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfstests-dev.git/log/?h=incore-unlinked-list

[PATCH 1/2] inject: skip tests when knob dir exists but knob doesn't

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

If the XFS error injection knob directory exists but the knob itself
doesn't, then we know that this kernel doesn't support the knob and
can skip the test.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 common/inject |    7 +++++++
 1 file changed, 7 insertions(+)


diff --git a/common/inject b/common/inject
index 903fb379..984ec209 100644
--- a/common/inject
+++ b/common/inject
@@ -62,6 +62,13 @@ _require_xfs_io_error_injection()
 	knob="$(_find_xfs_mountdev_errortag_knob "${TEST_DEV}" "${type}")"
 	test -w "${knob}" && return
 
+	# If the directory containing the sysfs error injection knob exists
+	# but the knob itself isn't usable, this kernel doesn't know about
+	# the knob.  Skip the test.
+	if [ -d "$(dirname "${knob}")" ]; then
+		_notrun "XFS error injection $type unknown on this kernel."
+	fi
+
 	# NOTE: We can't actually test error injection here because xfs
 	# hasn't always range checked the argument to xfs_errortag_add.
 	# We also don't want to trip an error before we're ready to deal

[PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

Create a test (+ helper program) that opens as many unlinked files as it
possibly can on the scratch filesystem, then closes all the files at
once to stress-test unlinked file cleanup.  Add an xfs-specific test to
make sure that the fallback code doesn't bitrot.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 src/Makefile          |    2 -
 src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
 tests/generic/710     |   65 +++++++++++++++++++++++++
 tests/generic/710.out |    2 +
 tests/generic/711     |   73 ++++++++++++++++++++++++++++
 tests/generic/711.out |    2 +
 tests/generic/group   |    2 +
 tests/xfs/736         |   71 +++++++++++++++++++++++++++
 tests/xfs/736.out     |    2 +
 tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
 tests/xfs/737.out     |    2 +
 tests/xfs/group       |    2 +
 12 files changed, 428 insertions(+), 1 deletion(-)
 create mode 100644 src/tmpfile.c
 create mode 100755 tests/generic/710
 create mode 100644 tests/generic/710.out
 create mode 100755 tests/generic/711
 create mode 100644 tests/generic/711.out
 create mode 100755 tests/xfs/736
 create mode 100644 tests/xfs/736.out
 create mode 100755 tests/xfs/737
 create mode 100644 tests/xfs/737.out


diff --git a/src/Makefile b/src/Makefile
index 41826585..5fce881d 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	renameat2 t_getcwd e4compact test-nextquota punch-alternating \
 	attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
 	dio-invalidate-cache stat_test t_encrypted_d_revalidate \
-	attr_replace_test swapon mkswap
+	attr_replace_test swapon mkswap tmpfile
 
 SUBDIRS = log-writes perf
 
diff --git a/src/tmpfile.c b/src/tmpfile.c
new file mode 100644
index 00000000..1b74dc72
--- /dev/null
+++ b/src/tmpfile.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Oracle.  All Rights Reserved.
+ * Author: Darrick J. Wong <darrick.wong@oracle.com>
+ *
+ * Test program to open unlinked files and leak them.
+ */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+#include <time.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+
+static int min_fd = -1;
+static int max_fd = -1;
+static unsigned int nr_opened = 0;
+static float start_time;
+
+void clock_time(float *time)
+{
+	static clockid_t clkid = CLOCK_MONOTONIC;
+	struct timespec ts;
+	int ret;
+
+retry:
+	ret = clock_gettime(clkid, &ts);
+	if (ret) {
+		if (clkid == CLOCK_MONOTONIC) {
+			clkid = CLOCK_REALTIME;
+			goto retry;
+		}
+		perror("clock_gettime");
+		exit(2);
+	}
+	*time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
+}
+
+/*
+ * Exit the program due to an error.
+ *
+ * If we've exhausted all the file descriptors, make sure we close all the
+ * open fds in the order we received them in order to exploit a quirk of ext4
+ * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
+ * lists, which will make removing the unlinked files maximally painful.
+ *
+ * If it's some other error, just die and let the kernel sort it out.
+ */
+void die(void)
+{
+	float end_time;
+	int fd;
+
+	switch (errno) {
+	case EMFILE:
+	case ENFILE:
+	case ENOSPC:
+		clock_time(&end_time);
+		printf("Opened %u files in %.2fs.\n", nr_opened,
+				end_time - start_time);
+		fflush(stdout);
+
+		clock_time(&start_time);
+		for (fd = min_fd; fd <= max_fd; fd++)
+			close(fd);
+		clock_time(&end_time);
+		printf("Closed %u files in %.2fs.\n", nr_opened,
+				end_time - start_time);
+		exit(0);
+		break;
+	default:
+		perror("open?");
+		exit(2);
+		break;
+	}
+}
+
+/* Remember how many file we open and all that. */
+void remember_fd(int fd)
+{
+	if (min_fd == -1 || min_fd > fd)
+		min_fd = fd;
+	if (max_fd == -1 || max_fd < fd)
+		max_fd = fd;
+	nr_opened++;
+}
+
+/* Put an opened file on the unlinked list and leak the fd. */
+void leak_tmpfile(void)
+{
+	int fd = -1;
+	int ret;
+
+	/* Try to create an O_TMPFILE and leak the fd. */
+#ifdef O_TMPFILE
+	fd = open(".", O_TMPFILE | O_RDWR, 0644);
+	if (fd >= 0) {
+		remember_fd(fd);
+		return;
+	}
+	if (fd < 0 && errno != EOPNOTSUPP)
+		die();
+#endif
+
+	/* Oh well, create a new file, unlink it, and leak the fd. */
+	fd = open("./moo", O_CREAT | O_RDWR, 0644);
+	if (fd < 0)
+		die();
+	ret = unlink("./moo");
+	if (ret)
+		die();
+	remember_fd(fd);
+}
+
+/* Try to put as many files on the unlinked list and then kill them. */
+int main(int argc, char *argv[])
+{
+	clock_time(&start_time);
+	while (1)
+		leak_tmpfile();
+	return 0;
+}
diff --git a/tests/generic/710 b/tests/generic/710
new file mode 100755
index 00000000..18aa9d34
--- /dev/null
+++ b/tests/generic/710
@@ -0,0 +1,65 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 710
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use only a single CPU to test the single threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/attr
+. ./common/filter
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(cat /proc/sys/fs/file-max)
+test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+program=$PWD/src/tmpfile
+(cd $SCRATCH_MNT ; $program >> $seqres.full)
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/710.out b/tests/generic/710.out
new file mode 100644
index 00000000..e0a55170
--- /dev/null
+++ b/tests/generic/710.out
@@ -0,0 +1,2 @@
+QA output created by 710
+silence is golden
diff --git a/tests/generic/711 b/tests/generic/711
new file mode 100755
index 00000000..11d76218
--- /dev/null
+++ b/tests/generic/711
@@ -0,0 +1,73 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 711
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use every CPU possible to stress the filesystem.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/attr
+. ./common/filter
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Try to load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
+test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+program=$PWD/src/tmpfile
+for i in $(seq 1 $nr_cpus); do
+	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
+done
+for i in $(seq 1 $nr_cpus); do
+	wait
+done
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/711.out b/tests/generic/711.out
new file mode 100644
index 00000000..cbbe36e9
--- /dev/null
+++ b/tests/generic/711.out
@@ -0,0 +1,2 @@
+QA output created by 711
+silence is golden
diff --git a/tests/generic/group b/tests/generic/group
index f56eb475..26999ca1 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -529,3 +529,5 @@
 524 auto quick
 525 auto quick rw
 709 auto quick
+710 auto quick unlink
+711 auto quick unlink
diff --git a/tests/xfs/736 b/tests/xfs/736
new file mode 100755
index 00000000..e33de0ae
--- /dev/null
+++ b/tests/xfs/736
@@ -0,0 +1,71 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 736
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Here we force the use of the slow iunlink bucket walk code in a single
+# threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/attr
+. ./common/filter
+. ./common/inject
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(cat /proc/sys/fs/file-max)
+test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Force xfs to use the iunlinked fallback 50% of the time
+_scratch_inject_error "iunlink_fallback" "2"
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+program=$PWD/src/tmpfile
+(cd $SCRATCH_MNT ; $program >> $seqres.full)
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/736.out b/tests/xfs/736.out
new file mode 100644
index 00000000..0258a248
--- /dev/null
+++ b/tests/xfs/736.out
@@ -0,0 +1,2 @@
+QA output created by 736
+silence is golden
diff --git a/tests/xfs/737 b/tests/xfs/737
new file mode 100755
index 00000000..47e65607
--- /dev/null
+++ b/tests/xfs/737
@@ -0,0 +1,79 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 737
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Here we force the use of the slow iunlink bucket walk code, using every
+# CPU possible.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/attr
+. ./common/filter
+. ./common/inject
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
+test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Force xfs to use the iunlinked fallback 50% of the time
+_scratch_inject_error "iunlink_fallback" "2"
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+program=$PWD/src/tmpfile
+for i in $(seq 1 $nr_cpus); do
+	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
+done
+for i in $(seq 1 $nr_cpus); do
+	wait
+done
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/737.out b/tests/xfs/737.out
new file mode 100644
index 00000000..bdc4966d
--- /dev/null
+++ b/tests/xfs/737.out
@@ -0,0 +1,2 @@
+QA output created by 737
+silence is golden
diff --git a/tests/xfs/group b/tests/xfs/group
index 7b7d69f1..d3189cd5 100644
--- a/tests/xfs/group
+++ b/tests/xfs/group
@@ -497,3 +497,5 @@
 497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
 498 dangerous_fuzzers dangerous_norepair
 499 auto quick
+736 auto quick unlink
+737 auto quick unlink

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Amir Goldstein]

On Tue, Feb 12, 2019 at 4:18 AM Darrick J. Wong <darrick.wong@oracle.com> wrote:
>
> From: Darrick J. Wong <darrick.wong@oracle.com>
>
> Create a test (+ helper program) that opens as many unlinked files as it
> possibly can on the scratch filesystem, then closes all the files at
> once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> make sure that the fallback code doesn't bitrot.
>
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---
>  src/Makefile          |    2 -
>  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/generic/710     |   65 +++++++++++++++++++++++++
>  tests/generic/710.out |    2 +
>  tests/generic/711     |   73 ++++++++++++++++++++++++++++
>  tests/generic/711.out |    2 +
>  tests/generic/group   |    2 +
>  tests/xfs/736         |   71 +++++++++++++++++++++++++++
>  tests/xfs/736.out     |    2 +
>  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
>  tests/xfs/737.out     |    2 +
>  tests/xfs/group       |    2 +
>  12 files changed, 428 insertions(+), 1 deletion(-)
>  create mode 100644 src/tmpfile.c
>  create mode 100755 tests/generic/710
>  create mode 100644 tests/generic/710.out
>  create mode 100755 tests/generic/711
>  create mode 100644 tests/generic/711.out
>  create mode 100755 tests/xfs/736
>  create mode 100644 tests/xfs/736.out
>  create mode 100755 tests/xfs/737
>  create mode 100644 tests/xfs/737.out
>
>
> diff --git a/src/Makefile b/src/Makefile
> index 41826585..5fce881d 100644
> --- a/src/Makefile
> +++ b/src/Makefile
> @@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
>         renameat2 t_getcwd e4compact test-nextquota punch-alternating \
>         attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
>         dio-invalidate-cache stat_test t_encrypted_d_revalidate \
> -       attr_replace_test swapon mkswap
> +       attr_replace_test swapon mkswap tmpfile
>
>  SUBDIRS = log-writes perf
>
> diff --git a/src/tmpfile.c b/src/tmpfile.c

Let's go crazy and call it t_open_tmpfiles ?
Missing .gitignore patch.

...
> +/* Put an opened file on the unlinked list and leak the fd. */
> +void leak_tmpfile(void)
> +{
> +       int fd = -1;
> +       int ret;
> +
> +       /* Try to create an O_TMPFILE and leak the fd. */
> +#ifdef O_TMPFILE
> +       fd = open(".", O_TMPFILE | O_RDWR, 0644);

Pass in path as argv[1], so you won't need to cd $SCRATCH_MNT
all the time. Or does it have a purpose?

Thanks,
Amir.

Re: [PATCH 1/2] inject: skip tests when knob dir exists but knob doesn't

[Brian Foster]

On Mon, Feb 11, 2019 at 06:17:48PM -0800, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> If the XFS error injection knob directory exists but the knob itself
> doesn't, then we know that this kernel doesn't support the knob and
> can skip the test.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---

Reviewed-by: Brian Foster <bfoster@redhat.com>

>  common/inject |    7 +++++++
>  1 file changed, 7 insertions(+)
> 
> 
> diff --git a/common/inject b/common/inject
> index 903fb379..984ec209 100644
> --- a/common/inject
> +++ b/common/inject
> @@ -62,6 +62,13 @@ _require_xfs_io_error_injection()
>  	knob="$(_find_xfs_mountdev_errortag_knob "${TEST_DEV}" "${type}")"
>  	test -w "${knob}" && return
>  
> +	# If the directory containing the sysfs error injection knob exists
> +	# but the knob itself isn't usable, this kernel doesn't know about
> +	# the knob.  Skip the test.
> +	if [ -d "$(dirname "${knob}")" ]; then
> +		_notrun "XFS error injection $type unknown on this kernel."
> +	fi
> +
>  	# NOTE: We can't actually test error injection here because xfs
>  	# hasn't always range checked the argument to xfs_errortag_add.
>  	# We also don't want to trip an error before we're ready to deal
> 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Brian Foster]

On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Create a test (+ helper program) that opens as many unlinked files as it
> possibly can on the scratch filesystem, then closes all the files at
> once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> make sure that the fallback code doesn't bitrot.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---
>  src/Makefile          |    2 -
>  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/generic/710     |   65 +++++++++++++++++++++++++
>  tests/generic/710.out |    2 +
>  tests/generic/711     |   73 ++++++++++++++++++++++++++++
>  tests/generic/711.out |    2 +
>  tests/generic/group   |    2 +
>  tests/xfs/736         |   71 +++++++++++++++++++++++++++
>  tests/xfs/736.out     |    2 +
>  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
>  tests/xfs/737.out     |    2 +
>  tests/xfs/group       |    2 +
>  12 files changed, 428 insertions(+), 1 deletion(-)
>  create mode 100644 src/tmpfile.c
>  create mode 100755 tests/generic/710
>  create mode 100644 tests/generic/710.out
>  create mode 100755 tests/generic/711
>  create mode 100644 tests/generic/711.out
>  create mode 100755 tests/xfs/736
>  create mode 100644 tests/xfs/736.out
>  create mode 100755 tests/xfs/737
>  create mode 100644 tests/xfs/737.out
> 
> 
> diff --git a/src/Makefile b/src/Makefile
> index 41826585..5fce881d 100644
> --- a/src/Makefile
> +++ b/src/Makefile
> @@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
>  	renameat2 t_getcwd e4compact test-nextquota punch-alternating \
>  	attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
>  	dio-invalidate-cache stat_test t_encrypted_d_revalidate \
> -	attr_replace_test swapon mkswap
> +	attr_replace_test swapon mkswap tmpfile
>  
>  SUBDIRS = log-writes perf
>  
> diff --git a/src/tmpfile.c b/src/tmpfile.c
> new file mode 100644
> index 00000000..1b74dc72
> --- /dev/null
> +++ b/src/tmpfile.c
> @@ -0,0 +1,127 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * Copyright (C) 2019 Oracle.  All Rights Reserved.
> + * Author: Darrick J. Wong <darrick.wong@oracle.com>
> + *
> + * Test program to open unlinked files and leak them.
> + */
> +#ifndef _GNU_SOURCE
> +# define _GNU_SOURCE
> +#endif
> +#include <time.h>
> +#include <unistd.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
> +#include <errno.h>
> +
> +static int min_fd = -1;
> +static int max_fd = -1;
> +static unsigned int nr_opened = 0;
> +static float start_time;
> +
> +void clock_time(float *time)
> +{
> +	static clockid_t clkid = CLOCK_MONOTONIC;
> +	struct timespec ts;
> +	int ret;
> +
> +retry:
> +	ret = clock_gettime(clkid, &ts);
> +	if (ret) {
> +		if (clkid == CLOCK_MONOTONIC) {
> +			clkid = CLOCK_REALTIME;
> +			goto retry;
> +		}
> +		perror("clock_gettime");
> +		exit(2);
> +	}
> +	*time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
> +}
> +
> +/*
> + * Exit the program due to an error.
> + *
> + * If we've exhausted all the file descriptors, make sure we close all the
> + * open fds in the order we received them in order to exploit a quirk of ext4
> + * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
> + * lists, which will make removing the unlinked files maximally painful.
> + *
> + * If it's some other error, just die and let the kernel sort it out.
> + */
> +void die(void)
> +{
> +	float end_time;
> +	int fd;
> +
> +	switch (errno) {
> +	case EMFILE:
> +	case ENFILE:
> +	case ENOSPC:
> +		clock_time(&end_time);
> +		printf("Opened %u files in %.2fs.\n", nr_opened,
> +				end_time - start_time);
> +		fflush(stdout);
> +
> +		clock_time(&start_time);
> +		for (fd = min_fd; fd <= max_fd; fd++)
> +			close(fd);
> +		clock_time(&end_time);
> +		printf("Closed %u files in %.2fs.\n", nr_opened,
> +				end_time - start_time);
> +		exit(0);
> +		break;
> +	default:
> +		perror("open?");
> +		exit(2);
> +		break;
> +	}
> +}
> +
> +/* Remember how many file we open and all that. */
> +void remember_fd(int fd)
> +{
> +	if (min_fd == -1 || min_fd > fd)
> +		min_fd = fd;
> +	if (max_fd == -1 || max_fd < fd)
> +		max_fd = fd;
> +	nr_opened++;
> +}
> +
> +/* Put an opened file on the unlinked list and leak the fd. */
> +void leak_tmpfile(void)
> +{
> +	int fd = -1;
> +	int ret;
> +
> +	/* Try to create an O_TMPFILE and leak the fd. */
> +#ifdef O_TMPFILE
> +	fd = open(".", O_TMPFILE | O_RDWR, 0644);
> +	if (fd >= 0) {
> +		remember_fd(fd);
> +		return;
> +	}
> +	if (fd < 0 && errno != EOPNOTSUPP)
> +		die();
> +#endif

Could we track lack of tmpfile support so we don't repeat the open()
call once we know it's going to fail?

> +
> +	/* Oh well, create a new file, unlink it, and leak the fd. */
> +	fd = open("./moo", O_CREAT | O_RDWR, 0644);
> +	if (fd < 0)
> +		die();
> +	ret = unlink("./moo");
> +	if (ret)
> +		die();
> +	remember_fd(fd);
> +}
> +
> +/* Try to put as many files on the unlinked list and then kill them. */
> +int main(int argc, char *argv[])
> +{
> +	clock_time(&start_time);
> +	while (1)
> +		leak_tmpfile();
> +	return 0;
> +}
> diff --git a/tests/generic/710 b/tests/generic/710
> new file mode 100755
> index 00000000..18aa9d34
> --- /dev/null
> +++ b/tests/generic/710
> @@ -0,0 +1,65 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 710
> +#
> +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> +# all at once, checking that we don't blow up the filesystem.  This is sort
> +# of a performance test for the xfs unlinked inode backref patchset, but it
> +# applies to most other filesystems.
> +#
> +# Use only a single CPU to test the single threaded situation.
> +#
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +tmp=/tmp/$$
> +status=1	# failure is the default!
> +testfile=$TEST_DIR/$seq.txt
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/attr
> +. ./common/filter

Do we need attr/filter?

> +
> +# real QA test starts here
> +_supported_fs generic
> +_supported_os Linux
> +_require_scratch
> +
> +rm -f $seqres.full
> +_scratch_mkfs >> $seqres.full 2>&1
> +_scratch_mount
> +
> +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> +# so that this test doesn't take forever or OOM the box
> +max_files=$((50000 * LOAD_FACTOR))
> +max_allowable_files=$(cat /proc/sys/fs/file-max)
> +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files

I see the following from the above line when I run this test:

+./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected

> +ulimit -n $max_files
> +
> +# Open a lot of unlinked files
> +echo create >> $seqres.full
> +program=$PWD/src/tmpfile
> +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> +
> +# Unmount to prove that we can clean it all
> +echo umount >> $seqres.full
> +before=$(date +%s)
> +_scratch_unmount
> +after=$(date +%s)
> +echo "Unmount took $((after - before))s." >> $seqres.full
> +
> +# Mount so that we can run the usual checks
> +echo silence is golden
> +_scratch_mount
> +status=0
> +exit
> diff --git a/tests/generic/710.out b/tests/generic/710.out
> new file mode 100644
> index 00000000..e0a55170
> --- /dev/null
> +++ b/tests/generic/710.out
> @@ -0,0 +1,2 @@
> +QA output created by 710
> +silence is golden
> diff --git a/tests/generic/711 b/tests/generic/711
> new file mode 100755
> index 00000000..11d76218
> --- /dev/null
> +++ b/tests/generic/711
> @@ -0,0 +1,73 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 711
> +#
> +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> +# all at once, checking that we don't blow up the filesystem.  This is sort
> +# of a performance test for the xfs unlinked inode backref patchset, but it
> +# applies to most other filesystems.
> +#
> +# Use every CPU possible to stress the filesystem.
> +#
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +tmp=/tmp/$$
> +status=1	# failure is the default!
> +testfile=$TEST_DIR/$seq.txt
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/attr
> +. ./common/filter
> +
> +# real QA test starts here
> +_supported_fs generic
> +_supported_os Linux
> +_require_scratch
> +
> +rm -f $seqres.full
> +_scratch_mkfs >> $seqres.full 2>&1
> +_scratch_mount
> +
> +# Try to load up all the CPUs, two threads per CPU.
> +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> +
> +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> +# so that this test doesn't take forever or OOM the box
> +max_files=$((50000 * LOAD_FACTOR))
> +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> +ulimit -n $max_files
> +
> +# Open a lot of unlinked files
> +echo create >> $seqres.full
> +program=$PWD/src/tmpfile
> +for i in $(seq 1 $nr_cpus); do
> +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> +done

Doesn't this make the first test kind of a subset of this one (where
nr_cpus == 1)? If so, could we just do a couple iterations with
different nr_cpus values?

I'm wondering if we should have a log recovery test as well, btw.

> +for i in $(seq 1 $nr_cpus); do
> +	wait
> +done

Can't we just pass the pids forked by the loop above? Though the manpage
says wait should wait for all child pids as it is, so perhaps the loop
is unnecessary?

Brian

> +
> +# Unmount to prove that we can clean it all
> +echo umount >> $seqres.full
> +before=$(date +%s)
> +_scratch_unmount
> +after=$(date +%s)
> +echo "Unmount took $((after - before))s." >> $seqres.full
> +
> +# Mount so that we can run the usual checks
> +echo silence is golden
> +_scratch_mount
> +status=0
> +exit
> diff --git a/tests/generic/711.out b/tests/generic/711.out
> new file mode 100644
> index 00000000..cbbe36e9
> --- /dev/null
> +++ b/tests/generic/711.out
> @@ -0,0 +1,2 @@
> +QA output created by 711
> +silence is golden
> diff --git a/tests/generic/group b/tests/generic/group
> index f56eb475..26999ca1 100644
> --- a/tests/generic/group
> +++ b/tests/generic/group
> @@ -529,3 +529,5 @@
>  524 auto quick
>  525 auto quick rw
>  709 auto quick
> +710 auto quick unlink
> +711 auto quick unlink
> diff --git a/tests/xfs/736 b/tests/xfs/736
> new file mode 100755
> index 00000000..e33de0ae
> --- /dev/null
> +++ b/tests/xfs/736
> @@ -0,0 +1,71 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 736
> +#
> +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> +# all at once, checking that we don't blow up the filesystem.  This is sort
> +# of a performance test for the xfs unlinked inode backref patchset, but it
> +# applies to most other filesystems.
> +#
> +# Here we force the use of the slow iunlink bucket walk code in a single
> +# threaded situation.
> +#
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +tmp=/tmp/$$
> +status=1	# failure is the default!
> +testfile=$TEST_DIR/$seq.txt
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/attr
> +. ./common/filter
> +. ./common/inject
> +
> +# real QA test starts here
> +_supported_fs generic
> +_supported_os Linux
> +_require_xfs_io_error_injection "iunlink_fallback"
> +_require_scratch
> +
> +rm -f $seqres.full
> +_scratch_mkfs >> $seqres.full 2>&1
> +_scratch_mount
> +
> +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> +# so that this test doesn't take forever or OOM the box
> +max_files=$((30000 * LOAD_FACTOR))
> +max_allowable_files=$(cat /proc/sys/fs/file-max)
> +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> +ulimit -n $max_files
> +
> +# Force xfs to use the iunlinked fallback 50% of the time
> +_scratch_inject_error "iunlink_fallback" "2"
> +
> +# Open a lot of unlinked files
> +echo create >> $seqres.full
> +program=$PWD/src/tmpfile
> +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> +
> +# Unmount to prove that we can clean it all
> +echo umount >> $seqres.full
> +before=$(date +%s)
> +_scratch_unmount
> +after=$(date +%s)
> +echo "Unmount took $((after - before))s." >> $seqres.full
> +
> +# Mount so that we can run the usual checks
> +echo silence is golden
> +_scratch_mount
> +status=0
> +exit
> diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> new file mode 100644
> index 00000000..0258a248
> --- /dev/null
> +++ b/tests/xfs/736.out
> @@ -0,0 +1,2 @@
> +QA output created by 736
> +silence is golden
> diff --git a/tests/xfs/737 b/tests/xfs/737
> new file mode 100755
> index 00000000..47e65607
> --- /dev/null
> +++ b/tests/xfs/737
> @@ -0,0 +1,79 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 737
> +#
> +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> +# all at once, checking that we don't blow up the filesystem.  This is sort
> +# of a performance test for the xfs unlinked inode backref patchset, but it
> +# applies to most other filesystems.
> +#
> +# Here we force the use of the slow iunlink bucket walk code, using every
> +# CPU possible.
> +#
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +tmp=/tmp/$$
> +status=1	# failure is the default!
> +testfile=$TEST_DIR/$seq.txt
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/attr
> +. ./common/filter
> +. ./common/inject
> +
> +# real QA test starts here
> +_supported_fs generic
> +_supported_os Linux
> +_require_xfs_io_error_injection "iunlink_fallback"
> +_require_scratch
> +
> +rm -f $seqres.full
> +_scratch_mkfs >> $seqres.full 2>&1
> +_scratch_mount
> +
> +# Load up all the CPUs, two threads per CPU.
> +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> +
> +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> +# so that this test doesn't take forever or OOM the box
> +max_files=$((30000 * LOAD_FACTOR))
> +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> +ulimit -n $max_files
> +
> +# Force xfs to use the iunlinked fallback 50% of the time
> +_scratch_inject_error "iunlink_fallback" "2"
> +
> +# Open a lot of unlinked files
> +echo create >> $seqres.full
> +program=$PWD/src/tmpfile
> +for i in $(seq 1 $nr_cpus); do
> +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> +done
> +for i in $(seq 1 $nr_cpus); do
> +	wait
> +done
> +
> +# Unmount to prove that we can clean it all
> +echo umount >> $seqres.full
> +before=$(date +%s)
> +_scratch_unmount
> +after=$(date +%s)
> +echo "Unmount took $((after - before))s." >> $seqres.full
> +
> +# Mount so that we can run the usual checks
> +echo silence is golden
> +_scratch_mount
> +status=0
> +exit
> diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> new file mode 100644
> index 00000000..bdc4966d
> --- /dev/null
> +++ b/tests/xfs/737.out
> @@ -0,0 +1,2 @@
> +QA output created by 737
> +silence is golden
> diff --git a/tests/xfs/group b/tests/xfs/group
> index 7b7d69f1..d3189cd5 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -497,3 +497,5 @@
>  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
>  498 dangerous_fuzzers dangerous_norepair
>  499 auto quick
> +736 auto quick unlink
> +737 auto quick unlink
> 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

On Tue, Feb 12, 2019 at 04:42:21AM +0200, Amir Goldstein wrote:
> On Tue, Feb 12, 2019 at 4:18 AM Darrick J. Wong <darrick.wong@oracle.com> wrote:
> >
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> >
> > Create a test (+ helper program) that opens as many unlinked files as it
> > possibly can on the scratch filesystem, then closes all the files at
> > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > make sure that the fallback code doesn't bitrot.
> >
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> >  src/Makefile          |    2 -
> >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/generic/710     |   65 +++++++++++++++++++++++++
> >  tests/generic/710.out |    2 +
> >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> >  tests/generic/711.out |    2 +
> >  tests/generic/group   |    2 +
> >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> >  tests/xfs/736.out     |    2 +
> >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> >  tests/xfs/737.out     |    2 +
> >  tests/xfs/group       |    2 +
> >  12 files changed, 428 insertions(+), 1 deletion(-)
> >  create mode 100644 src/tmpfile.c
> >  create mode 100755 tests/generic/710
> >  create mode 100644 tests/generic/710.out
> >  create mode 100755 tests/generic/711
> >  create mode 100644 tests/generic/711.out
> >  create mode 100755 tests/xfs/736
> >  create mode 100644 tests/xfs/736.out
> >  create mode 100755 tests/xfs/737
> >  create mode 100644 tests/xfs/737.out
> >
> >
> > diff --git a/src/Makefile b/src/Makefile
> > index 41826585..5fce881d 100644
> > --- a/src/Makefile
> > +++ b/src/Makefile
> > @@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
> >         renameat2 t_getcwd e4compact test-nextquota punch-alternating \
> >         attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
> >         dio-invalidate-cache stat_test t_encrypted_d_revalidate \
> > -       attr_replace_test swapon mkswap
> > +       attr_replace_test swapon mkswap tmpfile
> >
> >  SUBDIRS = log-writes perf
> >
> > diff --git a/src/tmpfile.c b/src/tmpfile.c
> 
> Let's go crazy and call it t_open_tmpfiles ?
> Missing .gitignore patch.

Will fix.

> ...
> > +/* Put an opened file on the unlinked list and leak the fd. */
> > +void leak_tmpfile(void)
> > +{
> > +       int fd = -1;
> > +       int ret;
> > +
> > +       /* Try to create an O_TMPFILE and leak the fd. */
> > +#ifdef O_TMPFILE
> > +       fd = open(".", O_TMPFILE | O_RDWR, 0644);
> 
> Pass in path as argv[1], so you won't need to cd $SCRATCH_MNT
> all the time. Or does it have a purpose?

Nope, just sillyness on my part.  Thanks for the suggestion.

--D

> Thanks,
> Amir.

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> > 
> > Create a test (+ helper program) that opens as many unlinked files as it
> > possibly can on the scratch filesystem, then closes all the files at
> > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > make sure that the fallback code doesn't bitrot.
> > 
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> >  src/Makefile          |    2 -
> >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/generic/710     |   65 +++++++++++++++++++++++++
> >  tests/generic/710.out |    2 +
> >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> >  tests/generic/711.out |    2 +
> >  tests/generic/group   |    2 +
> >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> >  tests/xfs/736.out     |    2 +
> >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> >  tests/xfs/737.out     |    2 +
> >  tests/xfs/group       |    2 +
> >  12 files changed, 428 insertions(+), 1 deletion(-)
> >  create mode 100644 src/tmpfile.c
> >  create mode 100755 tests/generic/710
> >  create mode 100644 tests/generic/710.out
> >  create mode 100755 tests/generic/711
> >  create mode 100644 tests/generic/711.out
> >  create mode 100755 tests/xfs/736
> >  create mode 100644 tests/xfs/736.out
> >  create mode 100755 tests/xfs/737
> >  create mode 100644 tests/xfs/737.out
> > 
> > 
> > diff --git a/src/Makefile b/src/Makefile
> > index 41826585..5fce881d 100644
> > --- a/src/Makefile
> > +++ b/src/Makefile
> > @@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
> >  	renameat2 t_getcwd e4compact test-nextquota punch-alternating \
> >  	attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
> >  	dio-invalidate-cache stat_test t_encrypted_d_revalidate \
> > -	attr_replace_test swapon mkswap
> > +	attr_replace_test swapon mkswap tmpfile
> >  
> >  SUBDIRS = log-writes perf
> >  
> > diff --git a/src/tmpfile.c b/src/tmpfile.c
> > new file mode 100644
> > index 00000000..1b74dc72
> > --- /dev/null
> > +++ b/src/tmpfile.c
> > @@ -0,0 +1,127 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * Copyright (C) 2019 Oracle.  All Rights Reserved.
> > + * Author: Darrick J. Wong <darrick.wong@oracle.com>
> > + *
> > + * Test program to open unlinked files and leak them.
> > + */
> > +#ifndef _GNU_SOURCE
> > +# define _GNU_SOURCE
> > +#endif
> > +#include <time.h>
> > +#include <unistd.h>
> > +#include <stdlib.h>
> > +#include <stdio.h>
> > +#include <sys/types.h>
> > +#include <sys/stat.h>
> > +#include <fcntl.h>
> > +#include <errno.h>
> > +
> > +static int min_fd = -1;
> > +static int max_fd = -1;
> > +static unsigned int nr_opened = 0;
> > +static float start_time;
> > +
> > +void clock_time(float *time)
> > +{
> > +	static clockid_t clkid = CLOCK_MONOTONIC;
> > +	struct timespec ts;
> > +	int ret;
> > +
> > +retry:
> > +	ret = clock_gettime(clkid, &ts);
> > +	if (ret) {
> > +		if (clkid == CLOCK_MONOTONIC) {
> > +			clkid = CLOCK_REALTIME;
> > +			goto retry;
> > +		}
> > +		perror("clock_gettime");
> > +		exit(2);
> > +	}
> > +	*time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
> > +}
> > +
> > +/*
> > + * Exit the program due to an error.
> > + *
> > + * If we've exhausted all the file descriptors, make sure we close all the
> > + * open fds in the order we received them in order to exploit a quirk of ext4
> > + * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
> > + * lists, which will make removing the unlinked files maximally painful.
> > + *
> > + * If it's some other error, just die and let the kernel sort it out.
> > + */
> > +void die(void)
> > +{
> > +	float end_time;
> > +	int fd;
> > +
> > +	switch (errno) {
> > +	case EMFILE:
> > +	case ENFILE:
> > +	case ENOSPC:
> > +		clock_time(&end_time);
> > +		printf("Opened %u files in %.2fs.\n", nr_opened,
> > +				end_time - start_time);
> > +		fflush(stdout);
> > +
> > +		clock_time(&start_time);
> > +		for (fd = min_fd; fd <= max_fd; fd++)
> > +			close(fd);
> > +		clock_time(&end_time);
> > +		printf("Closed %u files in %.2fs.\n", nr_opened,
> > +				end_time - start_time);
> > +		exit(0);
> > +		break;
> > +	default:
> > +		perror("open?");
> > +		exit(2);
> > +		break;
> > +	}
> > +}
> > +
> > +/* Remember how many file we open and all that. */
> > +void remember_fd(int fd)
> > +{
> > +	if (min_fd == -1 || min_fd > fd)
> > +		min_fd = fd;
> > +	if (max_fd == -1 || max_fd < fd)
> > +		max_fd = fd;
> > +	nr_opened++;
> > +}
> > +
> > +/* Put an opened file on the unlinked list and leak the fd. */
> > +void leak_tmpfile(void)
> > +{
> > +	int fd = -1;
> > +	int ret;
> > +
> > +	/* Try to create an O_TMPFILE and leak the fd. */
> > +#ifdef O_TMPFILE
> > +	fd = open(".", O_TMPFILE | O_RDWR, 0644);
> > +	if (fd >= 0) {
> > +		remember_fd(fd);
> > +		return;
> > +	}
> > +	if (fd < 0 && errno != EOPNOTSUPP)
> > +		die();
> > +#endif
> 
> Could we track lack of tmpfile support so we don't repeat the open()
> call once we know it's going to fail?

Ok.

> > +
> > +	/* Oh well, create a new file, unlink it, and leak the fd. */
> > +	fd = open("./moo", O_CREAT | O_RDWR, 0644);
> > +	if (fd < 0)
> > +		die();
> > +	ret = unlink("./moo");
> > +	if (ret)
> > +		die();
> > +	remember_fd(fd);
> > +}
> > +
> > +/* Try to put as many files on the unlinked list and then kill them. */
> > +int main(int argc, char *argv[])
> > +{
> > +	clock_time(&start_time);
> > +	while (1)
> > +		leak_tmpfile();
> > +	return 0;
> > +}
> > diff --git a/tests/generic/710 b/tests/generic/710
> > new file mode 100755
> > index 00000000..18aa9d34
> > --- /dev/null
> > +++ b/tests/generic/710
> > @@ -0,0 +1,65 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0+
> > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 710
> > +#
> > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > +# applies to most other filesystems.
> > +#
> > +# Use only a single CPU to test the single threaded situation.
> > +#
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +tmp=/tmp/$$
> > +status=1	# failure is the default!
> > +testfile=$TEST_DIR/$seq.txt
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/attr
> > +. ./common/filter
> 
> Do we need attr/filter?

Nope.

> > +
> > +# real QA test starts here
> > +_supported_fs generic
> > +_supported_os Linux
> > +_require_scratch
> > +
> > +rm -f $seqres.full
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_scratch_mount
> > +
> > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > +# so that this test doesn't take forever or OOM the box
> > +max_files=$((50000 * LOAD_FACTOR))
> > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> 
> I see the following from the above line when I run this test:
> 
> +./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected

Weird... it's supposed to be set based on the amount of RAM you have.

> > +ulimit -n $max_files
> > +
> > +# Open a lot of unlinked files
> > +echo create >> $seqres.full
> > +program=$PWD/src/tmpfile
> > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > +
> > +# Unmount to prove that we can clean it all
> > +echo umount >> $seqres.full
> > +before=$(date +%s)
> > +_scratch_unmount
> > +after=$(date +%s)
> > +echo "Unmount took $((after - before))s." >> $seqres.full
> > +
> > +# Mount so that we can run the usual checks
> > +echo silence is golden
> > +_scratch_mount
> > +status=0
> > +exit
> > diff --git a/tests/generic/710.out b/tests/generic/710.out
> > new file mode 100644
> > index 00000000..e0a55170
> > --- /dev/null
> > +++ b/tests/generic/710.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 710
> > +silence is golden
> > diff --git a/tests/generic/711 b/tests/generic/711
> > new file mode 100755
> > index 00000000..11d76218
> > --- /dev/null
> > +++ b/tests/generic/711
> > @@ -0,0 +1,73 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0+
> > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 711
> > +#
> > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > +# applies to most other filesystems.
> > +#
> > +# Use every CPU possible to stress the filesystem.
> > +#
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +tmp=/tmp/$$
> > +status=1	# failure is the default!
> > +testfile=$TEST_DIR/$seq.txt
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/attr
> > +. ./common/filter
> > +
> > +# real QA test starts here
> > +_supported_fs generic
> > +_supported_os Linux
> > +_require_scratch
> > +
> > +rm -f $seqres.full
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_scratch_mount
> > +
> > +# Try to load up all the CPUs, two threads per CPU.
> > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > +
> > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > +# so that this test doesn't take forever or OOM the box
> > +max_files=$((50000 * LOAD_FACTOR))
> > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > +ulimit -n $max_files
> > +
> > +# Open a lot of unlinked files
> > +echo create >> $seqres.full
> > +program=$PWD/src/tmpfile
> > +for i in $(seq 1 $nr_cpus); do
> > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > +done
> 
> Doesn't this make the first test kind of a subset of this one (where
> nr_cpus == 1)? If so, could we just do a couple iterations with
> different nr_cpus values?

I think it's fine only to have the multithreaded version, that should
stress the AGs well enough.

> I'm wondering if we should have a log recovery test as well, btw.

Yes.  I'll turn g/710 and x/736 into the log recovery tests.

(Oh wow flood of asserts this is going to take a while to straighten
out)

> > +for i in $(seq 1 $nr_cpus); do
> > +	wait
> > +done
> 
> Can't we just pass the pids forked by the loop above? Though the manpage
> says wait should wait for all child pids as it is, so perhaps the loop
> is unnecessary?

Oh, I did not know that.  Thanks for the review!

> Brian
> 
> > +
> > +# Unmount to prove that we can clean it all
> > +echo umount >> $seqres.full
> > +before=$(date +%s)
> > +_scratch_unmount
> > +after=$(date +%s)
> > +echo "Unmount took $((after - before))s." >> $seqres.full
> > +
> > +# Mount so that we can run the usual checks
> > +echo silence is golden
> > +_scratch_mount
> > +status=0
> > +exit
> > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > new file mode 100644
> > index 00000000..cbbe36e9
> > --- /dev/null
> > +++ b/tests/generic/711.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 711
> > +silence is golden
> > diff --git a/tests/generic/group b/tests/generic/group
> > index f56eb475..26999ca1 100644
> > --- a/tests/generic/group
> > +++ b/tests/generic/group
> > @@ -529,3 +529,5 @@
> >  524 auto quick
> >  525 auto quick rw
> >  709 auto quick
> > +710 auto quick unlink
> > +711 auto quick unlink
> > diff --git a/tests/xfs/736 b/tests/xfs/736
> > new file mode 100755
> > index 00000000..e33de0ae
> > --- /dev/null
> > +++ b/tests/xfs/736
> > @@ -0,0 +1,71 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0+
> > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 736
> > +#
> > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > +# applies to most other filesystems.
> > +#
> > +# Here we force the use of the slow iunlink bucket walk code in a single
> > +# threaded situation.
> > +#
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +tmp=/tmp/$$
> > +status=1	# failure is the default!
> > +testfile=$TEST_DIR/$seq.txt
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/attr
> > +. ./common/filter
> > +. ./common/inject
> > +
> > +# real QA test starts here
> > +_supported_fs generic
> > +_supported_os Linux
> > +_require_xfs_io_error_injection "iunlink_fallback"
> > +_require_scratch
> > +
> > +rm -f $seqres.full
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_scratch_mount
> > +
> > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > +# so that this test doesn't take forever or OOM the box
> > +max_files=$((30000 * LOAD_FACTOR))
> > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > +ulimit -n $max_files
> > +
> > +# Force xfs to use the iunlinked fallback 50% of the time
> > +_scratch_inject_error "iunlink_fallback" "2"
> > +
> > +# Open a lot of unlinked files
> > +echo create >> $seqres.full
> > +program=$PWD/src/tmpfile
> > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > +
> > +# Unmount to prove that we can clean it all
> > +echo umount >> $seqres.full
> > +before=$(date +%s)
> > +_scratch_unmount
> > +after=$(date +%s)
> > +echo "Unmount took $((after - before))s." >> $seqres.full
> > +
> > +# Mount so that we can run the usual checks
> > +echo silence is golden
> > +_scratch_mount
> > +status=0
> > +exit
> > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > new file mode 100644
> > index 00000000..0258a248
> > --- /dev/null
> > +++ b/tests/xfs/736.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 736
> > +silence is golden
> > diff --git a/tests/xfs/737 b/tests/xfs/737
> > new file mode 100755
> > index 00000000..47e65607
> > --- /dev/null
> > +++ b/tests/xfs/737
> > @@ -0,0 +1,79 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0+
> > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 737
> > +#
> > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > +# applies to most other filesystems.
> > +#
> > +# Here we force the use of the slow iunlink bucket walk code, using every
> > +# CPU possible.
> > +#
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +tmp=/tmp/$$
> > +status=1	# failure is the default!
> > +testfile=$TEST_DIR/$seq.txt
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/attr
> > +. ./common/filter
> > +. ./common/inject
> > +
> > +# real QA test starts here
> > +_supported_fs generic
> > +_supported_os Linux
> > +_require_xfs_io_error_injection "iunlink_fallback"
> > +_require_scratch
> > +
> > +rm -f $seqres.full
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_scratch_mount
> > +
> > +# Load up all the CPUs, two threads per CPU.
> > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > +
> > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > +# so that this test doesn't take forever or OOM the box
> > +max_files=$((30000 * LOAD_FACTOR))
> > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > +ulimit -n $max_files
> > +
> > +# Force xfs to use the iunlinked fallback 50% of the time
> > +_scratch_inject_error "iunlink_fallback" "2"
> > +
> > +# Open a lot of unlinked files
> > +echo create >> $seqres.full
> > +program=$PWD/src/tmpfile
> > +for i in $(seq 1 $nr_cpus); do
> > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > +done
> > +for i in $(seq 1 $nr_cpus); do
> > +	wait
> > +done
> > +
> > +# Unmount to prove that we can clean it all
> > +echo umount >> $seqres.full
> > +before=$(date +%s)
> > +_scratch_unmount
> > +after=$(date +%s)
> > +echo "Unmount took $((after - before))s." >> $seqres.full
> > +
> > +# Mount so that we can run the usual checks
> > +echo silence is golden
> > +_scratch_mount
> > +status=0
> > +exit
> > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > new file mode 100644
> > index 00000000..bdc4966d
> > --- /dev/null
> > +++ b/tests/xfs/737.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 737
> > +silence is golden
> > diff --git a/tests/xfs/group b/tests/xfs/group
> > index 7b7d69f1..d3189cd5 100644
> > --- a/tests/xfs/group
> > +++ b/tests/xfs/group
> > @@ -497,3 +497,5 @@
> >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> >  498 dangerous_fuzzers dangerous_norepair
> >  499 auto quick
> > +736 auto quick unlink
> > +737 auto quick unlink
> > 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

On Tue, Feb 12, 2019 at 08:51:19PM -0800, Darrick J. Wong wrote:
> On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> > On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > > From: Darrick J. Wong <darrick.wong@oracle.com>
> > > 
> > > Create a test (+ helper program) that opens as many unlinked files as it
> > > possibly can on the scratch filesystem, then closes all the files at
> > > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > > make sure that the fallback code doesn't bitrot.
> > > 
> > > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > > ---
> > >  src/Makefile          |    2 -
> > >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> > >  tests/generic/710     |   65 +++++++++++++++++++++++++
> > >  tests/generic/710.out |    2 +
> > >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> > >  tests/generic/711.out |    2 +
> > >  tests/generic/group   |    2 +
> > >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> > >  tests/xfs/736.out     |    2 +
> > >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> > >  tests/xfs/737.out     |    2 +
> > >  tests/xfs/group       |    2 +
> > >  12 files changed, 428 insertions(+), 1 deletion(-)
> > >  create mode 100644 src/tmpfile.c
> > >  create mode 100755 tests/generic/710
> > >  create mode 100644 tests/generic/710.out
> > >  create mode 100755 tests/generic/711
> > >  create mode 100644 tests/generic/711.out
> > >  create mode 100755 tests/xfs/736
> > >  create mode 100644 tests/xfs/736.out
> > >  create mode 100755 tests/xfs/737
> > >  create mode 100644 tests/xfs/737.out
> > > 
> > > 
> > > diff --git a/src/Makefile b/src/Makefile
> > > index 41826585..5fce881d 100644
> > > --- a/src/Makefile
> > > +++ b/src/Makefile
> > > @@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
> > >  	renameat2 t_getcwd e4compact test-nextquota punch-alternating \
> > >  	attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
> > >  	dio-invalidate-cache stat_test t_encrypted_d_revalidate \
> > > -	attr_replace_test swapon mkswap
> > > +	attr_replace_test swapon mkswap tmpfile
> > >  
> > >  SUBDIRS = log-writes perf
> > >  
> > > diff --git a/src/tmpfile.c b/src/tmpfile.c
> > > new file mode 100644
> > > index 00000000..1b74dc72
> > > --- /dev/null
> > > +++ b/src/tmpfile.c
> > > @@ -0,0 +1,127 @@
> > > +// SPDX-License-Identifier: GPL-2.0+
> > > +/*
> > > + * Copyright (C) 2019 Oracle.  All Rights Reserved.
> > > + * Author: Darrick J. Wong <darrick.wong@oracle.com>
> > > + *
> > > + * Test program to open unlinked files and leak them.
> > > + */
> > > +#ifndef _GNU_SOURCE
> > > +# define _GNU_SOURCE
> > > +#endif
> > > +#include <time.h>
> > > +#include <unistd.h>
> > > +#include <stdlib.h>
> > > +#include <stdio.h>
> > > +#include <sys/types.h>
> > > +#include <sys/stat.h>
> > > +#include <fcntl.h>
> > > +#include <errno.h>
> > > +
> > > +static int min_fd = -1;
> > > +static int max_fd = -1;
> > > +static unsigned int nr_opened = 0;
> > > +static float start_time;
> > > +
> > > +void clock_time(float *time)
> > > +{
> > > +	static clockid_t clkid = CLOCK_MONOTONIC;
> > > +	struct timespec ts;
> > > +	int ret;
> > > +
> > > +retry:
> > > +	ret = clock_gettime(clkid, &ts);
> > > +	if (ret) {
> > > +		if (clkid == CLOCK_MONOTONIC) {
> > > +			clkid = CLOCK_REALTIME;
> > > +			goto retry;
> > > +		}
> > > +		perror("clock_gettime");
> > > +		exit(2);
> > > +	}
> > > +	*time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
> > > +}
> > > +
> > > +/*
> > > + * Exit the program due to an error.
> > > + *
> > > + * If we've exhausted all the file descriptors, make sure we close all the
> > > + * open fds in the order we received them in order to exploit a quirk of ext4
> > > + * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
> > > + * lists, which will make removing the unlinked files maximally painful.
> > > + *
> > > + * If it's some other error, just die and let the kernel sort it out.
> > > + */
> > > +void die(void)
> > > +{
> > > +	float end_time;
> > > +	int fd;
> > > +
> > > +	switch (errno) {
> > > +	case EMFILE:
> > > +	case ENFILE:
> > > +	case ENOSPC:
> > > +		clock_time(&end_time);
> > > +		printf("Opened %u files in %.2fs.\n", nr_opened,
> > > +				end_time - start_time);
> > > +		fflush(stdout);
> > > +
> > > +		clock_time(&start_time);
> > > +		for (fd = min_fd; fd <= max_fd; fd++)
> > > +			close(fd);
> > > +		clock_time(&end_time);
> > > +		printf("Closed %u files in %.2fs.\n", nr_opened,
> > > +				end_time - start_time);
> > > +		exit(0);
> > > +		break;
> > > +	default:
> > > +		perror("open?");
> > > +		exit(2);
> > > +		break;
> > > +	}
> > > +}
> > > +
> > > +/* Remember how many file we open and all that. */
> > > +void remember_fd(int fd)
> > > +{
> > > +	if (min_fd == -1 || min_fd > fd)
> > > +		min_fd = fd;
> > > +	if (max_fd == -1 || max_fd < fd)
> > > +		max_fd = fd;
> > > +	nr_opened++;
> > > +}
> > > +
> > > +/* Put an opened file on the unlinked list and leak the fd. */
> > > +void leak_tmpfile(void)
> > > +{
> > > +	int fd = -1;
> > > +	int ret;
> > > +
> > > +	/* Try to create an O_TMPFILE and leak the fd. */
> > > +#ifdef O_TMPFILE
> > > +	fd = open(".", O_TMPFILE | O_RDWR, 0644);
> > > +	if (fd >= 0) {
> > > +		remember_fd(fd);
> > > +		return;
> > > +	}
> > > +	if (fd < 0 && errno != EOPNOTSUPP)
> > > +		die();
> > > +#endif
> > 
> > Could we track lack of tmpfile support so we don't repeat the open()
> > call once we know it's going to fail?
> 
> Ok.
> 
> > > +
> > > +	/* Oh well, create a new file, unlink it, and leak the fd. */
> > > +	fd = open("./moo", O_CREAT | O_RDWR, 0644);
> > > +	if (fd < 0)
> > > +		die();
> > > +	ret = unlink("./moo");
> > > +	if (ret)
> > > +		die();
> > > +	remember_fd(fd);
> > > +}
> > > +
> > > +/* Try to put as many files on the unlinked list and then kill them. */
> > > +int main(int argc, char *argv[])
> > > +{
> > > +	clock_time(&start_time);
> > > +	while (1)
> > > +		leak_tmpfile();
> > > +	return 0;
> > > +}
> > > diff --git a/tests/generic/710 b/tests/generic/710
> > > new file mode 100755
> > > index 00000000..18aa9d34
> > > --- /dev/null
> > > +++ b/tests/generic/710
> > > @@ -0,0 +1,65 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 710
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Use only a single CPU to test the single threaded situation.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > 
> > Do we need attr/filter?
> 
> Nope.
> 
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((50000 * LOAD_FACTOR))
> > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > 
> > I see the following from the above line when I run this test:
> > 
> > +./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected
> 
> Weird... it's supposed to be set based on the amount of RAM you have.
> 
> > > +ulimit -n $max_files
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/generic/710.out b/tests/generic/710.out
> > > new file mode 100644
> > > index 00000000..e0a55170
> > > --- /dev/null
> > > +++ b/tests/generic/710.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 710
> > > +silence is golden
> > > diff --git a/tests/generic/711 b/tests/generic/711
> > > new file mode 100755
> > > index 00000000..11d76218
> > > --- /dev/null
> > > +++ b/tests/generic/711
> > > @@ -0,0 +1,73 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 711
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Use every CPU possible to stress the filesystem.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Try to load up all the CPUs, two threads per CPU.
> > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((50000 * LOAD_FACTOR))
> > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > +done
> > 
> > Doesn't this make the first test kind of a subset of this one (where
> > nr_cpus == 1)? If so, could we just do a couple iterations with
> > different nr_cpus values?
> 
> I think it's fine only to have the multithreaded version, that should
> stress the AGs well enough.
> 
> > I'm wondering if we should have a log recovery test as well, btw.
> 
> Yes.  I'll turn g/710 and x/736 into the log recovery tests.
> 
> (Oh wow flood of asserts this is going to take a while to straighten
> out)

NFI what this is about, but here's the crash:

XFS: Assertion failed: VFS_I(ip)->i_nlink == 0, file: fs/xfs/xfs_log_recover.c, line: 5072
WARNING: CPU: 0 PID: 2936 at fs/xfs/xfs_message.c:104 assfail+0x27/0x2a [xfs]
Modules linked in: xfs libcrc32c bfq dax_pmem nd_pmem device_dax sch_fq_codel ip_tables x_tables nfsv4 af_packet
CPU: 0 PID: 2936 Comm: mount Tainted: G        W         4.20.0-rc6-djw #rc6
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.10.2-1ubuntu1 04/01/2014
RIP: 0010:assfail+0x27/0x2a [xfs]
Code: 0f 0b c3 0f 1f 44 00 00 48 89 f1 41 89 d0 48 c7 c6 50 aa 34 a0 48 89 fa 31 ff e8 6d f9 ff ff 80 3d 9c 58 0a 00 00 74 02 0f 0b <0f> 0b c3 48 8b b3 a8 02 00 00 48 c7 c7 c8 ae 34 a0 c6 05 08 54 0a
RSP: 0018:ffffc90003607c90 EFLAGS: 00010246
RAX: 0000000000000000 RBX: ffff88802f763a80 RCX: 0000000000000000
RDX: 00000000ffffffc0 RSI: 000000000000000a RDI: ffffffffa033ce6f
RBP: ffff88802f763c50 R08: 0000000000000000 R09: 0000000000000000
R10: 000000000000000a R11: f000000000000000 R12: 0000000000000000
R13: 0000000000000000 R14: ffff88803d6b5000 R15: 0000000000000000
FS:  00007f67fe2ea080(0000) GS:ffff88803e800000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055ac0c1ec1d8 CR3: 000000003d47e006 CR4: 00000000001606b0
Call Trace:
 xlog_recover_process_one_iunlink+0x15e/0x170 [xfs]
 xlog_recover_process_iunlinks.isra.50+0x7c/0xc0 [xfs]
 xlog_recover_finish+0x33/0xa0 [xfs]
 xfs_log_mount_finish+0x5f/0x100 [xfs]
 xfs_mountfs+0x568/0x990 [xfs]
 ? xfs_mru_cache_create+0x172/0x1d0 [xfs]
 xfs_fs_fill_super+0x4d1/0x6d0 [xfs]
 ? xfs_test_remount_options+0x60/0x60 [xfs]
 mount_bdev+0x17f/0x1b0
 mount_fs+0x15/0x7d
 vfs_kern_mount.part.43+0x54/0x160
 do_mount+0x1d2/0xd90
 ? memdup_user+0x4b/0x70
 ksys_mount+0xba/0xd0
 __x64_sys_mount+0x21/0x30
 do_syscall_64+0x50/0x160
 entry_SYSCALL_64_after_hwframe+0x49/0xbe
RIP: 0033:0x7f67fdbad3ca
Code: 48 8b 0d c1 8a 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 49 89 ca b8 a5 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 8e 8a 2c 00 f7 d8 64 89 01 48
RSP: 002b:00007fff05981278 EFLAGS: 00000206 ORIG_RAX: 00000000000000a5
RAX: ffffffffffffffda RBX: 000055ac0c1e4a40 RCX: 00007f67fdbad3ca
RDX: 000055ac0c1ebd90 RSI: 000055ac0c1e4c40 RDI: 000055ac0c1e4c20
RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
R10: 00000000c0ed0000 R11: 0000000000000206 R12: 000055ac0c1e4c20
R13: 000055ac0c1ebd90 R14: 0000000000000000 R15: 00007f67fe0ce8a4
irq event stamp: 60960
hardirqs last  enabled at (60959): [<ffffffff810cac15>] console_unlock+0x435/0x5e0
hardirqs last disabled at (60960): [<ffffffff81001ba0>] trace_hardirqs_off_thunk+0x1a/0x1c
softirqs last  enabled at (60956): [<ffffffff81a003a8>] __do_softirq+0x3a8/0x4bd
softirqs last disabled at (60943): [<ffffffff8106166c>] irq_exit+0xbc/0xe0
---[ end trace f0ce4aa6d3ed581f ]---

<over and over and over>

XFS (pmem4): xfs_do_force_shutdown(0x8) called from line 368 of file fs/xfs/xfs_trans.c. Return address = ffffffffa02edf93
XFS (pmem4): Corruption of in-memory data detected.  Shutting down filesystem
XFS (pmem4): Please unmount the filesystem and rectify the problem(s)
XFS (pmem4): xfs_imap_to_bp: xfs_trans_read_buf() returned error -5.
XFS (pmem4): xlog_recover_clear_agi_bucket: failed to clear agi 0. Continuing.

<over and over and over>

Reproduces on scsi disk too.  Nightnight.

--D

> > > +for i in $(seq 1 $nr_cpus); do
> > > +	wait
> > > +done
> > 
> > Can't we just pass the pids forked by the loop above? Though the manpage
> > says wait should wait for all child pids as it is, so perhaps the loop
> > is unnecessary?
> 
> Oh, I did not know that.  Thanks for the review!
> 
> > Brian
> > 
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > > new file mode 100644
> > > index 00000000..cbbe36e9
> > > --- /dev/null
> > > +++ b/tests/generic/711.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 711
> > > +silence is golden
> > > diff --git a/tests/generic/group b/tests/generic/group
> > > index f56eb475..26999ca1 100644
> > > --- a/tests/generic/group
> > > +++ b/tests/generic/group
> > > @@ -529,3 +529,5 @@
> > >  524 auto quick
> > >  525 auto quick rw
> > >  709 auto quick
> > > +710 auto quick unlink
> > > +711 auto quick unlink
> > > diff --git a/tests/xfs/736 b/tests/xfs/736
> > > new file mode 100755
> > > index 00000000..e33de0ae
> > > --- /dev/null
> > > +++ b/tests/xfs/736
> > > @@ -0,0 +1,71 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 736
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Here we force the use of the slow iunlink bucket walk code in a single
> > > +# threaded situation.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +. ./common/inject
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((30000 * LOAD_FACTOR))
> > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > +_scratch_inject_error "iunlink_fallback" "2"
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > > new file mode 100644
> > > index 00000000..0258a248
> > > --- /dev/null
> > > +++ b/tests/xfs/736.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 736
> > > +silence is golden
> > > diff --git a/tests/xfs/737 b/tests/xfs/737
> > > new file mode 100755
> > > index 00000000..47e65607
> > > --- /dev/null
> > > +++ b/tests/xfs/737
> > > @@ -0,0 +1,79 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 737
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Here we force the use of the slow iunlink bucket walk code, using every
> > > +# CPU possible.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +. ./common/inject
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Load up all the CPUs, two threads per CPU.
> > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((30000 * LOAD_FACTOR))
> > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > +_scratch_inject_error "iunlink_fallback" "2"
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > +done
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	wait
> > > +done
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > > new file mode 100644
> > > index 00000000..bdc4966d
> > > --- /dev/null
> > > +++ b/tests/xfs/737.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 737
> > > +silence is golden
> > > diff --git a/tests/xfs/group b/tests/xfs/group
> > > index 7b7d69f1..d3189cd5 100644
> > > --- a/tests/xfs/group
> > > +++ b/tests/xfs/group
> > > @@ -497,3 +497,5 @@
> > >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> > >  498 dangerous_fuzzers dangerous_norepair
> > >  499 auto quick
> > > +736 auto quick unlink
> > > +737 auto quick unlink
> > > 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Brian Foster]

On Tue, Feb 12, 2019 at 08:51:19PM -0800, Darrick J. Wong wrote:
> On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> > On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > > From: Darrick J. Wong <darrick.wong@oracle.com>
> > > 
> > > Create a test (+ helper program) that opens as many unlinked files as it
> > > possibly can on the scratch filesystem, then closes all the files at
> > > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > > make sure that the fallback code doesn't bitrot.
> > > 
> > > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > > ---
> > >  src/Makefile          |    2 -
> > >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> > >  tests/generic/710     |   65 +++++++++++++++++++++++++
> > >  tests/generic/710.out |    2 +
> > >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> > >  tests/generic/711.out |    2 +
> > >  tests/generic/group   |    2 +
> > >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> > >  tests/xfs/736.out     |    2 +
> > >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> > >  tests/xfs/737.out     |    2 +
> > >  tests/xfs/group       |    2 +
> > >  12 files changed, 428 insertions(+), 1 deletion(-)
> > >  create mode 100644 src/tmpfile.c
> > >  create mode 100755 tests/generic/710
> > >  create mode 100644 tests/generic/710.out
> > >  create mode 100755 tests/generic/711
> > >  create mode 100644 tests/generic/711.out
> > >  create mode 100755 tests/xfs/736
> > >  create mode 100644 tests/xfs/736.out
> > >  create mode 100755 tests/xfs/737
> > >  create mode 100644 tests/xfs/737.out
> > > 
> > > 
...
> > > diff --git a/tests/generic/710 b/tests/generic/710
> > > new file mode 100755
> > > index 00000000..18aa9d34
> > > --- /dev/null
> > > +++ b/tests/generic/710
> > > @@ -0,0 +1,65 @@
...
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((50000 * LOAD_FACTOR))
> > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > 
> > I see the following from the above line when I run this test:
> > 
> > +./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected
> 
> Weird... it's supposed to be set based on the amount of RAM you have.
> 

Strange indeed. I added some tracepoints that show it's initialized to
something reasonable on boot. I added another to determine what might be
changing it to -1:

          <idle>-0     [000] ...1     0.630877: files_maxfiles_init: 386: nr_pages 996312 nr_free_pages 994058 memreserve 3381
          <idle>-0     [000] ...1     0.630879: files_maxfiles_init: 392: memreserve 3381 n 397172 NR_FILE 8192 max_files 397172
         systemd-1     [002] ....     2.732128: proc_filemax: 2879: max_files 18446744073709551615
         systemd-1     [000] ....     9.303595: proc_filemax: 2879: max_files 18446744073709551615

So systemd is changing it for some reason. *shrug* not sure it really
matters that much, but it might be worth massaging the input to handle
the max value somehow or another.

Brian

> > > +ulimit -n $max_files
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/generic/710.out b/tests/generic/710.out
> > > new file mode 100644
> > > index 00000000..e0a55170
> > > --- /dev/null
> > > +++ b/tests/generic/710.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 710
> > > +silence is golden
> > > diff --git a/tests/generic/711 b/tests/generic/711
> > > new file mode 100755
> > > index 00000000..11d76218
> > > --- /dev/null
> > > +++ b/tests/generic/711
> > > @@ -0,0 +1,73 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 711
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Use every CPU possible to stress the filesystem.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Try to load up all the CPUs, two threads per CPU.
> > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((50000 * LOAD_FACTOR))
> > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > +done
> > 
> > Doesn't this make the first test kind of a subset of this one (where
> > nr_cpus == 1)? If so, could we just do a couple iterations with
> > different nr_cpus values?
> 
> I think it's fine only to have the multithreaded version, that should
> stress the AGs well enough.
> 
> > I'm wondering if we should have a log recovery test as well, btw.
> 
> Yes.  I'll turn g/710 and x/736 into the log recovery tests.
> 
> (Oh wow flood of asserts this is going to take a while to straighten
> out)
> 
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	wait
> > > +done
> > 
> > Can't we just pass the pids forked by the loop above? Though the manpage
> > says wait should wait for all child pids as it is, so perhaps the loop
> > is unnecessary?
> 
> Oh, I did not know that.  Thanks for the review!
> 
> > Brian
> > 
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > > new file mode 100644
> > > index 00000000..cbbe36e9
> > > --- /dev/null
> > > +++ b/tests/generic/711.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 711
> > > +silence is golden
> > > diff --git a/tests/generic/group b/tests/generic/group
> > > index f56eb475..26999ca1 100644
> > > --- a/tests/generic/group
> > > +++ b/tests/generic/group
> > > @@ -529,3 +529,5 @@
> > >  524 auto quick
> > >  525 auto quick rw
> > >  709 auto quick
> > > +710 auto quick unlink
> > > +711 auto quick unlink
> > > diff --git a/tests/xfs/736 b/tests/xfs/736
> > > new file mode 100755
> > > index 00000000..e33de0ae
> > > --- /dev/null
> > > +++ b/tests/xfs/736
> > > @@ -0,0 +1,71 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 736
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Here we force the use of the slow iunlink bucket walk code in a single
> > > +# threaded situation.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +. ./common/inject
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((30000 * LOAD_FACTOR))
> > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > +_scratch_inject_error "iunlink_fallback" "2"
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > > new file mode 100644
> > > index 00000000..0258a248
> > > --- /dev/null
> > > +++ b/tests/xfs/736.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 736
> > > +silence is golden
> > > diff --git a/tests/xfs/737 b/tests/xfs/737
> > > new file mode 100755
> > > index 00000000..47e65607
> > > --- /dev/null
> > > +++ b/tests/xfs/737
> > > @@ -0,0 +1,79 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 737
> > > +#
> > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > +# applies to most other filesystems.
> > > +#
> > > +# Here we force the use of the slow iunlink bucket walk code, using every
> > > +# CPU possible.
> > > +#
> > > +seq=`basename $0`
> > > +seqres=$RESULT_DIR/$seq
> > > +echo "QA output created by $seq"
> > > +tmp=/tmp/$$
> > > +status=1	# failure is the default!
> > > +testfile=$TEST_DIR/$seq.txt
> > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	rm -f $tmp.*
> > > +}
> > > +
> > > +# get standard environment, filters and checks
> > > +. ./common/rc
> > > +. ./common/attr
> > > +. ./common/filter
> > > +. ./common/inject
> > > +
> > > +# real QA test starts here
> > > +_supported_fs generic
> > > +_supported_os Linux
> > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > +_require_scratch
> > > +
> > > +rm -f $seqres.full
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_scratch_mount
> > > +
> > > +# Load up all the CPUs, two threads per CPU.
> > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > +
> > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > > +# so that this test doesn't take forever or OOM the box
> > > +max_files=$((30000 * LOAD_FACTOR))
> > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > +ulimit -n $max_files
> > > +
> > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > +_scratch_inject_error "iunlink_fallback" "2"
> > > +
> > > +# Open a lot of unlinked files
> > > +echo create >> $seqres.full
> > > +program=$PWD/src/tmpfile
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > +done
> > > +for i in $(seq 1 $nr_cpus); do
> > > +	wait
> > > +done
> > > +
> > > +# Unmount to prove that we can clean it all
> > > +echo umount >> $seqres.full
> > > +before=$(date +%s)
> > > +_scratch_unmount
> > > +after=$(date +%s)
> > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > +
> > > +# Mount so that we can run the usual checks
> > > +echo silence is golden
> > > +_scratch_mount
> > > +status=0
> > > +exit
> > > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > > new file mode 100644
> > > index 00000000..bdc4966d
> > > --- /dev/null
> > > +++ b/tests/xfs/737.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 737
> > > +silence is golden
> > > diff --git a/tests/xfs/group b/tests/xfs/group
> > > index 7b7d69f1..d3189cd5 100644
> > > --- a/tests/xfs/group
> > > +++ b/tests/xfs/group
> > > @@ -497,3 +497,5 @@
> > >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> > >  498 dangerous_fuzzers dangerous_norepair
> > >  499 auto quick
> > > +736 auto quick unlink
> > > +737 auto quick unlink
> > > 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Brian Foster]

On Tue, Feb 12, 2019 at 09:11:42PM -0800, Darrick J. Wong wrote:
> On Tue, Feb 12, 2019 at 08:51:19PM -0800, Darrick J. Wong wrote:
> > On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> > > On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > > > From: Darrick J. Wong <darrick.wong@oracle.com>
> > > > 
> > > > Create a test (+ helper program) that opens as many unlinked files as it
> > > > possibly can on the scratch filesystem, then closes all the files at
> > > > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > > > make sure that the fallback code doesn't bitrot.
> > > > 
> > > > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > > > ---
> > > >  src/Makefile          |    2 -
> > > >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> > > >  tests/generic/710     |   65 +++++++++++++++++++++++++
> > > >  tests/generic/710.out |    2 +
> > > >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> > > >  tests/generic/711.out |    2 +
> > > >  tests/generic/group   |    2 +
> > > >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> > > >  tests/xfs/736.out     |    2 +
> > > >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> > > >  tests/xfs/737.out     |    2 +
> > > >  tests/xfs/group       |    2 +
> > > >  12 files changed, 428 insertions(+), 1 deletion(-)
> > > >  create mode 100644 src/tmpfile.c
> > > >  create mode 100755 tests/generic/710
> > > >  create mode 100644 tests/generic/710.out
> > > >  create mode 100755 tests/generic/711
> > > >  create mode 100644 tests/generic/711.out
> > > >  create mode 100755 tests/xfs/736
> > > >  create mode 100644 tests/xfs/736.out
> > > >  create mode 100755 tests/xfs/737
> > > >  create mode 100644 tests/xfs/737.out
> > > > 
> > > > 
...
> > 
> > > I'm wondering if we should have a log recovery test as well, btw.
> > 
> > Yes.  I'll turn g/710 and x/736 into the log recovery tests.
> > 
> > (Oh wow flood of asserts this is going to take a while to straighten
> > out)
> 
> NFI what this is about, but here's the crash:
> 
> XFS: Assertion failed: VFS_I(ip)->i_nlink == 0, file: fs/xfs/xfs_log_recover.c, line: 5072

Heh, we somehow end up with (what looks like) a linked inode on the
unlinked list..?

Brian

> WARNING: CPU: 0 PID: 2936 at fs/xfs/xfs_message.c:104 assfail+0x27/0x2a [xfs]
> Modules linked in: xfs libcrc32c bfq dax_pmem nd_pmem device_dax sch_fq_codel ip_tables x_tables nfsv4 af_packet
> CPU: 0 PID: 2936 Comm: mount Tainted: G        W         4.20.0-rc6-djw #rc6
> Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.10.2-1ubuntu1 04/01/2014
> RIP: 0010:assfail+0x27/0x2a [xfs]
> Code: 0f 0b c3 0f 1f 44 00 00 48 89 f1 41 89 d0 48 c7 c6 50 aa 34 a0 48 89 fa 31 ff e8 6d f9 ff ff 80 3d 9c 58 0a 00 00 74 02 0f 0b <0f> 0b c3 48 8b b3 a8 02 00 00 48 c7 c7 c8 ae 34 a0 c6 05 08 54 0a
> RSP: 0018:ffffc90003607c90 EFLAGS: 00010246
> RAX: 0000000000000000 RBX: ffff88802f763a80 RCX: 0000000000000000
> RDX: 00000000ffffffc0 RSI: 000000000000000a RDI: ffffffffa033ce6f
> RBP: ffff88802f763c50 R08: 0000000000000000 R09: 0000000000000000
> R10: 000000000000000a R11: f000000000000000 R12: 0000000000000000
> R13: 0000000000000000 R14: ffff88803d6b5000 R15: 0000000000000000
> FS:  00007f67fe2ea080(0000) GS:ffff88803e800000(0000) knlGS:0000000000000000
> CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
> CR2: 000055ac0c1ec1d8 CR3: 000000003d47e006 CR4: 00000000001606b0
> Call Trace:
>  xlog_recover_process_one_iunlink+0x15e/0x170 [xfs]
>  xlog_recover_process_iunlinks.isra.50+0x7c/0xc0 [xfs]
>  xlog_recover_finish+0x33/0xa0 [xfs]
>  xfs_log_mount_finish+0x5f/0x100 [xfs]
>  xfs_mountfs+0x568/0x990 [xfs]
>  ? xfs_mru_cache_create+0x172/0x1d0 [xfs]
>  xfs_fs_fill_super+0x4d1/0x6d0 [xfs]
>  ? xfs_test_remount_options+0x60/0x60 [xfs]
>  mount_bdev+0x17f/0x1b0
>  mount_fs+0x15/0x7d
>  vfs_kern_mount.part.43+0x54/0x160
>  do_mount+0x1d2/0xd90
>  ? memdup_user+0x4b/0x70
>  ksys_mount+0xba/0xd0
>  __x64_sys_mount+0x21/0x30
>  do_syscall_64+0x50/0x160
>  entry_SYSCALL_64_after_hwframe+0x49/0xbe
> RIP: 0033:0x7f67fdbad3ca
> Code: 48 8b 0d c1 8a 2c 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 49 89 ca b8 a5 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 8e 8a 2c 00 f7 d8 64 89 01 48
> RSP: 002b:00007fff05981278 EFLAGS: 00000206 ORIG_RAX: 00000000000000a5
> RAX: ffffffffffffffda RBX: 000055ac0c1e4a40 RCX: 00007f67fdbad3ca
> RDX: 000055ac0c1ebd90 RSI: 000055ac0c1e4c40 RDI: 000055ac0c1e4c20
> RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
> R10: 00000000c0ed0000 R11: 0000000000000206 R12: 000055ac0c1e4c20
> R13: 000055ac0c1ebd90 R14: 0000000000000000 R15: 00007f67fe0ce8a4
> irq event stamp: 60960
> hardirqs last  enabled at (60959): [<ffffffff810cac15>] console_unlock+0x435/0x5e0
> hardirqs last disabled at (60960): [<ffffffff81001ba0>] trace_hardirqs_off_thunk+0x1a/0x1c
> softirqs last  enabled at (60956): [<ffffffff81a003a8>] __do_softirq+0x3a8/0x4bd
> softirqs last disabled at (60943): [<ffffffff8106166c>] irq_exit+0xbc/0xe0
> ---[ end trace f0ce4aa6d3ed581f ]---
> 
> <over and over and over>
> 
> XFS (pmem4): xfs_do_force_shutdown(0x8) called from line 368 of file fs/xfs/xfs_trans.c. Return address = ffffffffa02edf93
> XFS (pmem4): Corruption of in-memory data detected.  Shutting down filesystem
> XFS (pmem4): Please unmount the filesystem and rectify the problem(s)
> XFS (pmem4): xfs_imap_to_bp: xfs_trans_read_buf() returned error -5.
> XFS (pmem4): xlog_recover_clear_agi_bucket: failed to clear agi 0. Continuing.
> 
> <over and over and over>
> 
> Reproduces on scsi disk too.  Nightnight.
> 
> --D
> 
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	wait
> > > > +done
> > > 
> > > Can't we just pass the pids forked by the loop above? Though the manpage
> > > says wait should wait for all child pids as it is, so perhaps the loop
> > > is unnecessary?
> > 
> > Oh, I did not know that.  Thanks for the review!
> > 
> > > Brian
> > > 
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > > > new file mode 100644
> > > > index 00000000..cbbe36e9
> > > > --- /dev/null
> > > > +++ b/tests/generic/711.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 711
> > > > +silence is golden
> > > > diff --git a/tests/generic/group b/tests/generic/group
> > > > index f56eb475..26999ca1 100644
> > > > --- a/tests/generic/group
> > > > +++ b/tests/generic/group
> > > > @@ -529,3 +529,5 @@
> > > >  524 auto quick
> > > >  525 auto quick rw
> > > >  709 auto quick
> > > > +710 auto quick unlink
> > > > +711 auto quick unlink
> > > > diff --git a/tests/xfs/736 b/tests/xfs/736
> > > > new file mode 100755
> > > > index 00000000..e33de0ae
> > > > --- /dev/null
> > > > +++ b/tests/xfs/736
> > > > @@ -0,0 +1,71 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 736
> > > > +#
> > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > +# applies to most other filesystems.
> > > > +#
> > > > +# Here we force the use of the slow iunlink bucket walk code in a single
> > > > +# threaded situation.
> > > > +#
> > > > +seq=`basename $0`
> > > > +seqres=$RESULT_DIR/$seq
> > > > +echo "QA output created by $seq"
> > > > +tmp=/tmp/$$
> > > > +status=1	# failure is the default!
> > > > +testfile=$TEST_DIR/$seq.txt
> > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	rm -f $tmp.*
> > > > +}
> > > > +
> > > > +# get standard environment, filters and checks
> > > > +. ./common/rc
> > > > +. ./common/attr
> > > > +. ./common/filter
> > > > +. ./common/inject
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > > > new file mode 100644
> > > > index 00000000..0258a248
> > > > --- /dev/null
> > > > +++ b/tests/xfs/736.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 736
> > > > +silence is golden
> > > > diff --git a/tests/xfs/737 b/tests/xfs/737
> > > > new file mode 100755
> > > > index 00000000..47e65607
> > > > --- /dev/null
> > > > +++ b/tests/xfs/737
> > > > @@ -0,0 +1,79 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 737
> > > > +#
> > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > +# applies to most other filesystems.
> > > > +#
> > > > +# Here we force the use of the slow iunlink bucket walk code, using every
> > > > +# CPU possible.
> > > > +#
> > > > +seq=`basename $0`
> > > > +seqres=$RESULT_DIR/$seq
> > > > +echo "QA output created by $seq"
> > > > +tmp=/tmp/$$
> > > > +status=1	# failure is the default!
> > > > +testfile=$TEST_DIR/$seq.txt
> > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	rm -f $tmp.*
> > > > +}
> > > > +
> > > > +# get standard environment, filters and checks
> > > > +. ./common/rc
> > > > +. ./common/attr
> > > > +. ./common/filter
> > > > +. ./common/inject
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Load up all the CPUs, two threads per CPU.
> > > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > > +done
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	wait
> > > > +done
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > > > new file mode 100644
> > > > index 00000000..bdc4966d
> > > > --- /dev/null
> > > > +++ b/tests/xfs/737.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 737
> > > > +silence is golden
> > > > diff --git a/tests/xfs/group b/tests/xfs/group
> > > > index 7b7d69f1..d3189cd5 100644
> > > > --- a/tests/xfs/group
> > > > +++ b/tests/xfs/group
> > > > @@ -497,3 +497,5 @@
> > > >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> > > >  498 dangerous_fuzzers dangerous_norepair
> > > >  499 auto quick
> > > > +736 auto quick unlink
> > > > +737 auto quick unlink
> > > > 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

On Wed, Feb 13, 2019 at 10:44:04AM -0500, Brian Foster wrote:
> On Tue, Feb 12, 2019 at 08:51:19PM -0800, Darrick J. Wong wrote:
> > On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> > > On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > > > From: Darrick J. Wong <darrick.wong@oracle.com>
> > > > 
> > > > Create a test (+ helper program) that opens as many unlinked files as it
> > > > possibly can on the scratch filesystem, then closes all the files at
> > > > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > > > make sure that the fallback code doesn't bitrot.
> > > > 
> > > > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > > > ---
> > > >  src/Makefile          |    2 -
> > > >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> > > >  tests/generic/710     |   65 +++++++++++++++++++++++++
> > > >  tests/generic/710.out |    2 +
> > > >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> > > >  tests/generic/711.out |    2 +
> > > >  tests/generic/group   |    2 +
> > > >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> > > >  tests/xfs/736.out     |    2 +
> > > >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> > > >  tests/xfs/737.out     |    2 +
> > > >  tests/xfs/group       |    2 +
> > > >  12 files changed, 428 insertions(+), 1 deletion(-)
> > > >  create mode 100644 src/tmpfile.c
> > > >  create mode 100755 tests/generic/710
> > > >  create mode 100644 tests/generic/710.out
> > > >  create mode 100755 tests/generic/711
> > > >  create mode 100644 tests/generic/711.out
> > > >  create mode 100755 tests/xfs/736
> > > >  create mode 100644 tests/xfs/736.out
> > > >  create mode 100755 tests/xfs/737
> > > >  create mode 100644 tests/xfs/737.out
> > > > 
> > > > 
> ...
> > > > diff --git a/tests/generic/710 b/tests/generic/710
> > > > new file mode 100755
> > > > index 00000000..18aa9d34
> > > > --- /dev/null
> > > > +++ b/tests/generic/710
> > > > @@ -0,0 +1,65 @@
> ...
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((50000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > 
> > > I see the following from the above line when I run this test:
> > > 
> > > +./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected
> > 
> > Weird... it's supposed to be set based on the amount of RAM you have.
> > 
> 
> Strange indeed. I added some tracepoints that show it's initialized to
> something reasonable on boot. I added another to determine what might be
> changing it to -1:
> 
>           <idle>-0     [000] ...1     0.630877: files_maxfiles_init: 386: nr_pages 996312 nr_free_pages 994058 memreserve 3381
>           <idle>-0     [000] ...1     0.630879: files_maxfiles_init: 392: memreserve 3381 n 397172 NR_FILE 8192 max_files 397172
>          systemd-1     [002] ....     2.732128: proc_filemax: 2879: max_files 18446744073709551615
>          systemd-1     [000] ....     9.303595: proc_filemax: 2879: max_files 18446744073709551615
> 
> So systemd is changing it for some reason. *shrug* not sure it really
> matters that much, but it might be worth massaging the input to handle
> the max value somehow or another.

Yeah, I'll change it to be more robust.  Do you have a setting in
/etc/sysctl* or is this some weird new systemd thing?

(FWIW systemd 237 on Ubuntu 18.04 doesn't seem to touch file-max)

--D

> Brian
> 
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/generic/710.out b/tests/generic/710.out
> > > > new file mode 100644
> > > > index 00000000..e0a55170
> > > > --- /dev/null
> > > > +++ b/tests/generic/710.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 710
> > > > +silence is golden
> > > > diff --git a/tests/generic/711 b/tests/generic/711
> > > > new file mode 100755
> > > > index 00000000..11d76218
> > > > --- /dev/null
> > > > +++ b/tests/generic/711
> > > > @@ -0,0 +1,73 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 711
> > > > +#
> > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > +# applies to most other filesystems.
> > > > +#
> > > > +# Use every CPU possible to stress the filesystem.
> > > > +#
> > > > +seq=`basename $0`
> > > > +seqres=$RESULT_DIR/$seq
> > > > +echo "QA output created by $seq"
> > > > +tmp=/tmp/$$
> > > > +status=1	# failure is the default!
> > > > +testfile=$TEST_DIR/$seq.txt
> > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	rm -f $tmp.*
> > > > +}
> > > > +
> > > > +# get standard environment, filters and checks
> > > > +. ./common/rc
> > > > +. ./common/attr
> > > > +. ./common/filter
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Try to load up all the CPUs, two threads per CPU.
> > > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((50000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > > +done
> > > 
> > > Doesn't this make the first test kind of a subset of this one (where
> > > nr_cpus == 1)? If so, could we just do a couple iterations with
> > > different nr_cpus values?
> > 
> > I think it's fine only to have the multithreaded version, that should
> > stress the AGs well enough.
> > 
> > > I'm wondering if we should have a log recovery test as well, btw.
> > 
> > Yes.  I'll turn g/710 and x/736 into the log recovery tests.
> > 
> > (Oh wow flood of asserts this is going to take a while to straighten
> > out)
> > 
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	wait
> > > > +done
> > > 
> > > Can't we just pass the pids forked by the loop above? Though the manpage
> > > says wait should wait for all child pids as it is, so perhaps the loop
> > > is unnecessary?
> > 
> > Oh, I did not know that.  Thanks for the review!
> > 
> > > Brian
> > > 
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > > > new file mode 100644
> > > > index 00000000..cbbe36e9
> > > > --- /dev/null
> > > > +++ b/tests/generic/711.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 711
> > > > +silence is golden
> > > > diff --git a/tests/generic/group b/tests/generic/group
> > > > index f56eb475..26999ca1 100644
> > > > --- a/tests/generic/group
> > > > +++ b/tests/generic/group
> > > > @@ -529,3 +529,5 @@
> > > >  524 auto quick
> > > >  525 auto quick rw
> > > >  709 auto quick
> > > > +710 auto quick unlink
> > > > +711 auto quick unlink
> > > > diff --git a/tests/xfs/736 b/tests/xfs/736
> > > > new file mode 100755
> > > > index 00000000..e33de0ae
> > > > --- /dev/null
> > > > +++ b/tests/xfs/736
> > > > @@ -0,0 +1,71 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 736
> > > > +#
> > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > +# applies to most other filesystems.
> > > > +#
> > > > +# Here we force the use of the slow iunlink bucket walk code in a single
> > > > +# threaded situation.
> > > > +#
> > > > +seq=`basename $0`
> > > > +seqres=$RESULT_DIR/$seq
> > > > +echo "QA output created by $seq"
> > > > +tmp=/tmp/$$
> > > > +status=1	# failure is the default!
> > > > +testfile=$TEST_DIR/$seq.txt
> > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	rm -f $tmp.*
> > > > +}
> > > > +
> > > > +# get standard environment, filters and checks
> > > > +. ./common/rc
> > > > +. ./common/attr
> > > > +. ./common/filter
> > > > +. ./common/inject
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > > > new file mode 100644
> > > > index 00000000..0258a248
> > > > --- /dev/null
> > > > +++ b/tests/xfs/736.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 736
> > > > +silence is golden
> > > > diff --git a/tests/xfs/737 b/tests/xfs/737
> > > > new file mode 100755
> > > > index 00000000..47e65607
> > > > --- /dev/null
> > > > +++ b/tests/xfs/737
> > > > @@ -0,0 +1,79 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 737
> > > > +#
> > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > +# applies to most other filesystems.
> > > > +#
> > > > +# Here we force the use of the slow iunlink bucket walk code, using every
> > > > +# CPU possible.
> > > > +#
> > > > +seq=`basename $0`
> > > > +seqres=$RESULT_DIR/$seq
> > > > +echo "QA output created by $seq"
> > > > +tmp=/tmp/$$
> > > > +status=1	# failure is the default!
> > > > +testfile=$TEST_DIR/$seq.txt
> > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	rm -f $tmp.*
> > > > +}
> > > > +
> > > > +# get standard environment, filters and checks
> > > > +. ./common/rc
> > > > +. ./common/attr
> > > > +. ./common/filter
> > > > +. ./common/inject
> > > > +
> > > > +# real QA test starts here
> > > > +_supported_fs generic
> > > > +_supported_os Linux
> > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > +_require_scratch
> > > > +
> > > > +rm -f $seqres.full
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_scratch_mount
> > > > +
> > > > +# Load up all the CPUs, two threads per CPU.
> > > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > > +
> > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > > > +# so that this test doesn't take forever or OOM the box
> > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > +ulimit -n $max_files
> > > > +
> > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > +
> > > > +# Open a lot of unlinked files
> > > > +echo create >> $seqres.full
> > > > +program=$PWD/src/tmpfile
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > > +done
> > > > +for i in $(seq 1 $nr_cpus); do
> > > > +	wait
> > > > +done
> > > > +
> > > > +# Unmount to prove that we can clean it all
> > > > +echo umount >> $seqres.full
> > > > +before=$(date +%s)
> > > > +_scratch_unmount
> > > > +after=$(date +%s)
> > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > +
> > > > +# Mount so that we can run the usual checks
> > > > +echo silence is golden
> > > > +_scratch_mount
> > > > +status=0
> > > > +exit
> > > > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > > > new file mode 100644
> > > > index 00000000..bdc4966d
> > > > --- /dev/null
> > > > +++ b/tests/xfs/737.out
> > > > @@ -0,0 +1,2 @@
> > > > +QA output created by 737
> > > > +silence is golden
> > > > diff --git a/tests/xfs/group b/tests/xfs/group
> > > > index 7b7d69f1..d3189cd5 100644
> > > > --- a/tests/xfs/group
> > > > +++ b/tests/xfs/group
> > > > @@ -497,3 +497,5 @@
> > > >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> > > >  498 dangerous_fuzzers dangerous_norepair
> > > >  499 auto quick
> > > > +736 auto quick unlink
> > > > +737 auto quick unlink
> > > > 

Re: [PATCH 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Brian Foster]

On Wed, Feb 13, 2019 at 08:20:56AM -0800, Darrick J. Wong wrote:
> On Wed, Feb 13, 2019 at 10:44:04AM -0500, Brian Foster wrote:
> > On Tue, Feb 12, 2019 at 08:51:19PM -0800, Darrick J. Wong wrote:
> > > On Tue, Feb 12, 2019 at 09:04:36AM -0500, Brian Foster wrote:
> > > > On Mon, Feb 11, 2019 at 06:17:54PM -0800, Darrick J. Wong wrote:
> > > > > From: Darrick J. Wong <darrick.wong@oracle.com>
> > > > > 
> > > > > Create a test (+ helper program) that opens as many unlinked files as it
> > > > > possibly can on the scratch filesystem, then closes all the files at
> > > > > once to stress-test unlinked file cleanup.  Add an xfs-specific test to
> > > > > make sure that the fallback code doesn't bitrot.
> > > > > 
> > > > > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > > > > ---
> > > > >  src/Makefile          |    2 -
> > > > >  src/tmpfile.c         |  127 +++++++++++++++++++++++++++++++++++++++++++++++++
> > > > >  tests/generic/710     |   65 +++++++++++++++++++++++++
> > > > >  tests/generic/710.out |    2 +
> > > > >  tests/generic/711     |   73 ++++++++++++++++++++++++++++
> > > > >  tests/generic/711.out |    2 +
> > > > >  tests/generic/group   |    2 +
> > > > >  tests/xfs/736         |   71 +++++++++++++++++++++++++++
> > > > >  tests/xfs/736.out     |    2 +
> > > > >  tests/xfs/737         |   79 ++++++++++++++++++++++++++++++
> > > > >  tests/xfs/737.out     |    2 +
> > > > >  tests/xfs/group       |    2 +
> > > > >  12 files changed, 428 insertions(+), 1 deletion(-)
> > > > >  create mode 100644 src/tmpfile.c
> > > > >  create mode 100755 tests/generic/710
> > > > >  create mode 100644 tests/generic/710.out
> > > > >  create mode 100755 tests/generic/711
> > > > >  create mode 100644 tests/generic/711.out
> > > > >  create mode 100755 tests/xfs/736
> > > > >  create mode 100644 tests/xfs/736.out
> > > > >  create mode 100755 tests/xfs/737
> > > > >  create mode 100644 tests/xfs/737.out
> > > > > 
> > > > > 
> > ...
> > > > > diff --git a/tests/generic/710 b/tests/generic/710
> > > > > new file mode 100755
> > > > > index 00000000..18aa9d34
> > > > > --- /dev/null
> > > > > +++ b/tests/generic/710
> > > > > @@ -0,0 +1,65 @@
> > ...
> > > > > +
> > > > > +# real QA test starts here
> > > > > +_supported_fs generic
> > > > > +_supported_os Linux
> > > > > +_require_scratch
> > > > > +
> > > > > +rm -f $seqres.full
> > > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > > +_scratch_mount
> > > > > +
> > > > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > > > +# so that this test doesn't take forever or OOM the box
> > > > > +max_files=$((50000 * LOAD_FACTOR))
> > > > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > 
> > > > I see the following from the above line when I run this test:
> > > > 
> > > > +./tests/generic/710: line 46: test: 18446744073709551615: integer expression expected
> > > 
> > > Weird... it's supposed to be set based on the amount of RAM you have.
> > > 
> > 
> > Strange indeed. I added some tracepoints that show it's initialized to
> > something reasonable on boot. I added another to determine what might be
> > changing it to -1:
> > 
> >           <idle>-0     [000] ...1     0.630877: files_maxfiles_init: 386: nr_pages 996312 nr_free_pages 994058 memreserve 3381
> >           <idle>-0     [000] ...1     0.630879: files_maxfiles_init: 392: memreserve 3381 n 397172 NR_FILE 8192 max_files 397172
> >          systemd-1     [002] ....     2.732128: proc_filemax: 2879: max_files 18446744073709551615
> >          systemd-1     [000] ....     9.303595: proc_filemax: 2879: max_files 18446744073709551615
> > 
> > So systemd is changing it for some reason. *shrug* not sure it really
> > matters that much, but it might be worth massaging the input to handle
> > the max value somehow or another.
> 
> Yeah, I'll change it to be more robust.  Do you have a setting in
> /etc/sysctl* or is this some weird new systemd thing?
> 

Nothing that I'm aware of or set explicitly. grep doesn't show anything,
but I'm not sure if there's some differently named setting somewhere
that might translate to file-max. This is systemd-241~rc2-2.fc30.x86_64
on fedora rawhide.

Brian

> (FWIW systemd 237 on Ubuntu 18.04 doesn't seem to touch file-max)
> 
> --D
> 
> > Brian
> > 
> > > > > +ulimit -n $max_files
> > > > > +
> > > > > +# Open a lot of unlinked files
> > > > > +echo create >> $seqres.full
> > > > > +program=$PWD/src/tmpfile
> > > > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > > > +
> > > > > +# Unmount to prove that we can clean it all
> > > > > +echo umount >> $seqres.full
> > > > > +before=$(date +%s)
> > > > > +_scratch_unmount
> > > > > +after=$(date +%s)
> > > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > > +
> > > > > +# Mount so that we can run the usual checks
> > > > > +echo silence is golden
> > > > > +_scratch_mount
> > > > > +status=0
> > > > > +exit
> > > > > diff --git a/tests/generic/710.out b/tests/generic/710.out
> > > > > new file mode 100644
> > > > > index 00000000..e0a55170
> > > > > --- /dev/null
> > > > > +++ b/tests/generic/710.out
> > > > > @@ -0,0 +1,2 @@
> > > > > +QA output created by 710
> > > > > +silence is golden
> > > > > diff --git a/tests/generic/711 b/tests/generic/711
> > > > > new file mode 100755
> > > > > index 00000000..11d76218
> > > > > --- /dev/null
> > > > > +++ b/tests/generic/711
> > > > > @@ -0,0 +1,73 @@
> > > > > +#! /bin/bash
> > > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > > +#
> > > > > +# FS QA Test No. 711
> > > > > +#
> > > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > > +# applies to most other filesystems.
> > > > > +#
> > > > > +# Use every CPU possible to stress the filesystem.
> > > > > +#
> > > > > +seq=`basename $0`
> > > > > +seqres=$RESULT_DIR/$seq
> > > > > +echo "QA output created by $seq"
> > > > > +tmp=/tmp/$$
> > > > > +status=1	# failure is the default!
> > > > > +testfile=$TEST_DIR/$seq.txt
> > > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > > +
> > > > > +_cleanup()
> > > > > +{
> > > > > +	cd /
> > > > > +	rm -f $tmp.*
> > > > > +}
> > > > > +
> > > > > +# get standard environment, filters and checks
> > > > > +. ./common/rc
> > > > > +. ./common/attr
> > > > > +. ./common/filter
> > > > > +
> > > > > +# real QA test starts here
> > > > > +_supported_fs generic
> > > > > +_supported_os Linux
> > > > > +_require_scratch
> > > > > +
> > > > > +rm -f $seqres.full
> > > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > > +_scratch_mount
> > > > > +
> > > > > +# Try to load up all the CPUs, two threads per CPU.
> > > > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > > > +
> > > > > +# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
> > > > > +# so that this test doesn't take forever or OOM the box
> > > > > +max_files=$((50000 * LOAD_FACTOR))
> > > > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > > +ulimit -n $max_files
> > > > > +
> > > > > +# Open a lot of unlinked files
> > > > > +echo create >> $seqres.full
> > > > > +program=$PWD/src/tmpfile
> > > > > +for i in $(seq 1 $nr_cpus); do
> > > > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > > > +done
> > > > 
> > > > Doesn't this make the first test kind of a subset of this one (where
> > > > nr_cpus == 1)? If so, could we just do a couple iterations with
> > > > different nr_cpus values?
> > > 
> > > I think it's fine only to have the multithreaded version, that should
> > > stress the AGs well enough.
> > > 
> > > > I'm wondering if we should have a log recovery test as well, btw.
> > > 
> > > Yes.  I'll turn g/710 and x/736 into the log recovery tests.
> > > 
> > > (Oh wow flood of asserts this is going to take a while to straighten
> > > out)
> > > 
> > > > > +for i in $(seq 1 $nr_cpus); do
> > > > > +	wait
> > > > > +done
> > > > 
> > > > Can't we just pass the pids forked by the loop above? Though the manpage
> > > > says wait should wait for all child pids as it is, so perhaps the loop
> > > > is unnecessary?
> > > 
> > > Oh, I did not know that.  Thanks for the review!
> > > 
> > > > Brian
> > > > 
> > > > > +
> > > > > +# Unmount to prove that we can clean it all
> > > > > +echo umount >> $seqres.full
> > > > > +before=$(date +%s)
> > > > > +_scratch_unmount
> > > > > +after=$(date +%s)
> > > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > > +
> > > > > +# Mount so that we can run the usual checks
> > > > > +echo silence is golden
> > > > > +_scratch_mount
> > > > > +status=0
> > > > > +exit
> > > > > diff --git a/tests/generic/711.out b/tests/generic/711.out
> > > > > new file mode 100644
> > > > > index 00000000..cbbe36e9
> > > > > --- /dev/null
> > > > > +++ b/tests/generic/711.out
> > > > > @@ -0,0 +1,2 @@
> > > > > +QA output created by 711
> > > > > +silence is golden
> > > > > diff --git a/tests/generic/group b/tests/generic/group
> > > > > index f56eb475..26999ca1 100644
> > > > > --- a/tests/generic/group
> > > > > +++ b/tests/generic/group
> > > > > @@ -529,3 +529,5 @@
> > > > >  524 auto quick
> > > > >  525 auto quick rw
> > > > >  709 auto quick
> > > > > +710 auto quick unlink
> > > > > +711 auto quick unlink
> > > > > diff --git a/tests/xfs/736 b/tests/xfs/736
> > > > > new file mode 100755
> > > > > index 00000000..e33de0ae
> > > > > --- /dev/null
> > > > > +++ b/tests/xfs/736
> > > > > @@ -0,0 +1,71 @@
> > > > > +#! /bin/bash
> > > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > > +#
> > > > > +# FS QA Test No. 736
> > > > > +#
> > > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > > +# applies to most other filesystems.
> > > > > +#
> > > > > +# Here we force the use of the slow iunlink bucket walk code in a single
> > > > > +# threaded situation.
> > > > > +#
> > > > > +seq=`basename $0`
> > > > > +seqres=$RESULT_DIR/$seq
> > > > > +echo "QA output created by $seq"
> > > > > +tmp=/tmp/$$
> > > > > +status=1	# failure is the default!
> > > > > +testfile=$TEST_DIR/$seq.txt
> > > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > > +
> > > > > +_cleanup()
> > > > > +{
> > > > > +	cd /
> > > > > +	rm -f $tmp.*
> > > > > +}
> > > > > +
> > > > > +# get standard environment, filters and checks
> > > > > +. ./common/rc
> > > > > +. ./common/attr
> > > > > +. ./common/filter
> > > > > +. ./common/inject
> > > > > +
> > > > > +# real QA test starts here
> > > > > +_supported_fs generic
> > > > > +_supported_os Linux
> > > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > > +_require_scratch
> > > > > +
> > > > > +rm -f $seqres.full
> > > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > > +_scratch_mount
> > > > > +
> > > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
> > > > > +# so that this test doesn't take forever or OOM the box
> > > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > > +max_allowable_files=$(cat /proc/sys/fs/file-max)
> > > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > > +ulimit -n $max_files
> > > > > +
> > > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > > +
> > > > > +# Open a lot of unlinked files
> > > > > +echo create >> $seqres.full
> > > > > +program=$PWD/src/tmpfile
> > > > > +(cd $SCRATCH_MNT ; $program >> $seqres.full)
> > > > > +
> > > > > +# Unmount to prove that we can clean it all
> > > > > +echo umount >> $seqres.full
> > > > > +before=$(date +%s)
> > > > > +_scratch_unmount
> > > > > +after=$(date +%s)
> > > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > > +
> > > > > +# Mount so that we can run the usual checks
> > > > > +echo silence is golden
> > > > > +_scratch_mount
> > > > > +status=0
> > > > > +exit
> > > > > diff --git a/tests/xfs/736.out b/tests/xfs/736.out
> > > > > new file mode 100644
> > > > > index 00000000..0258a248
> > > > > --- /dev/null
> > > > > +++ b/tests/xfs/736.out
> > > > > @@ -0,0 +1,2 @@
> > > > > +QA output created by 736
> > > > > +silence is golden
> > > > > diff --git a/tests/xfs/737 b/tests/xfs/737
> > > > > new file mode 100755
> > > > > index 00000000..47e65607
> > > > > --- /dev/null
> > > > > +++ b/tests/xfs/737
> > > > > @@ -0,0 +1,79 @@
> > > > > +#! /bin/bash
> > > > > +# SPDX-License-Identifier: GPL-2.0+
> > > > > +# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
> > > > > +#
> > > > > +# FS QA Test No. 737
> > > > > +#
> > > > > +# Stress test creating a lot of unlinked O_TMPFILE files and closing them
> > > > > +# all at once, checking that we don't blow up the filesystem.  This is sort
> > > > > +# of a performance test for the xfs unlinked inode backref patchset, but it
> > > > > +# applies to most other filesystems.
> > > > > +#
> > > > > +# Here we force the use of the slow iunlink bucket walk code, using every
> > > > > +# CPU possible.
> > > > > +#
> > > > > +seq=`basename $0`
> > > > > +seqres=$RESULT_DIR/$seq
> > > > > +echo "QA output created by $seq"
> > > > > +tmp=/tmp/$$
> > > > > +status=1	# failure is the default!
> > > > > +testfile=$TEST_DIR/$seq.txt
> > > > > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > > > > +
> > > > > +_cleanup()
> > > > > +{
> > > > > +	cd /
> > > > > +	rm -f $tmp.*
> > > > > +}
> > > > > +
> > > > > +# get standard environment, filters and checks
> > > > > +. ./common/rc
> > > > > +. ./common/attr
> > > > > +. ./common/filter
> > > > > +. ./common/inject
> > > > > +
> > > > > +# real QA test starts here
> > > > > +_supported_fs generic
> > > > > +_supported_os Linux
> > > > > +_require_xfs_io_error_injection "iunlink_fallback"
> > > > > +_require_scratch
> > > > > +
> > > > > +rm -f $seqres.full
> > > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > > +_scratch_mount
> > > > > +
> > > > > +# Load up all the CPUs, two threads per CPU.
> > > > > +nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
> > > > > +
> > > > > +# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
> > > > > +# so that this test doesn't take forever or OOM the box
> > > > > +max_files=$((30000 * LOAD_FACTOR))
> > > > > +max_allowable_files=$(( $(cat /proc/sys/fs/file-max) / nr_cpus ))
> > > > > +test $max_files -gt $max_allowable_files && max_files=$max_allowable_files
> > > > > +ulimit -n $max_files
> > > > > +
> > > > > +# Force xfs to use the iunlinked fallback 50% of the time
> > > > > +_scratch_inject_error "iunlink_fallback" "2"
> > > > > +
> > > > > +# Open a lot of unlinked files
> > > > > +echo create >> $seqres.full
> > > > > +program=$PWD/src/tmpfile
> > > > > +for i in $(seq 1 $nr_cpus); do
> > > > > +	(mkdir $SCRATCH_MNT/$i ; cd $SCRATCH_MNT/$i ; $program >> $seqres.full) &
> > > > > +done
> > > > > +for i in $(seq 1 $nr_cpus); do
> > > > > +	wait
> > > > > +done
> > > > > +
> > > > > +# Unmount to prove that we can clean it all
> > > > > +echo umount >> $seqres.full
> > > > > +before=$(date +%s)
> > > > > +_scratch_unmount
> > > > > +after=$(date +%s)
> > > > > +echo "Unmount took $((after - before))s." >> $seqres.full
> > > > > +
> > > > > +# Mount so that we can run the usual checks
> > > > > +echo silence is golden
> > > > > +_scratch_mount
> > > > > +status=0
> > > > > +exit
> > > > > diff --git a/tests/xfs/737.out b/tests/xfs/737.out
> > > > > new file mode 100644
> > > > > index 00000000..bdc4966d
> > > > > --- /dev/null
> > > > > +++ b/tests/xfs/737.out
> > > > > @@ -0,0 +1,2 @@
> > > > > +QA output created by 737
> > > > > +silence is golden
> > > > > diff --git a/tests/xfs/group b/tests/xfs/group
> > > > > index 7b7d69f1..d3189cd5 100644
> > > > > --- a/tests/xfs/group
> > > > > +++ b/tests/xfs/group
> > > > > @@ -497,3 +497,5 @@
> > > > >  497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
> > > > >  498 dangerous_fuzzers dangerous_norepair
> > > > >  499 auto quick
> > > > > +736 auto quick unlink
> > > > > +737 auto quick unlink
> > > > > 

[PATCH v2 2/2] generic: check the behavior of programs opening a lot of O_TMPFILE files

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

Create a test (+ helper program) that opens as many unlinked files as it
possibly can on the scratch filesystem, then closes all the files at
once to stress-test unlinked file cleanup.  Add an xfs-specific test to
make sure that the fallback code doesn't bitrot.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
v2: g/710 and x/736 now test log recovery processing of unlinked inodes
---
 .gitignore            |    1 
 src/Makefile          |    2 -
 src/t_open_tmpfiles.c |  166 +++++++++++++++++++++++++++++++++++++++++++++++++
 tests/generic/710     |   64 +++++++++++++++++++
 tests/generic/710.out |    2 +
 tests/generic/711     |   70 +++++++++++++++++++++
 tests/generic/711.out |    2 +
 tests/generic/group   |    2 +
 tests/xfs/736         |   90 +++++++++++++++++++++++++++
 tests/xfs/736.out     |    2 +
 tests/xfs/737         |   75 ++++++++++++++++++++++
 tests/xfs/737.out     |    2 +
 tests/xfs/group       |    2 +
 13 files changed, 479 insertions(+), 1 deletion(-)
 create mode 100644 src/t_open_tmpfiles.c
 create mode 100755 tests/generic/710
 create mode 100644 tests/generic/710.out
 create mode 100755 tests/generic/711
 create mode 100644 tests/generic/711.out
 create mode 100755 tests/xfs/736
 create mode 100644 tests/xfs/736.out
 create mode 100755 tests/xfs/737
 create mode 100644 tests/xfs/737.out

diff --git a/.gitignore b/.gitignore
index 0933dc7d..6454e4d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -135,6 +135,7 @@
 /src/t_mmap_writev
 /src/t_mtab
 /src/t_ofd_locks
+/src/t_open_tmpfiles
 /src/t_readdir_1
 /src/t_readdir_2
 /src/t_rename_overwrite
diff --git a/src/Makefile b/src/Makefile
index ae09eb0a..dbba7c78 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -27,7 +27,7 @@ LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \
 	renameat2 t_getcwd e4compact test-nextquota punch-alternating \
 	attr-list-by-handle-cursor-test listxattr dio-interleaved t_dir_type \
 	dio-invalidate-cache stat_test t_encrypted_d_revalidate \
-	attr_replace_test swapon mkswap t_attr_corruption
+	attr_replace_test swapon mkswap t_attr_corruption t_open_tmpfiles
 
 SUBDIRS = log-writes perf
 
diff --git a/src/t_open_tmpfiles.c b/src/t_open_tmpfiles.c
new file mode 100644
index 00000000..9e70fbaa
--- /dev/null
+++ b/src/t_open_tmpfiles.c
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Oracle.  All Rights Reserved.
+ * Author: Darrick J. Wong <darrick.wong@oracle.com>
+ *
+ * Test program to open unlinked files and leak them.
+ */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+#include <time.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include "global.h"
+
+static int min_fd = -1;
+static int max_fd = -1;
+static unsigned int nr_opened = 0;
+static float start_time;
+static int shutdown_fs = 0;
+
+void clock_time(float *time)
+{
+	static clockid_t clkid = CLOCK_MONOTONIC;
+	struct timespec ts;
+	int ret;
+
+retry:
+	ret = clock_gettime(clkid, &ts);
+	if (ret) {
+		if (clkid == CLOCK_MONOTONIC) {
+			clkid = CLOCK_REALTIME;
+			goto retry;
+		}
+		perror("clock_gettime");
+		exit(2);
+	}
+	*time = ts.tv_sec + ((float)ts.tv_nsec / 1000000000);
+}
+
+/*
+ * Exit the program due to an error.
+ *
+ * If we've exhausted all the file descriptors, make sure we close all the
+ * open fds in the order we received them in order to exploit a quirk of ext4
+ * and xfs where the oldest unlinked inodes are at the /end/ of the unlinked
+ * lists, which will make removing the unlinked files maximally painful.
+ *
+ * If it's some other error, just die and let the kernel sort it out.
+ */
+void die(void)
+{
+	float end_time;
+	int fd;
+
+	switch (errno) {
+	case EMFILE:
+	case ENFILE:
+	case ENOSPC:
+		clock_time(&end_time);
+		printf("Opened %u files in %.2fs.\n", nr_opened,
+				end_time - start_time);
+		fflush(stdout);
+
+		if (shutdown_fs) {
+			int flag = XFS_FSOP_GOING_FLAGS_NOLOGFLUSH;
+			int ret;
+
+			ret = ioctl(min_fd, XFS_IOC_GOINGDOWN, &flag);
+			if (ret) {
+				perror("shutdown");
+				exit(2);
+			}
+			exit(0);
+		}
+
+		clock_time(&start_time);
+		for (fd = min_fd; fd <= max_fd; fd++)
+			close(fd);
+		clock_time(&end_time);
+		printf("Closed %u files in %.2fs.\n", nr_opened,
+				end_time - start_time);
+		exit(0);
+		break;
+	default:
+		perror("open?");
+		exit(2);
+		break;
+	}
+}
+
+/* Remember how many file we open and all that. */
+void remember_fd(int fd)
+{
+	if (min_fd == -1 || min_fd > fd)
+		min_fd = fd;
+	if (max_fd == -1 || max_fd < fd)
+		max_fd = fd;
+	nr_opened++;
+}
+
+/* Put an opened file on the unlinked list and leak the fd. */
+void leak_tmpfile(void)
+{
+	int fd = -1;
+	int ret;
+#ifdef O_TMPFILE
+	static int try_o_tmpfile = 1;
+#endif
+
+	/* Try to create an O_TMPFILE and leak the fd. */
+#ifdef O_TMPFILE
+	if (try_o_tmpfile) {
+		fd = open(".", O_TMPFILE | O_RDWR, 0644);
+		if (fd >= 0) {
+			remember_fd(fd);
+			return;
+		}
+		if (fd < 0) {
+			if (errno == EOPNOTSUPP)
+				try_o_tmpfile = 0;
+			else
+				die();
+		}
+	}
+#endif
+
+	/* Oh well, create a new file, unlink it, and leak the fd. */
+	fd = open("./moo", O_CREAT | O_RDWR, 0644);
+	if (fd < 0)
+		die();
+	ret = unlink("./moo");
+	if (ret)
+		die();
+	remember_fd(fd);
+}
+
+/*
+ * Try to put as many files on the unlinked list and then kill them.
+ * The first argument is a directory to chdir into; passing any second arg
+ * will shut down the fs instead of closing files.
+ */
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	if (argc > 1) {
+		ret = chdir(argv[1]);
+		if (ret)
+			perror(argv[1]);
+	}
+	if (argc > 2 && !strcmp(argv[2], "shutdown"))
+		shutdown_fs = 1;
+
+	clock_time(&start_time);
+	while (1)
+		leak_tmpfile();
+	return 0;
+}
diff --git a/tests/generic/710 b/tests/generic/710
new file mode 100755
index 00000000..e1e72c4d
--- /dev/null
+++ b/tests/generic/710
@@ -0,0 +1,64 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 710
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and recovering them
+# after a crash, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use only a single CPU to test the single threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+_require_scratch_shutdown
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+	max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+src/t_open_tmpfiles $SCRATCH_MNT/$i shutdown >> $seqres.full
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/710.out b/tests/generic/710.out
new file mode 100644
index 00000000..e0a55170
--- /dev/null
+++ b/tests/generic/710.out
@@ -0,0 +1,2 @@
+QA output created by 710
+silence is golden
diff --git a/tests/generic/711 b/tests/generic/711
new file mode 100755
index 00000000..0dd151b9
--- /dev/null
+++ b/tests/generic/711
@@ -0,0 +1,70 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 711
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset, but it
+# applies to most other filesystems.
+#
+# Use every CPU possible to stress the filesystem.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+
+# real QA test starts here
+_supported_fs generic
+_supported_os Linux
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Try to load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 50000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((50000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+	max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+for i in $(seq 1 $nr_cpus); do
+	mkdir $SCRATCH_MNT/$i
+	src/t_open_tmpfiles $SCRATCH_MNT/$i >> $seqres.full &
+done
+wait
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/generic/711.out b/tests/generic/711.out
new file mode 100644
index 00000000..cbbe36e9
--- /dev/null
+++ b/tests/generic/711.out
@@ -0,0 +1,2 @@
+QA output created by 711
+silence is golden
diff --git a/tests/generic/group b/tests/generic/group
index b3086154..0233d5e8 100644
--- a/tests/generic/group
+++ b/tests/generic/group
@@ -529,4 +529,6 @@
 524 auto quick
 525 auto quick rw
 709 auto quick
+710 auto quick unlink
+711 auto quick unlink
 712 auto quick attr
diff --git a/tests/xfs/736 b/tests/xfs/736
new file mode 100755
index 00000000..b791cf5c
--- /dev/null
+++ b/tests/xfs/736
@@ -0,0 +1,90 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 736
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and recovering them
+# after a crash, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset.
+#
+# Here we force the use of the slow iunlink bucket walk code in a single
+# threaded situation.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+delay_knob="/sys/fs/xfs/debug/log_recovery_delay"
+
+_cleanup()
+{
+	cd /
+	test -e "$delay_knob" && echo 0 > "$delay_knob"
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/inject
+
+# real QA test starts here
+_supported_fs xfs
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_xfs_sysfs debug/log_recovery_delay
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+	max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+src/t_open_tmpfiles $SCRATCH_MNT/$i shutdown >> $seqres.full
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Force xfs to use the iunlinked fallback 50% of the time
+injector() {
+	# Slow down log recovery by 5s to give us enough time to set up
+	# error injection.
+	echo 5 > "$delay_knob"
+
+	# Try for 10s to set our knob.
+	knob="$(_find_xfs_mountdev_errortag_knob "${SCRATCH_DEV}" iunlink_fallback)"
+	nr=0
+	while [ ! -e "$knob" ] && [ "$nr" -lt 20 ]; do
+		sleep 0.5
+		nr=$((nr+1))
+	done
+	if [ -e "$knob" ]; then
+		echo 2 > "$knob"
+	else
+		echo "unable to set iunlink_fallback?"
+	fi
+}
+
+# Mount so that we can run the usual checks
+echo silence is golden
+injector &
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/736.out b/tests/xfs/736.out
new file mode 100644
index 00000000..0258a248
--- /dev/null
+++ b/tests/xfs/736.out
@@ -0,0 +1,2 @@
+QA output created by 736
+silence is golden
diff --git a/tests/xfs/737 b/tests/xfs/737
new file mode 100755
index 00000000..2d8a32be
--- /dev/null
+++ b/tests/xfs/737
@@ -0,0 +1,75 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (c) 2019 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 737
+#
+# Stress test creating a lot of unlinked O_TMPFILE files and closing them
+# all at once, checking that we don't blow up the filesystem.  This is sort
+# of a performance test for the xfs unlinked inode backref patchset.
+#
+# Here we force the use of the slow iunlink bucket walk code, using every
+# CPU possible.
+#
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+tmp=/tmp/$$
+status=1	# failure is the default!
+testfile=$TEST_DIR/$seq.txt
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/inject
+
+# real QA test starts here
+_supported_fs xfs
+_supported_os Linux
+_require_xfs_io_error_injection "iunlink_fallback"
+_require_scratch
+
+rm -f $seqres.full
+_scratch_mkfs >> $seqres.full 2>&1
+_scratch_mount
+
+# Load up all the CPUs, two threads per CPU.
+nr_cpus=$(( $(getconf _NPROCESSORS_ONLN) * 2 ))
+
+# Set ULIMIT_NOFILE to min(file-max, 30000 files per cpu per LOAD_FACTOR)
+# so that this test doesn't take forever or OOM the box
+max_files=$((30000 * LOAD_FACTOR))
+max_allowable_files=$(( $(cat /proc/sys/fs/file-max) ))
+test $max_allowable_files -gt 0 && test $max_files -gt $max_allowable_files && \
+	max_files=$max_allowable_files
+ulimit -n $max_files
+
+# Force xfs to use the iunlinked fallback 50% of the time
+_scratch_inject_error "iunlink_fallback" "2"
+
+# Open a lot of unlinked files
+echo create >> $seqres.full
+for i in $(seq 1 $nr_cpus); do
+	mkdir $SCRATCH_MNT/$i
+	src/t_open_tmpfiles $SCRATCH_MNT/$i >> $seqres.full &
+done
+wait
+
+# Unmount to prove that we can clean it all
+echo umount >> $seqres.full
+before=$(date +%s)
+_scratch_unmount
+after=$(date +%s)
+echo "Unmount took $((after - before))s." >> $seqres.full
+
+# Mount so that we can run the usual checks
+echo silence is golden
+_scratch_mount
+status=0
+exit
diff --git a/tests/xfs/737.out b/tests/xfs/737.out
new file mode 100644
index 00000000..bdc4966d
--- /dev/null
+++ b/tests/xfs/737.out
@@ -0,0 +1,2 @@
+QA output created by 737
+silence is golden
diff --git a/tests/xfs/group b/tests/xfs/group
index 7b7d69f1..d3189cd5 100644
--- a/tests/xfs/group
+++ b/tests/xfs/group
@@ -497,3 +497,5 @@
 497 dangerous_fuzzers dangerous_scrub dangerous_online_repair
 498 dangerous_fuzzers dangerous_norepair
 499 auto quick
+736 auto quick unlink
+737 auto quick unlink