[PATCH v3 0/3] fix up generic dmlogwrites tests to work with XFS

[PATCH v3 0/3] fix up generic dmlogwrites tests to work with XFS

[Brian Foster]

Hi all,

This version removes the discard zeroing check as Christoph points out
that discard behavior is generally not predictable. The remaining
generic dm-logwrites tests are still updated to use dm-thinp such that
discards from the recovery tool clear block ranges reasonably reliably.
This allows the tests to function correctly on filesystems like XFS
while a more generic solution is investigated.

Brian

v3:
- Re-add dm-thinp changes.
- Drop discard checks (from v1) and XFS disablement (from v2).
- Use _require_scratch_nocheck in generic/470 to avoid spurious repair
  noise now that the test no longer runs mkfs on the scratch dev.
v2: https://lore.kernel.org/fstests/20200827145329.435398-1-bfoster@redhat.com/
- Drop all dmthinp changes. Unconditionally disable tests on XFS.
v1: https://lore.kernel.org/fstests/20200826143815.360002-2-bfoster@redhat.com/

Brian Foster (3):
  generic/455: use thin volume for dmlogwrites target device
  generic/457: use thin volume for dmlogwrites target device
  generic/470: use thin volume for dmlogwrites target device

 tests/generic/455 | 36 ++++++++++++++++++++++--------------
 tests/generic/457 | 33 +++++++++++++++++++++------------
 tests/generic/470 | 23 +++++++++++++++++------
 3 files changed, 60 insertions(+), 32 deletions(-)

-- 
2.25.4

[PATCH v3 1/3] generic/455: use thin volume for dmlogwrites target device

[Brian Foster]

dmlogwrites support for XFS depends on discard zeroing support of
the intended target device. Update the test to use a thin volume and
allow it to run consistently and reliably on XFS.

Signed-off-by: Brian Foster <bfoster@redhat.com>
---
 tests/generic/455 | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/tests/generic/455 b/tests/generic/455
index 05621220..72a44fda 100755
--- a/tests/generic/455
+++ b/tests/generic/455
@@ -16,12 +16,14 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_log_writes_cleanup
+	_dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -30,6 +32,7 @@ _supported_os Linux
 _require_test
 _require_scratch_nocheck
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -42,13 +45,12 @@ check_files()
 		local filename=$(basename $i)
 		local mark="${filename##*.}"
 		echo "checking $filename" >> $seqres.full
-		_log_writes_replay_log $filename $SCRATCH_DEV
-		_scratch_mount
+		_log_writes_replay_log $filename $DMTHIN_VOL_DEV
+		_dmthin_mount
 		local expected_md5=$(_md5_checksum $i)
 		local md5=$(_md5_checksum $SCRATCH_MNT/$name)
 		[ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-		_scratch_unmount
-		_check_scratch_fs
+		_dmthin_check_fs
 	done
 }
 
@@ -56,8 +58,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -88,14 +98,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
 	check_files testfile$j
@@ -103,14 +112,13 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
 	md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
 	[ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
 done
-_scratch_unmount
-_check_scratch_fs
+_dmthin_check_fs
 
 echo "Silence is golden"
 status=0
-- 
2.25.4

[PATCH v3 2/3] generic/457: use thin volume for dmlogwrites target device

[Brian Foster]

dmlogwrites support for XFS depends on discard zeroing support of
the intended target device. Update the test to use a thin volume and
allow it to run consistently and reliably on XFS.

Signed-off-by: Brian Foster <bfoster@redhat.com>
---
 tests/generic/457 | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/tests/generic/457 b/tests/generic/457
index 82367304..42a064d8 100755
--- a/tests/generic/457
+++ b/tests/generic/457
@@ -16,6 +16,7 @@ status=1	# failure is the default!
 _cleanup()
 {
 	_log_writes_cleanup
+	_dmthin_cleanup
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
@@ -23,6 +24,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common/rc
 . ./common/filter
 . ./common/reflink
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # real QA test starts here
@@ -32,6 +34,7 @@ _require_test
 _require_scratch_reflink
 _require_cp_reflink
 _require_log_writes
+_require_dm_target thin-pool
 
 rm -f $seqres.full
 
@@ -44,13 +47,12 @@ check_files()
 		local filename=$(basename $i)
 		local mark="${filename##*.}"
 		echo "checking $filename" >> $seqres.full
-		_log_writes_replay_log $filename $SCRATCH_DEV
-		_scratch_mount
+		_log_writes_replay_log $filename $DMTHIN_VOL_DEV
+		_dmthin_mount
 		local expected_md5=$(_md5_checksum $i)
 		local md5=$(_md5_checksum $SCRATCH_MNT/$name)
 		[ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
-		_scratch_unmount
-		_check_scratch_fs
+		_dmthin_check_fs
 	done
 }
 
@@ -58,8 +60,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
 rm -rf $SANITY_DIR
 mkdir $SANITY_DIR
 
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
 # Create the log
-_log_writes_init $SCRATCH_DEV
+_log_writes_init $DMTHIN_VOL_DEV
 
 _log_writes_mkfs >> $seqres.full 2>&1
 
@@ -92,14 +102,13 @@ _log_writes_mark last
 _log_writes_unmount
 _log_writes_mark end
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # check pre umount
 echo "checking pre umount" >> $seqres.full
-_log_writes_replay_log last $SCRATCH_DEV
-_scratch_mount
-_scratch_unmount
-_check_scratch_fs
+_log_writes_replay_log last $DMTHIN_VOL_DEV
+_dmthin_mount
+_dmthin_check_fs
 
 for j in `seq 0 $((NUM_FILES-1))`; do
 	check_files testfile$j
@@ -107,8 +116,8 @@ done
 
 # Check the end
 echo "checking post umount" >> $seqres.full
-_log_writes_replay_log end $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log end $DMTHIN_VOL_DEV
+_dmthin_mount
 for j in `seq 0 $((NUM_FILES-1))`; do
 	md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
 	[ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
-- 
2.25.4

[PATCH v3 3/3] generic/470: use thin volume for dmlogwrites target device

[Brian Foster]

dmlogwrites support for XFS depends on discard zeroing support of
the intended target device. Update the test to use a thin volume and
allow it to run consistently and reliably on XFS.

Signed-off-by: Brian Foster <bfoster@redhat.com>
---
 tests/generic/470 | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/tests/generic/470 b/tests/generic/470
index fd6da563..c77499a2 100755
--- a/tests/generic/470
+++ b/tests/generic/470
@@ -20,12 +20,14 @@ _cleanup()
 {
 	cd /
 	_log_writes_cleanup
+	_dmthin_cleanup
 	rm -f $tmp.*
 }
 
 # get standard environment, filters and checks
 . ./common/rc
 . ./common/filter
+. ./common/dmthin
 . ./common/dmlogwrites
 
 # remove previous $seqres.full before test
@@ -34,12 +36,21 @@ rm -f $seqres.full
 # real QA test starts here
 _supported_fs generic
 _supported_os Linux
-_require_scratch
+_require_scratch_nocheck
 _require_log_writes_dax_mountopt "dax"
+_require_dm_target thin-pool
 _require_xfs_io_command "mmap" "-S"
 _require_xfs_io_command "log_writes"
 
-_log_writes_init $SCRATCH_DEV
+devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
+csize=$((1024*64 / 512))                # 64k cluster size
+lowspace=$((1024*1024 / 512))           # 1m low space threshold
+
+# Use a thin device to provide deterministic discard behavior. Discards are used
+# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
+_dmthin_init $devsize $devsize $csize $lowspace
+
+_log_writes_init $DMTHIN_VOL_DEV
 _log_writes_mkfs >> $seqres.full 2>&1
 _log_writes_mount -o dax
 
@@ -52,14 +63,14 @@ $XFS_IO_PROG -t -c "truncate $LEN" -c "mmap -S 0 $LEN" -c "mwrite 0 $LEN" \
 # Unmount the scratch dir and tear down the log writes target
 _log_writes_unmount
 _log_writes_remove
-_check_scratch_fs
+_dmthin_check_fs
 
 # destroy previous filesystem so we can be sure our rebuild works
-_scratch_mkfs >> $seqres.full 2>&1
+_mkfs_dev $DMTHIN_VOL_DEV >> $seqres.full 2>&1
 
 # check pre-unmap state
-_log_writes_replay_log preunmap $SCRATCH_DEV
-_scratch_mount
+_log_writes_replay_log preunmap $DMTHIN_VOL_DEV
+_dmthin_mount
 
 # We should see $SCRATCH_MNT/test as having 1 MiB in block allocations
 du -sh $SCRATCH_MNT/test | _filter_scratch | _filter_spaces
-- 
2.25.4

Re: [PATCH v3 1/3] generic/455: use thin volume for dmlogwrites target device

[Amir Goldstein]

On Tue, Sep 1, 2020 at 4:48 PM Brian Foster <bfoster@redhat.com> wrote:
>
> dmlogwrites support for XFS depends on discard zeroing support of
> the intended target device. Update the test to use a thin volume and
> allow it to run consistently and reliably on XFS.
>
> Signed-off-by: Brian Foster <bfoster@redhat.com>
> ---
Reviewed-by: Amir Goldstein <amir73il@gmail.com>


>  tests/generic/455 | 36 ++++++++++++++++++++++--------------
>  1 file changed, 22 insertions(+), 14 deletions(-)
>
> diff --git a/tests/generic/455 b/tests/generic/455
> index 05621220..72a44fda 100755
> --- a/tests/generic/455
> +++ b/tests/generic/455
> @@ -16,12 +16,14 @@ status=1    # failure is the default!
>  _cleanup()
>  {
>         _log_writes_cleanup
> +       _dmthin_cleanup
>  }
>  trap "_cleanup; exit \$status" 0 1 2 3 15
>
>  # get standard environment, filters and checks
>  . ./common/rc
>  . ./common/filter
> +. ./common/dmthin
>  . ./common/dmlogwrites
>
>  # real QA test starts here
> @@ -30,6 +32,7 @@ _supported_os Linux
>  _require_test
>  _require_scratch_nocheck
>  _require_log_writes
> +_require_dm_target thin-pool
>
>  rm -f $seqres.full
>
> @@ -42,13 +45,12 @@ check_files()
>                 local filename=$(basename $i)
>                 local mark="${filename##*.}"
>                 echo "checking $filename" >> $seqres.full
> -               _log_writes_replay_log $filename $SCRATCH_DEV
> -               _scratch_mount
> +               _log_writes_replay_log $filename $DMTHIN_VOL_DEV
> +               _dmthin_mount
>                 local expected_md5=$(_md5_checksum $i)
>                 local md5=$(_md5_checksum $SCRATCH_MNT/$name)
>                 [ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
> -               _scratch_unmount
> -               _check_scratch_fs
> +               _dmthin_check_fs
>         done
>  }
>
> @@ -56,8 +58,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
>  rm -rf $SANITY_DIR
>  mkdir $SANITY_DIR
>
> +devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
> +csize=$((1024*64 / 512))                # 64k cluster size
> +lowspace=$((1024*1024 / 512))           # 1m low space threshold
> +
> +# Use a thin device to provide deterministic discard behavior. Discards are used
> +# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
> +_dmthin_init $devsize $devsize $csize $lowspace
> +
>  # Create the log
> -_log_writes_init $SCRATCH_DEV
> +_log_writes_init $DMTHIN_VOL_DEV
>
>  _log_writes_mkfs >> $seqres.full 2>&1
>
> @@ -88,14 +98,13 @@ _log_writes_mark last
>  _log_writes_unmount
>  _log_writes_mark end
>  _log_writes_remove
> -_check_scratch_fs
> +_dmthin_check_fs
>
>  # check pre umount
>  echo "checking pre umount" >> $seqres.full
> -_log_writes_replay_log last $SCRATCH_DEV
> -_scratch_mount
> -_scratch_unmount
> -_check_scratch_fs
> +_log_writes_replay_log last $DMTHIN_VOL_DEV
> +_dmthin_mount
> +_dmthin_check_fs
>
>  for j in `seq 0 $((NUM_FILES-1))`; do
>         check_files testfile$j
> @@ -103,14 +112,13 @@ done
>
>  # Check the end
>  echo "checking post umount" >> $seqres.full
> -_log_writes_replay_log end $SCRATCH_DEV
> -_scratch_mount
> +_log_writes_replay_log end $DMTHIN_VOL_DEV
> +_dmthin_mount
>  for j in `seq 0 $((NUM_FILES-1))`; do
>         md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
>         [ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
>  done
> -_scratch_unmount
> -_check_scratch_fs
> +_dmthin_check_fs
>
>  echo "Silence is golden"
>  status=0
> --
> 2.25.4
>

Re: [PATCH v3 2/3] generic/457: use thin volume for dmlogwrites target device

[Amir Goldstein]

On Tue, Sep 1, 2020 at 4:49 PM Brian Foster <bfoster@redhat.com> wrote:
>
> dmlogwrites support for XFS depends on discard zeroing support of
> the intended target device. Update the test to use a thin volume and
> allow it to run consistently and reliably on XFS.
>
> Signed-off-by: Brian Foster <bfoster@redhat.com>
> ---

Reviewed-by: Amir Goldstein <amir73il@gmail.com>

>  tests/generic/457 | 33 +++++++++++++++++++++------------
>  1 file changed, 21 insertions(+), 12 deletions(-)
>
> diff --git a/tests/generic/457 b/tests/generic/457
> index 82367304..42a064d8 100755
> --- a/tests/generic/457
> +++ b/tests/generic/457
> @@ -16,6 +16,7 @@ status=1      # failure is the default!
>  _cleanup()
>  {
>         _log_writes_cleanup
> +       _dmthin_cleanup
>  }
>  trap "_cleanup; exit \$status" 0 1 2 3 15
>
> @@ -23,6 +24,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
>  . ./common/rc
>  . ./common/filter
>  . ./common/reflink
> +. ./common/dmthin
>  . ./common/dmlogwrites
>
>  # real QA test starts here
> @@ -32,6 +34,7 @@ _require_test
>  _require_scratch_reflink
>  _require_cp_reflink
>  _require_log_writes
> +_require_dm_target thin-pool
>
>  rm -f $seqres.full
>
> @@ -44,13 +47,12 @@ check_files()
>                 local filename=$(basename $i)
>                 local mark="${filename##*.}"
>                 echo "checking $filename" >> $seqres.full
> -               _log_writes_replay_log $filename $SCRATCH_DEV
> -               _scratch_mount
> +               _log_writes_replay_log $filename $DMTHIN_VOL_DEV
> +               _dmthin_mount
>                 local expected_md5=$(_md5_checksum $i)
>                 local md5=$(_md5_checksum $SCRATCH_MNT/$name)
>                 [ "${md5}" != "${expected_md5}" ] && _fail "$filename md5sum mismatched"
> -               _scratch_unmount
> -               _check_scratch_fs
> +               _dmthin_check_fs
>         done
>  }
>
> @@ -58,8 +60,16 @@ SANITY_DIR=$TEST_DIR/fsxtests
>  rm -rf $SANITY_DIR
>  mkdir $SANITY_DIR
>
> +devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
> +csize=$((1024*64 / 512))                # 64k cluster size
> +lowspace=$((1024*1024 / 512))           # 1m low space threshold
> +
> +# Use a thin device to provide deterministic discard behavior. Discards are used
> +# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
> +_dmthin_init $devsize $devsize $csize $lowspace
> +
>  # Create the log
> -_log_writes_init $SCRATCH_DEV
> +_log_writes_init $DMTHIN_VOL_DEV
>
>  _log_writes_mkfs >> $seqres.full 2>&1
>
> @@ -92,14 +102,13 @@ _log_writes_mark last
>  _log_writes_unmount
>  _log_writes_mark end
>  _log_writes_remove
> -_check_scratch_fs
> +_dmthin_check_fs
>
>  # check pre umount
>  echo "checking pre umount" >> $seqres.full
> -_log_writes_replay_log last $SCRATCH_DEV
> -_scratch_mount
> -_scratch_unmount
> -_check_scratch_fs
> +_log_writes_replay_log last $DMTHIN_VOL_DEV
> +_dmthin_mount
> +_dmthin_check_fs
>
>  for j in `seq 0 $((NUM_FILES-1))`; do
>         check_files testfile$j
> @@ -107,8 +116,8 @@ done
>
>  # Check the end
>  echo "checking post umount" >> $seqres.full
> -_log_writes_replay_log end $SCRATCH_DEV
> -_scratch_mount
> +_log_writes_replay_log end $DMTHIN_VOL_DEV
> +_dmthin_mount
>  for j in `seq 0 $((NUM_FILES-1))`; do
>         md5=$(_md5_checksum $SCRATCH_MNT/testfile$j)
>         [ "${md5}" != "${test_md5[$j]}" ] && _fail "testfile$j end md5sum mismatched"
> --
> 2.25.4
>

Re: [PATCH v3 3/3] generic/470: use thin volume for dmlogwrites target device

[Amir Goldstein]

On Tue, Sep 1, 2020 at 4:48 PM Brian Foster <bfoster@redhat.com> wrote:
>
> dmlogwrites support for XFS depends on discard zeroing support of
> the intended target device. Update the test to use a thin volume and
> allow it to run consistently and reliably on XFS.
>
> Signed-off-by: Brian Foster <bfoster@redhat.com>
> ---
Reviewed-by: Amir Goldstein <amir73il@gmail.com>

>  tests/generic/470 | 23 +++++++++++++++++------
>  1 file changed, 17 insertions(+), 6 deletions(-)
>
> diff --git a/tests/generic/470 b/tests/generic/470
> index fd6da563..c77499a2 100755
> --- a/tests/generic/470
> +++ b/tests/generic/470
> @@ -20,12 +20,14 @@ _cleanup()
>  {
>         cd /
>         _log_writes_cleanup
> +       _dmthin_cleanup
>         rm -f $tmp.*
>  }
>
>  # get standard environment, filters and checks
>  . ./common/rc
>  . ./common/filter
> +. ./common/dmthin
>  . ./common/dmlogwrites
>
>  # remove previous $seqres.full before test
> @@ -34,12 +36,21 @@ rm -f $seqres.full
>  # real QA test starts here
>  _supported_fs generic
>  _supported_os Linux
> -_require_scratch
> +_require_scratch_nocheck
>  _require_log_writes_dax_mountopt "dax"
> +_require_dm_target thin-pool
>  _require_xfs_io_command "mmap" "-S"
>  _require_xfs_io_command "log_writes"
>
> -_log_writes_init $SCRATCH_DEV
> +devsize=$((1024*1024*200 / 512))        # 200m phys/virt size
> +csize=$((1024*64 / 512))                # 64k cluster size
> +lowspace=$((1024*1024 / 512))           # 1m low space threshold
> +
> +# Use a thin device to provide deterministic discard behavior. Discards are used
> +# by the log replay tool for fast zeroing to prevent out-of-order replay issues.
> +_dmthin_init $devsize $devsize $csize $lowspace
> +
> +_log_writes_init $DMTHIN_VOL_DEV
>  _log_writes_mkfs >> $seqres.full 2>&1
>  _log_writes_mount -o dax
>
> @@ -52,14 +63,14 @@ $XFS_IO_PROG -t -c "truncate $LEN" -c "mmap -S 0 $LEN" -c "mwrite 0 $LEN" \
>  # Unmount the scratch dir and tear down the log writes target
>  _log_writes_unmount
>  _log_writes_remove
> -_check_scratch_fs
> +_dmthin_check_fs
>
>  # destroy previous filesystem so we can be sure our rebuild works
> -_scratch_mkfs >> $seqres.full 2>&1
> +_mkfs_dev $DMTHIN_VOL_DEV >> $seqres.full 2>&1
>
>  # check pre-unmap state
> -_log_writes_replay_log preunmap $SCRATCH_DEV
> -_scratch_mount
> +_log_writes_replay_log preunmap $DMTHIN_VOL_DEV
> +_dmthin_mount
>
>  # We should see $SCRATCH_MNT/test as having 1 MiB in block allocations
>  du -sh $SCRATCH_MNT/test | _filter_scratch | _filter_spaces
> --
> 2.25.4
>