[PATCH RFCv4 0/2] xfstests: add inode btree blocks counters to the AGI header

[PATCH RFCv4 0/2] xfstests: add inode btree blocks counters to the AGI header

[Darrick J. Wong]

Hi all,

Years ago, Christoph diagnosed a problem where freeing an inode on a
totally full filesystem could fail due to finobt expansion not being
able to allocate enough blocks.  He solved the problem by using the
per-AG block reservation system to ensure that there are always enough
blocks for finobt expansion, but that came at the cost of having to walk
the entire finobt at mount time.  This new feature solves that
performance regression by adding inode btree block counts to the AGI
header.  The patches in this series amend fstests to handle the new
metadata fields and to test that upgrades work properly.

v2: rebase kernel to 5.9
v3: split logical changes into separate patches
v4: support inobtcounts && !finobt properly

If you're going to start using this mess, you probably ought to just
pull from my git trees, which are linked below.

This is an extraordinary way to destroy everything.  Enjoy!
Comments and questions are, as always, welcome.

--D

kernel git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfs-linux.git/log/?h=inobt-counters

xfsprogs git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfsprogs-dev.git/log/?h=inobt-counters

fstests git tree:
https://git.kernel.org/cgit/linux/kernel/git/djwong/xfstests-dev.git/log/?h=inobt-counters
---
 common/xfs        |   16 ++++++++++++
 tests/xfs/010     |    3 +-
 tests/xfs/030     |    2 +
 tests/xfs/122.out |    2 +
 tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/xfs/910.out |    3 ++
 tests/xfs/group   |    1 +
 7 files changed, 97 insertions(+), 2 deletions(-)
 create mode 100755 tests/xfs/910
 create mode 100644 tests/xfs/910.out

[PATCH 1/2] xfs/122: embiggen struct xfs_agi size for inobtcount feature

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

Make the expected AGI size larger for the inobtcount feature.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 tests/xfs/010     |    3 ++-
 tests/xfs/030     |    2 ++
 tests/xfs/122.out |    2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)


diff --git a/tests/xfs/010 b/tests/xfs/010
index 1f9bcdff..95cc2555 100755
--- a/tests/xfs/010
+++ b/tests/xfs/010
@@ -113,7 +113,8 @@ _check_scratch_fs
 _corrupt_finobt_root $SCRATCH_DEV
 
 filter_finobt_repair() {
-	sed -e '/^agi has bad CRC/d' | \
+	sed -e '/^agi has bad CRC/d' \
+	    -e '/^bad finobt block/d' | \
 		_filter_repair_lostblocks
 }
 
diff --git a/tests/xfs/030 b/tests/xfs/030
index 04440f9c..906d9019 100755
--- a/tests/xfs/030
+++ b/tests/xfs/030
@@ -44,6 +44,8 @@ _check_ag()
 			    -e '/^bad agbno AGBNO for refcntbt/d' \
 			    -e '/^agf has bad CRC/d' \
 			    -e '/^agi has bad CRC/d' \
+			    -e '/^bad inobt block count/d' \
+			    -e '/^bad finobt block count/d' \
 			    -e '/^Missing reverse-mapping record.*/d' \
 			    -e '/^bad levels LEVELS for [a-z]* root.*/d' \
 			    -e '/^unknown block state, ag AGNO, block.*/d'
diff --git a/tests/xfs/122.out b/tests/xfs/122.out
index cfe09c6d..b0773756 100644
--- a/tests/xfs/122.out
+++ b/tests/xfs/122.out
@@ -113,7 +113,7 @@ sizeof(struct xfs_scrub_metadata) = 64
 sizeof(struct xfs_unmount_log_format) = 8
 sizeof(xfs_agf_t) = 224
 sizeof(xfs_agfl_t) = 36
-sizeof(xfs_agi_t) = 336
+sizeof(xfs_agi_t) = 344
 sizeof(xfs_alloc_key_t) = 8
 sizeof(xfs_alloc_rec_incore_t) = 8
 sizeof(xfs_alloc_rec_t) = 8

[PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

From: Darrick J. Wong <darrick.wong@oracle.com>

Make sure we can actually upgrade filesystems to support inobtcounts.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
---
 common/xfs        |   16 ++++++++++++
 tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/xfs/910.out |    3 ++
 tests/xfs/group   |    1 +
 4 files changed, 92 insertions(+)
 create mode 100755 tests/xfs/910
 create mode 100644 tests/xfs/910.out


diff --git a/common/xfs b/common/xfs
index 3f5c14ba..e548a0a1 100644
--- a/common/xfs
+++ b/common/xfs
@@ -978,3 +978,19 @@ _require_xfs_copy()
 	[ "$USE_EXTERNAL" = yes ] && \
 		_notrun "Cannot xfs_copy with external devices"
 }
+
+_require_xfs_mkfs_inobtcount()
+{
+	_scratch_mkfs_xfs_supported -m inobtcount=1 >/dev/null 2>&1 \
+	   || _notrun "mkfs.xfs doesn't have inobtcount feature"
+}
+
+_require_xfs_scratch_inobtcount()
+{
+	_require_scratch
+
+	_scratch_mkfs -m inobtcount=1 > /dev/null
+	_try_scratch_mount || \
+		_notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
+	_scratch_unmount
+}
diff --git a/tests/xfs/910 b/tests/xfs/910
new file mode 100755
index 00000000..1924d9ea
--- /dev/null
+++ b/tests/xfs/910
@@ -0,0 +1,72 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2020, Oracle and/or its affiliates.  All Rights Reserved.
+#
+# FS QA Test No. 910
+#
+# Check that we can upgrade a filesystem to support inobtcount and that
+# everything works properly after the upgrade.
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1    # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# We have very specific formatting parameters, so don't let things get complex
+# with realtime devices and external logs.
+unset USE_EXTERNAL
+
+# real QA test starts here
+_supported_fs xfs
+_require_command "$XFS_ADMIN_PROG" "xfs_admin"
+_require_xfs_mkfs_inobtcount
+_require_xfs_scratch_inobtcount
+
+rm -f $seqres.full
+
+# Make sure we can't format a filesystem with inobtcount and not finobt.
+_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
+	echo "Should not be able to format with inobtcount but not finobt."
+
+# Make sure we can't upgrade a filesystem to inobtcount without finobt.
+_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 &> $seqres.full
+_scratch_xfs_admin -O inobtcount >> $seqres.full && \
+	echo "Should not be able to upgrade to inobtcount without finobt."
+
+# Format V5 filesystem without inode btree counter support and populate it
+_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
+_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' >> $seqres.full
+_scratch_mount >> $seqres.full
+
+echo moo > $SCRATCH_MNT/urk
+
+_scratch_unmount
+_check_scratch_fs
+
+# Now upgrade to inobtcount support
+_scratch_xfs_admin -O inobtcount >> $seqres.full
+_check_scratch_fs
+_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
+
+# Mount again, look at our files
+_scratch_mount >> $seqres.full
+cat $SCRATCH_MNT/urk
+
+# success, all done
+echo Silence is golden.
+status=0
+exit
diff --git a/tests/xfs/910.out b/tests/xfs/910.out
new file mode 100644
index 00000000..83992f49
--- /dev/null
+++ b/tests/xfs/910.out
@@ -0,0 +1,3 @@
+QA output created by 910
+moo
+Silence is golden.
diff --git a/tests/xfs/group b/tests/xfs/group
index 4b0caea4..862df3be 100644
--- a/tests/xfs/group
+++ b/tests/xfs/group
@@ -524,6 +524,7 @@
 760 auto quick rw collapse punch insert zero prealloc
 761 auto quick realtime
 763 auto quick rw realtime
+910 auto quick inobtcount
 915 auto quick quota
 917 auto quick db
 918 auto quick db

Re: [PATCH 1/2] xfs/122: embiggen struct xfs_agi size for inobtcount feature

[Brian Foster]

On Tue, Oct 27, 2020 at 12:03:42PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Make the expected AGI size larger for the inobtcount feature.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---

It would be nice for the commit log to have a sentence or two about the
other changes as well, but either way:

Reviewed-by: Brian Foster <bfoster@redhat.com>

>  tests/xfs/010     |    3 ++-
>  tests/xfs/030     |    2 ++
>  tests/xfs/122.out |    2 +-
>  3 files changed, 5 insertions(+), 2 deletions(-)
> 
> 
> diff --git a/tests/xfs/010 b/tests/xfs/010
> index 1f9bcdff..95cc2555 100755
> --- a/tests/xfs/010
> +++ b/tests/xfs/010
> @@ -113,7 +113,8 @@ _check_scratch_fs
>  _corrupt_finobt_root $SCRATCH_DEV
>  
>  filter_finobt_repair() {
> -	sed -e '/^agi has bad CRC/d' | \
> +	sed -e '/^agi has bad CRC/d' \
> +	    -e '/^bad finobt block/d' | \
>  		_filter_repair_lostblocks
>  }
>  
> diff --git a/tests/xfs/030 b/tests/xfs/030
> index 04440f9c..906d9019 100755
> --- a/tests/xfs/030
> +++ b/tests/xfs/030
> @@ -44,6 +44,8 @@ _check_ag()
>  			    -e '/^bad agbno AGBNO for refcntbt/d' \
>  			    -e '/^agf has bad CRC/d' \
>  			    -e '/^agi has bad CRC/d' \
> +			    -e '/^bad inobt block count/d' \
> +			    -e '/^bad finobt block count/d' \
>  			    -e '/^Missing reverse-mapping record.*/d' \
>  			    -e '/^bad levels LEVELS for [a-z]* root.*/d' \
>  			    -e '/^unknown block state, ag AGNO, block.*/d'
> diff --git a/tests/xfs/122.out b/tests/xfs/122.out
> index cfe09c6d..b0773756 100644
> --- a/tests/xfs/122.out
> +++ b/tests/xfs/122.out
> @@ -113,7 +113,7 @@ sizeof(struct xfs_scrub_metadata) = 64
>  sizeof(struct xfs_unmount_log_format) = 8
>  sizeof(xfs_agf_t) = 224
>  sizeof(xfs_agfl_t) = 36
> -sizeof(xfs_agi_t) = 336
> +sizeof(xfs_agi_t) = 344
>  sizeof(xfs_alloc_key_t) = 8
>  sizeof(xfs_alloc_rec_incore_t) = 8
>  sizeof(xfs_alloc_rec_t) = 8
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Brian Foster]

On Tue, Oct 27, 2020 at 12:03:48PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Make sure we can actually upgrade filesystems to support inobtcounts.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---
>  common/xfs        |   16 ++++++++++++
>  tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/xfs/910.out |    3 ++
>  tests/xfs/group   |    1 +
>  4 files changed, 92 insertions(+)
>  create mode 100755 tests/xfs/910
>  create mode 100644 tests/xfs/910.out
> 
> 
...
> diff --git a/tests/xfs/910 b/tests/xfs/910
> new file mode 100755
> index 00000000..1924d9ea
> --- /dev/null
> +++ b/tests/xfs/910
> @@ -0,0 +1,72 @@
...
> +
> +# Now upgrade to inobtcount support
> +_scratch_xfs_admin -O inobtcount >> $seqres.full
> +_check_scratch_fs
> +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
> +
> +# Mount again, look at our files
> +_scratch_mount >> $seqres.full
> +cat $SCRATCH_MNT/urk
> +

I think we probably want some more explicit form of validation here.
Perhaps dump the inobt block counters from the above xfs_db command to
the golden output..? As it is, we can comment out the xfs_admin command
and the test still passes.

> +# success, all done
> +echo Silence is golden.

We can also probably drop this if we have some other form of output from
the test.

Brian

> +status=0
> +exit
> diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> new file mode 100644
> index 00000000..83992f49
> --- /dev/null
> +++ b/tests/xfs/910.out
> @@ -0,0 +1,3 @@
> +QA output created by 910
> +moo
> +Silence is golden.
> diff --git a/tests/xfs/group b/tests/xfs/group
> index 4b0caea4..862df3be 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -524,6 +524,7 @@
>  760 auto quick rw collapse punch insert zero prealloc
>  761 auto quick realtime
>  763 auto quick rw realtime
> +910 auto quick inobtcount
>  915 auto quick quota
>  917 auto quick db
>  918 auto quick db
> 

Re: [PATCH 1/2] xfs/122: embiggen struct xfs_agi size for inobtcount feature

[Darrick J. Wong]

On Thu, Oct 29, 2020 at 01:39:20PM -0400, Brian Foster wrote:
> On Tue, Oct 27, 2020 at 12:03:42PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> > 
> > Make the expected AGI size larger for the inobtcount feature.
> > 
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> 
> It would be nice for the commit log to have a sentence or two about the
> other changes as well, but either way:

I think I'll just make them a separate patch.

--D

> Reviewed-by: Brian Foster <bfoster@redhat.com>
> 
> >  tests/xfs/010     |    3 ++-
> >  tests/xfs/030     |    2 ++
> >  tests/xfs/122.out |    2 +-
> >  3 files changed, 5 insertions(+), 2 deletions(-)
> > 
> > 
> > diff --git a/tests/xfs/010 b/tests/xfs/010
> > index 1f9bcdff..95cc2555 100755
> > --- a/tests/xfs/010
> > +++ b/tests/xfs/010
> > @@ -113,7 +113,8 @@ _check_scratch_fs
> >  _corrupt_finobt_root $SCRATCH_DEV
> >  
> >  filter_finobt_repair() {
> > -	sed -e '/^agi has bad CRC/d' | \
> > +	sed -e '/^agi has bad CRC/d' \
> > +	    -e '/^bad finobt block/d' | \
> >  		_filter_repair_lostblocks
> >  }
> >  
> > diff --git a/tests/xfs/030 b/tests/xfs/030
> > index 04440f9c..906d9019 100755
> > --- a/tests/xfs/030
> > +++ b/tests/xfs/030
> > @@ -44,6 +44,8 @@ _check_ag()
> >  			    -e '/^bad agbno AGBNO for refcntbt/d' \
> >  			    -e '/^agf has bad CRC/d' \
> >  			    -e '/^agi has bad CRC/d' \
> > +			    -e '/^bad inobt block count/d' \
> > +			    -e '/^bad finobt block count/d' \
> >  			    -e '/^Missing reverse-mapping record.*/d' \
> >  			    -e '/^bad levels LEVELS for [a-z]* root.*/d' \
> >  			    -e '/^unknown block state, ag AGNO, block.*/d'
> > diff --git a/tests/xfs/122.out b/tests/xfs/122.out
> > index cfe09c6d..b0773756 100644
> > --- a/tests/xfs/122.out
> > +++ b/tests/xfs/122.out
> > @@ -113,7 +113,7 @@ sizeof(struct xfs_scrub_metadata) = 64
> >  sizeof(struct xfs_unmount_log_format) = 8
> >  sizeof(xfs_agf_t) = 224
> >  sizeof(xfs_agfl_t) = 36
> > -sizeof(xfs_agi_t) = 336
> > +sizeof(xfs_agi_t) = 344
> >  sizeof(xfs_alloc_key_t) = 8
> >  sizeof(xfs_alloc_rec_incore_t) = 8
> >  sizeof(xfs_alloc_rec_t) = 8
> > 
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

On Thu, Oct 29, 2020 at 01:40:23PM -0400, Brian Foster wrote:
> On Tue, Oct 27, 2020 at 12:03:48PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> > 
> > Make sure we can actually upgrade filesystems to support inobtcounts.
> > 
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> >  common/xfs        |   16 ++++++++++++
> >  tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/xfs/910.out |    3 ++
> >  tests/xfs/group   |    1 +
> >  4 files changed, 92 insertions(+)
> >  create mode 100755 tests/xfs/910
> >  create mode 100644 tests/xfs/910.out
> > 
> > 
> ...
> > diff --git a/tests/xfs/910 b/tests/xfs/910
> > new file mode 100755
> > index 00000000..1924d9ea
> > --- /dev/null
> > +++ b/tests/xfs/910
> > @@ -0,0 +1,72 @@
> ...
> > +
> > +# Now upgrade to inobtcount support
> > +_scratch_xfs_admin -O inobtcount >> $seqres.full
> > +_check_scratch_fs
> > +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
> > +
> > +# Mount again, look at our files
> > +_scratch_mount >> $seqres.full
> > +cat $SCRATCH_MNT/urk
> > +
> 
> I think we probably want some more explicit form of validation here.
> Perhaps dump the inobt block counters from the above xfs_db command to
> the golden output..? As it is, we can comment out the xfs_admin command
> and the test still passes.

Ok.

> > +# success, all done
> > +echo Silence is golden.
> 
> We can also probably drop this if we have some other form of output from
> the test.

Will do.  Thanks for reviewing!

> Brian
> 
> > +status=0
> > +exit
> > diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> > new file mode 100644
> > index 00000000..83992f49
> > --- /dev/null
> > +++ b/tests/xfs/910.out
> > @@ -0,0 +1,3 @@
> > +QA output created by 910
> > +moo
> > +Silence is golden.
> > diff --git a/tests/xfs/group b/tests/xfs/group
> > index 4b0caea4..862df3be 100644
> > --- a/tests/xfs/group
> > +++ b/tests/xfs/group
> > @@ -524,6 +524,7 @@
> >  760 auto quick rw collapse punch insert zero prealloc
> >  761 auto quick realtime
> >  763 auto quick rw realtime
> > +910 auto quick inobtcount
> >  915 auto quick quota
> >  917 auto quick db
> >  918 auto quick db
> > 
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Eric Sandeen]

On 10/27/20 2:03 PM, Darrick J. Wong wrote:
> From: Darrick J. Wong <darrick.wong@oracle.com>
> 
> Make sure we can actually upgrade filesystems to support inobtcounts.
> 
> Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> ---
>  common/xfs        |   16 ++++++++++++
>  tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/xfs/910.out |    3 ++
>  tests/xfs/group   |    1 +
>  4 files changed, 92 insertions(+)
>  create mode 100755 tests/xfs/910
>  create mode 100644 tests/xfs/910.out
> 
> 
> diff --git a/common/xfs b/common/xfs
> index 3f5c14ba..e548a0a1 100644
> --- a/common/xfs
> +++ b/common/xfs
> @@ -978,3 +978,19 @@ _require_xfs_copy()
>  	[ "$USE_EXTERNAL" = yes ] && \
>  		_notrun "Cannot xfs_copy with external devices"
>  }
> +
> +_require_xfs_mkfs_inobtcount()
> +{
> +	_scratch_mkfs_xfs_supported -m inobtcount=1 >/dev/null 2>&1 \
> +	   || _notrun "mkfs.xfs doesn't have inobtcount feature"
> +}

I'd like to also add:

+_require_xfs_admin_upgrade()
+{
+	local feature="$1"
+
+	_require_scratch
+	# Catch missing "-O" or missing feature handling
+	_scratch_xfs_admin -O $feature 2>&1 | grep "illegal option\|Cannot change" \
+		&& _notrun "xfs_admin does not support upgrading $1"
+}

and then:

...

> +# real QA test starts here
> +_supported_fs xfs
> +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> +_require_xfs_mkfs_inobtcount
> +_require_xfs_scratch_inobtcount

+_require_xfs_admin_upgrade "inobtcount"

to be sure that the upgrade command is also supported by xfs_admin.  By the time
we get to release, both mkfs & xfs_admin should both support it, but I'm hedging
my bets on the upgrade path just a little,and it seems best to explicitly test
both requirements.

that helper can be re-used on the bigtime upgrade test as well and anything else
that requires an xfs_admin upgrade path ...

Thanks,
-Eric

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

On Wed, Nov 18, 2020 at 10:44:40AM -0600, Eric Sandeen wrote:
> On 10/27/20 2:03 PM, Darrick J. Wong wrote:
> > From: Darrick J. Wong <darrick.wong@oracle.com>
> > 
> > Make sure we can actually upgrade filesystems to support inobtcounts.
> > 
> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
> > ---
> >  common/xfs        |   16 ++++++++++++
> >  tests/xfs/910     |   72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/xfs/910.out |    3 ++
> >  tests/xfs/group   |    1 +
> >  4 files changed, 92 insertions(+)
> >  create mode 100755 tests/xfs/910
> >  create mode 100644 tests/xfs/910.out
> > 
> > 
> > diff --git a/common/xfs b/common/xfs
> > index 3f5c14ba..e548a0a1 100644
> > --- a/common/xfs
> > +++ b/common/xfs
> > @@ -978,3 +978,19 @@ _require_xfs_copy()
> >  	[ "$USE_EXTERNAL" = yes ] && \
> >  		_notrun "Cannot xfs_copy with external devices"
> >  }
> > +
> > +_require_xfs_mkfs_inobtcount()
> > +{
> > +	_scratch_mkfs_xfs_supported -m inobtcount=1 >/dev/null 2>&1 \
> > +	   || _notrun "mkfs.xfs doesn't have inobtcount feature"
> > +}
> 
> I'd like to also add:
> 
> +_require_xfs_admin_upgrade()
> +{
> +	local feature="$1"
> +
> +	_require_scratch
> +	# Catch missing "-O" or missing feature handling
> +	_scratch_xfs_admin -O $feature 2>&1 | grep "illegal option\|Cannot change" \
> +		&& _notrun "xfs_admin does not support upgrading $1"
> +}

Well it occurred to me that the xfs_db version command has a help
screen that lists all of the things that it knows how to upgrade.
So, it should be easy enough to detect whether or not it makes sense to
test an upgrade path.  Thanks for the suggestion, sorry I was rambling
about this being harder a few days ago... :/

--D

> 
> and then:
> 
> ...
> 
> > +# real QA test starts here
> > +_supported_fs xfs
> > +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> > +_require_xfs_mkfs_inobtcount
> > +_require_xfs_scratch_inobtcount
> 
> +_require_xfs_admin_upgrade "inobtcount"
> 
> to be sure that the upgrade command is also supported by xfs_admin.  By the time
> we get to release, both mkfs & xfs_admin should both support it, but I'm hedging
> my bets on the upgrade path just a little,and it seems best to explicitly test
> both requirements.
> 
> that helper can be re-used on the bigtime upgrade test as well and anything else
> that requires an xfs_admin upgrade path ...
> 
> Thanks,
> -Eric

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

On Wed, Apr 21, 2021 at 08:52:39AM +0300, Amir Goldstein wrote:
> On Wed, Apr 21, 2021 at 3:23 AM Darrick J. Wong <djwong@kernel.org> wrote:
> >
> > From: Darrick J. Wong <djwong@kernel.org>
> >
> > Make sure we can actually upgrade filesystems to support inode btree
> > counters.
> >
> > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > ---
> >  common/xfs        |    8 +++-
> >  tests/xfs/910     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/xfs/910.out |   23 ++++++++++++
> >  tests/xfs/group   |    1 +
> >  4 files changed, 127 insertions(+), 3 deletions(-)
> >  create mode 100755 tests/xfs/910
> >  create mode 100644 tests/xfs/910.out
> >
> >
> > diff --git a/common/xfs b/common/xfs
> > index 5abc7034..3d660858 100644
> > --- a/common/xfs
> > +++ b/common/xfs
> > @@ -1153,13 +1153,15 @@ _require_xfs_repair_upgrade()
> >                 _notrun "xfs_repair does not support upgrading fs with $type"
> >  }
> >
> > -_require_xfs_scratch_inobtcount()
> > +# Require that the scratch device exists, that mkfs can format with inobtcount
> > +# enabled, and that the kernel can mount such a filesystem.
> > +_require_scratch_xfs_inobtcount()

<groan> I clearly changed the wrong patch in response to reviewer
comments, sorry, will resubmit...

> >  {
> >         _require_scratch
> >
> >         _scratch_mkfs -m inobtcount=1 &> /dev/null || \
> > -               _notrun "mkfs.xfs doesn't have inobtcount feature"
> > +               _notrun "mkfs.xfs doesn't support inobtcount feature"
> >         _try_scratch_mount || \
> > -               _notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
> > +               _notrun "kernel doesn't support xfs inobtcount feature"
> >         _scratch_unmount
> >  }
> > diff --git a/tests/xfs/910 b/tests/xfs/910
> > new file mode 100755
> > index 00000000..237d0a35
> > --- /dev/null
> > +++ b/tests/xfs/910
> > @@ -0,0 +1,98 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0-or-later
> > +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 910
> > +#
> > +# Check that we can upgrade a filesystem to support inobtcount and that
> > +# everything works properly after the upgrade.
> > +
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +
> > +here=`pwd`
> > +tmp=/tmp/$$
> > +status=1    # failure is the default!
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +       cd /
> > +       rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/filter
> > +
> > +# real QA test starts here
> > +_supported_fs xfs
> > +_require_scratch_xfs_inobtcount
> > +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> > +_require_xfs_repair_upgrade inobtcount
> > +
> > +rm -f $seqres.full
> > +
> > +# Make sure we can't format a filesystem with inobtcount and not finobt.
> > +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> > +       echo "Should not be able to format with inobtcount but not finobt."
> > +
> > +# Make sure we can't upgrade a V4 filesystem
> > +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_check_scratch_xfs_features INOBTCNT
> > +
> > +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> > +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_check_scratch_xfs_features INOBTCNT
> > +
> > +# Format V5 filesystem without inode btree counter support and populate it.
> > +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> > +_scratch_mount
> > +
> > +mkdir $SCRATCH_MNT/stress
> > +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> > +echo moo > $SCRATCH_MNT/urk
> > +
> > +_scratch_unmount
> > +
> > +# Upgrade filesystem to have the counters and inject failure into repair and
> > +# make sure that the only path forward is to re-run repair on the filesystem.
> > +echo "Fail partway through upgrading"
> > +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> > +test $? -eq 137 || echo "repair should have been killed??"
> > +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> > +_try_scratch_mount &> $tmp.mount
> > +res=$?
> > +_filter_scratch < $tmp.mount
> > +if [ $res -eq 0 ]; then
> > +       echo "needsrepair should have prevented mount"
> > +       _scratch_unmount
> > +fi
> > +
> > +echo "Re-run repair to finish upgrade"
> > +_scratch_xfs_repair 2>> $seqres.full
> > +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> > +
> > +echo "Filesystem should be usable again"
> > +_scratch_mount
> > +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> > +_scratch_unmount
> > +_check_scratch_fs
> > +_check_scratch_xfs_features INOBTCNT
> > +
> > +echo "Make sure we have nonzero counters"
> > +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> > +       sed -e 's/= [1-9]*/= NONZERO/g'
> > +
> 
> I don't think = 100 translates to = NONZERO...

Urk, yes, will fix.

> > +echo "Make sure we can't re-add inobtcount"
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +
> > +echo "Mount again, look at our files"
> > +_scratch_mount >> $seqres.full
> > +cat $SCRATCH_MNT/urk
> > +
> > +status=0
> > +exit
> > diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> > new file mode 100644
> > index 00000000..1bf040d5
> > --- /dev/null
> > +++ b/tests/xfs/910.out
> > @@ -0,0 +1,23 @@
> > +QA output created by 910
> > +Running xfs_repair to upgrade filesystem.
> > +Inode btree count feature only supported on V5 filesystems.
> > +FEATURES: INOBTCNT:NO
> > +Running xfs_repair to upgrade filesystem.
> > +Inode btree count feature requires free inode btree.
> > +FEATURES: INOBTCNT:NO
> > +Fail partway through upgrading
> > +Adding inode btree counts to filesystem.
> > +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> > +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> > +Re-run repair to finish upgrade
> > +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> > +Filesystem should be usable again
> > +FEATURES: INOBTCNT:YES
> > +Make sure we have nonzero counters
> > +ino_blocks = NONZERO
> > +fino_blocks = NONZERO
> > +Make sure we can't re-add inobtcount
> > +Running xfs_repair to upgrade filesystem.
> > +Filesystem already has inode btree counts.
> > +Mount again, look at our files
> > +moo
> 
> This test is quite rigid in the format of messages expected from
> xfs_repair. I suppose there is no precedent here...

<shrug> There /is/ precedent -- if we ever need to change the stdout, we
filter the golden output.

--D

> 
> > diff --git a/tests/xfs/group b/tests/xfs/group
> > index a2309465..bd47333c 100644
> > --- a/tests/xfs/group
> > +++ b/tests/xfs/group
> > @@ -526,3 +526,4 @@
> >  768 auto quick repair
> >  770 auto repair
> >  773 auto quick repair
> > +910 auto quick inobtcount
> >
> +repair?
> 
> Thanks,
> Amir.

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Allison Henderson]

On 4/20/21 5:22 PM, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> Make sure we can actually upgrade filesystems to support inode btree
> counters.
> 
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
Seems ok to me
Reviewed-by: Allison Henderson <allison.henderson@oracle.com>

> ---
>   common/xfs        |    8 +++-
>   tests/xfs/910     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>   tests/xfs/910.out |   23 ++++++++++++
>   tests/xfs/group   |    1 +
>   4 files changed, 127 insertions(+), 3 deletions(-)
>   create mode 100755 tests/xfs/910
>   create mode 100644 tests/xfs/910.out
> 
> 
> diff --git a/common/xfs b/common/xfs
> index 5abc7034..3d660858 100644
> --- a/common/xfs
> +++ b/common/xfs
> @@ -1153,13 +1153,15 @@ _require_xfs_repair_upgrade()
>   		_notrun "xfs_repair does not support upgrading fs with $type"
>   }
>   
> -_require_xfs_scratch_inobtcount()
> +# Require that the scratch device exists, that mkfs can format with inobtcount
> +# enabled, and that the kernel can mount such a filesystem.
> +_require_scratch_xfs_inobtcount()
>   {
>   	_require_scratch
>   
>   	_scratch_mkfs -m inobtcount=1 &> /dev/null || \
> -		_notrun "mkfs.xfs doesn't have inobtcount feature"
> +		_notrun "mkfs.xfs doesn't support inobtcount feature"
>   	_try_scratch_mount || \
> -		_notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
> +		_notrun "kernel doesn't support xfs inobtcount feature"
>   	_scratch_unmount
>   }
> diff --git a/tests/xfs/910 b/tests/xfs/910
> new file mode 100755
> index 00000000..237d0a35
> --- /dev/null
> +++ b/tests/xfs/910
> @@ -0,0 +1,98 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> +#
> +# FS QA Test No. 910
> +#
> +# Check that we can upgrade a filesystem to support inobtcount and that
> +# everything works properly after the upgrade.
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1    # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +
> +# real QA test starts here
> +_supported_fs xfs
> +_require_scratch_xfs_inobtcount
> +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> +_require_xfs_repair_upgrade inobtcount
> +
> +rm -f $seqres.full
> +
> +# Make sure we can't format a filesystem with inobtcount and not finobt.
> +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> +	echo "Should not be able to format with inobtcount but not finobt."
> +
> +# Make sure we can't upgrade a V4 filesystem
> +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Format V5 filesystem without inode btree counter support and populate it.
> +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> +_scratch_mount
> +
> +mkdir $SCRATCH_MNT/stress
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +echo moo > $SCRATCH_MNT/urk
> +
> +_scratch_unmount
> +
> +# Upgrade filesystem to have the counters and inject failure into repair and
> +# make sure that the only path forward is to re-run repair on the filesystem.
> +echo "Fail partway through upgrading"
> +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> +test $? -eq 137 || echo "repair should have been killed??"
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +_try_scratch_mount &> $tmp.mount
> +res=$?
> +_filter_scratch < $tmp.mount
> +if [ $res -eq 0 ]; then
> +	echo "needsrepair should have prevented mount"
> +	_scratch_unmount
> +fi
> +
> +echo "Re-run repair to finish upgrade"
> +_scratch_xfs_repair 2>> $seqres.full
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +
> +echo "Filesystem should be usable again"
> +_scratch_mount
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +_scratch_unmount
> +_check_scratch_fs
> +_check_scratch_xfs_features INOBTCNT
> +
> +echo "Make sure we have nonzero counters"
> +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> +	sed -e 's/= [1-9]*/= NONZERO/g'
> +
> +echo "Make sure we can't re-add inobtcount"
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +
> +echo "Mount again, look at our files"
> +_scratch_mount >> $seqres.full
> +cat $SCRATCH_MNT/urk
> +
> +status=0
> +exit
> diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> new file mode 100644
> index 00000000..1bf040d5
> --- /dev/null
> +++ b/tests/xfs/910.out
> @@ -0,0 +1,23 @@
> +QA output created by 910
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature only supported on V5 filesystems.
> +FEATURES: INOBTCNT:NO
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature requires free inode btree.
> +FEATURES: INOBTCNT:NO
> +Fail partway through upgrading
> +Adding inode btree counts to filesystem.
> +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> +Re-run repair to finish upgrade
> +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> +Filesystem should be usable again
> +FEATURES: INOBTCNT:YES
> +Make sure we have nonzero counters
> +ino_blocks = NONZERO
> +fino_blocks = NONZERO
> +Make sure we can't re-add inobtcount
> +Running xfs_repair to upgrade filesystem.
> +Filesystem already has inode btree counts.
> +Mount again, look at our files
> +moo
> diff --git a/tests/xfs/group b/tests/xfs/group
> index a2309465..bd47333c 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -526,3 +526,4 @@
>   768 auto quick repair
>   770 auto repair
>   773 auto quick repair
> +910 auto quick inobtcount
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Brian Foster]

On Tue, Apr 20, 2021 at 05:22:57PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> Make sure we can actually upgrade filesystems to support inode btree
> counters.
> 
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> ---

LGTM:

Reviewed-by: Brian Foster <bfoster@redhat.com>

>  common/xfs        |    8 +++-
>  tests/xfs/910     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/xfs/910.out |   23 ++++++++++++
>  tests/xfs/group   |    1 +
>  4 files changed, 127 insertions(+), 3 deletions(-)
>  create mode 100755 tests/xfs/910
>  create mode 100644 tests/xfs/910.out
> 
> 
> diff --git a/common/xfs b/common/xfs
> index 5abc7034..3d660858 100644
> --- a/common/xfs
> +++ b/common/xfs
> @@ -1153,13 +1153,15 @@ _require_xfs_repair_upgrade()
>  		_notrun "xfs_repair does not support upgrading fs with $type"
>  }
>  
> -_require_xfs_scratch_inobtcount()
> +# Require that the scratch device exists, that mkfs can format with inobtcount
> +# enabled, and that the kernel can mount such a filesystem.
> +_require_scratch_xfs_inobtcount()
>  {
>  	_require_scratch
>  
>  	_scratch_mkfs -m inobtcount=1 &> /dev/null || \
> -		_notrun "mkfs.xfs doesn't have inobtcount feature"
> +		_notrun "mkfs.xfs doesn't support inobtcount feature"
>  	_try_scratch_mount || \
> -		_notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
> +		_notrun "kernel doesn't support xfs inobtcount feature"
>  	_scratch_unmount
>  }
> diff --git a/tests/xfs/910 b/tests/xfs/910
> new file mode 100755
> index 00000000..237d0a35
> --- /dev/null
> +++ b/tests/xfs/910
> @@ -0,0 +1,98 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> +#
> +# FS QA Test No. 910
> +#
> +# Check that we can upgrade a filesystem to support inobtcount and that
> +# everything works properly after the upgrade.
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1    # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +
> +# real QA test starts here
> +_supported_fs xfs
> +_require_scratch_xfs_inobtcount
> +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> +_require_xfs_repair_upgrade inobtcount
> +
> +rm -f $seqres.full
> +
> +# Make sure we can't format a filesystem with inobtcount and not finobt.
> +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> +	echo "Should not be able to format with inobtcount but not finobt."
> +
> +# Make sure we can't upgrade a V4 filesystem
> +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Format V5 filesystem without inode btree counter support and populate it.
> +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> +_scratch_mount
> +
> +mkdir $SCRATCH_MNT/stress
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +echo moo > $SCRATCH_MNT/urk
> +
> +_scratch_unmount
> +
> +# Upgrade filesystem to have the counters and inject failure into repair and
> +# make sure that the only path forward is to re-run repair on the filesystem.
> +echo "Fail partway through upgrading"
> +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> +test $? -eq 137 || echo "repair should have been killed??"
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +_try_scratch_mount &> $tmp.mount
> +res=$?
> +_filter_scratch < $tmp.mount
> +if [ $res -eq 0 ]; then
> +	echo "needsrepair should have prevented mount"
> +	_scratch_unmount
> +fi
> +
> +echo "Re-run repair to finish upgrade"
> +_scratch_xfs_repair 2>> $seqres.full
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +
> +echo "Filesystem should be usable again"
> +_scratch_mount
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +_scratch_unmount
> +_check_scratch_fs
> +_check_scratch_xfs_features INOBTCNT
> +
> +echo "Make sure we have nonzero counters"
> +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> +	sed -e 's/= [1-9]*/= NONZERO/g'
> +
> +echo "Make sure we can't re-add inobtcount"
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +
> +echo "Mount again, look at our files"
> +_scratch_mount >> $seqres.full
> +cat $SCRATCH_MNT/urk
> +
> +status=0
> +exit
> diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> new file mode 100644
> index 00000000..1bf040d5
> --- /dev/null
> +++ b/tests/xfs/910.out
> @@ -0,0 +1,23 @@
> +QA output created by 910
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature only supported on V5 filesystems.
> +FEATURES: INOBTCNT:NO
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature requires free inode btree.
> +FEATURES: INOBTCNT:NO
> +Fail partway through upgrading
> +Adding inode btree counts to filesystem.
> +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> +Re-run repair to finish upgrade
> +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> +Filesystem should be usable again
> +FEATURES: INOBTCNT:YES
> +Make sure we have nonzero counters
> +ino_blocks = NONZERO
> +fino_blocks = NONZERO
> +Make sure we can't re-add inobtcount
> +Running xfs_repair to upgrade filesystem.
> +Filesystem already has inode btree counts.
> +Mount again, look at our files
> +moo
> diff --git a/tests/xfs/group b/tests/xfs/group
> index a2309465..bd47333c 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -526,3 +526,4 @@
>  768 auto quick repair
>  770 auto repair
>  773 auto quick repair
> +910 auto quick inobtcount
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Amir Goldstein]

On Wed, Apr 21, 2021 at 3:23 AM Darrick J. Wong <djwong@kernel.org> wrote:
>
> From: Darrick J. Wong <djwong@kernel.org>
>
> Make sure we can actually upgrade filesystems to support inode btree
> counters.
>
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> ---
>  common/xfs        |    8 +++-
>  tests/xfs/910     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/xfs/910.out |   23 ++++++++++++
>  tests/xfs/group   |    1 +
>  4 files changed, 127 insertions(+), 3 deletions(-)
>  create mode 100755 tests/xfs/910
>  create mode 100644 tests/xfs/910.out
>
>
> diff --git a/common/xfs b/common/xfs
> index 5abc7034..3d660858 100644
> --- a/common/xfs
> +++ b/common/xfs
> @@ -1153,13 +1153,15 @@ _require_xfs_repair_upgrade()
>                 _notrun "xfs_repair does not support upgrading fs with $type"
>  }
>
> -_require_xfs_scratch_inobtcount()
> +# Require that the scratch device exists, that mkfs can format with inobtcount
> +# enabled, and that the kernel can mount such a filesystem.
> +_require_scratch_xfs_inobtcount()
>  {
>         _require_scratch
>
>         _scratch_mkfs -m inobtcount=1 &> /dev/null || \
> -               _notrun "mkfs.xfs doesn't have inobtcount feature"
> +               _notrun "mkfs.xfs doesn't support inobtcount feature"
>         _try_scratch_mount || \
> -               _notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
> +               _notrun "kernel doesn't support xfs inobtcount feature"
>         _scratch_unmount
>  }
> diff --git a/tests/xfs/910 b/tests/xfs/910
> new file mode 100755
> index 00000000..237d0a35
> --- /dev/null
> +++ b/tests/xfs/910
> @@ -0,0 +1,98 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> +#
> +# FS QA Test No. 910
> +#
> +# Check that we can upgrade a filesystem to support inobtcount and that
> +# everything works properly after the upgrade.
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1    # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +       cd /
> +       rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +
> +# real QA test starts here
> +_supported_fs xfs
> +_require_scratch_xfs_inobtcount
> +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> +_require_xfs_repair_upgrade inobtcount
> +
> +rm -f $seqres.full
> +
> +# Make sure we can't format a filesystem with inobtcount and not finobt.
> +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> +       echo "Should not be able to format with inobtcount but not finobt."
> +
> +# Make sure we can't upgrade a V4 filesystem
> +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Format V5 filesystem without inode btree counter support and populate it.
> +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> +_scratch_mount
> +
> +mkdir $SCRATCH_MNT/stress
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +echo moo > $SCRATCH_MNT/urk
> +
> +_scratch_unmount
> +
> +# Upgrade filesystem to have the counters and inject failure into repair and
> +# make sure that the only path forward is to re-run repair on the filesystem.
> +echo "Fail partway through upgrading"
> +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> +test $? -eq 137 || echo "repair should have been killed??"
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +_try_scratch_mount &> $tmp.mount
> +res=$?
> +_filter_scratch < $tmp.mount
> +if [ $res -eq 0 ]; then
> +       echo "needsrepair should have prevented mount"
> +       _scratch_unmount
> +fi
> +
> +echo "Re-run repair to finish upgrade"
> +_scratch_xfs_repair 2>> $seqres.full
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +
> +echo "Filesystem should be usable again"
> +_scratch_mount
> +$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
> +_scratch_unmount
> +_check_scratch_fs
> +_check_scratch_xfs_features INOBTCNT
> +
> +echo "Make sure we have nonzero counters"
> +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> +       sed -e 's/= [1-9]*/= NONZERO/g'
> +

I don't think = 100 translates to = NONZERO...

> +echo "Make sure we can't re-add inobtcount"
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +
> +echo "Mount again, look at our files"
> +_scratch_mount >> $seqres.full
> +cat $SCRATCH_MNT/urk
> +
> +status=0
> +exit
> diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> new file mode 100644
> index 00000000..1bf040d5
> --- /dev/null
> +++ b/tests/xfs/910.out
> @@ -0,0 +1,23 @@
> +QA output created by 910
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature only supported on V5 filesystems.
> +FEATURES: INOBTCNT:NO
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature requires free inode btree.
> +FEATURES: INOBTCNT:NO
> +Fail partway through upgrading
> +Adding inode btree counts to filesystem.
> +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> +Re-run repair to finish upgrade
> +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> +Filesystem should be usable again
> +FEATURES: INOBTCNT:YES
> +Make sure we have nonzero counters
> +ino_blocks = NONZERO
> +fino_blocks = NONZERO
> +Make sure we can't re-add inobtcount
> +Running xfs_repair to upgrade filesystem.
> +Filesystem already has inode btree counts.
> +Mount again, look at our files
> +moo

This test is quite rigid in the format of messages expected from
xfs_repair. I suppose there is no precedent here...

> diff --git a/tests/xfs/group b/tests/xfs/group
> index a2309465..bd47333c 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -526,3 +526,4 @@
>  768 auto quick repair
>  770 auto repair
>  773 auto quick repair
> +910 auto quick inobtcount
>
+repair?

Thanks,
Amir.

[PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

From: Darrick J. Wong <djwong@kernel.org>

Make sure we can actually upgrade filesystems to support inode btree
counters.

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
---
 common/xfs        |    8 +++-
 tests/xfs/910     |   98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/xfs/910.out |   23 ++++++++++++
 tests/xfs/group   |    1 +
 4 files changed, 127 insertions(+), 3 deletions(-)
 create mode 100755 tests/xfs/910
 create mode 100644 tests/xfs/910.out


diff --git a/common/xfs b/common/xfs
index 5abc7034..3d660858 100644
--- a/common/xfs
+++ b/common/xfs
@@ -1153,13 +1153,15 @@ _require_xfs_repair_upgrade()
 		_notrun "xfs_repair does not support upgrading fs with $type"
 }
 
-_require_xfs_scratch_inobtcount()
+# Require that the scratch device exists, that mkfs can format with inobtcount
+# enabled, and that the kernel can mount such a filesystem.
+_require_scratch_xfs_inobtcount()
 {
 	_require_scratch
 
 	_scratch_mkfs -m inobtcount=1 &> /dev/null || \
-		_notrun "mkfs.xfs doesn't have inobtcount feature"
+		_notrun "mkfs.xfs doesn't support inobtcount feature"
 	_try_scratch_mount || \
-		_notrun "inobtcount not supported by scratch filesystem type: $FSTYP"
+		_notrun "kernel doesn't support xfs inobtcount feature"
 	_scratch_unmount
 }
diff --git a/tests/xfs/910 b/tests/xfs/910
new file mode 100755
index 00000000..237d0a35
--- /dev/null
+++ b/tests/xfs/910
@@ -0,0 +1,98 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2021 Oracle.  All Rights Reserved.
+#
+# FS QA Test No. 910
+#
+# Check that we can upgrade a filesystem to support inobtcount and that
+# everything works properly after the upgrade.
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1    # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# real QA test starts here
+_supported_fs xfs
+_require_scratch_xfs_inobtcount
+_require_command "$XFS_ADMIN_PROG" "xfs_admin"
+_require_xfs_repair_upgrade inobtcount
+
+rm -f $seqres.full
+
+# Make sure we can't format a filesystem with inobtcount and not finobt.
+_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
+	echo "Should not be able to format with inobtcount but not finobt."
+
+# Make sure we can't upgrade a V4 filesystem
+_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_check_scratch_xfs_features INOBTCNT
+
+# Make sure we can't upgrade a filesystem to inobtcount without finobt.
+_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_check_scratch_xfs_features INOBTCNT
+
+# Format V5 filesystem without inode btree counter support and populate it.
+_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
+_scratch_mount
+
+mkdir $SCRATCH_MNT/stress
+$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
+echo moo > $SCRATCH_MNT/urk
+
+_scratch_unmount
+
+# Upgrade filesystem to have the counters and inject failure into repair and
+# make sure that the only path forward is to re-run repair on the filesystem.
+echo "Fail partway through upgrading"
+XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
+test $? -eq 137 || echo "repair should have been killed??"
+_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
+_try_scratch_mount &> $tmp.mount
+res=$?
+_filter_scratch < $tmp.mount
+if [ $res -eq 0 ]; then
+	echo "needsrepair should have prevented mount"
+	_scratch_unmount
+fi
+
+echo "Re-run repair to finish upgrade"
+_scratch_xfs_repair 2>> $seqres.full
+_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
+
+echo "Filesystem should be usable again"
+_scratch_mount
+$FSSTRESS_PROG -d $SCRATCH_MNT/stress -n 1000 >> $seqres.full
+_scratch_unmount
+_check_scratch_fs
+_check_scratch_xfs_features INOBTCNT
+
+echo "Make sure we have nonzero counters"
+_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
+	sed -e 's/= [1-9]*/= NONZERO/g'
+
+echo "Make sure we can't re-add inobtcount"
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+
+echo "Mount again, look at our files"
+_scratch_mount >> $seqres.full
+cat $SCRATCH_MNT/urk
+
+status=0
+exit
diff --git a/tests/xfs/910.out b/tests/xfs/910.out
new file mode 100644
index 00000000..1bf040d5
--- /dev/null
+++ b/tests/xfs/910.out
@@ -0,0 +1,23 @@
+QA output created by 910
+Running xfs_repair to upgrade filesystem.
+Inode btree count feature only supported on V5 filesystems.
+FEATURES: INOBTCNT:NO
+Running xfs_repair to upgrade filesystem.
+Inode btree count feature requires free inode btree.
+FEATURES: INOBTCNT:NO
+Fail partway through upgrading
+Adding inode btree counts to filesystem.
+FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
+mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
+Re-run repair to finish upgrade
+FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
+Filesystem should be usable again
+FEATURES: INOBTCNT:YES
+Make sure we have nonzero counters
+ino_blocks = NONZERO
+fino_blocks = NONZERO
+Make sure we can't re-add inobtcount
+Running xfs_repair to upgrade filesystem.
+Filesystem already has inode btree counts.
+Mount again, look at our files
+moo
diff --git a/tests/xfs/group b/tests/xfs/group
index a2309465..bd47333c 100644
--- a/tests/xfs/group
+++ b/tests/xfs/group
@@ -526,3 +526,4 @@
 768 auto quick repair
 770 auto repair
 773 auto quick repair
+910 auto quick inobtcount

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

On Wed, Mar 31, 2021 at 01:36:47PM -0400, Brian Foster wrote:
> On Tue, Mar 30, 2021 at 06:08:34PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <djwong@kernel.org>
> > 
> > Make sure we can actually upgrade filesystems to support inode btree
> > counters.
> > 
> > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > ---
> >  tests/xfs/910     |  112 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/xfs/910.out |   23 +++++++++++
> >  tests/xfs/group   |    1 
> >  3 files changed, 136 insertions(+)
> >  create mode 100755 tests/xfs/910
> >  create mode 100644 tests/xfs/910.out
> > 
> > 
> > diff --git a/tests/xfs/910 b/tests/xfs/910
> > new file mode 100755
> > index 00000000..5f095324
> > --- /dev/null
> > +++ b/tests/xfs/910
> > @@ -0,0 +1,112 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0-or-later
> > +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 910
> > +#
> > +# Check that we can upgrade a filesystem to support inobtcount and that
> > +# everything works properly after the upgrade.
> > +
> > +seq=`basename $0`
> > +seqres=$RESULT_DIR/$seq
> > +echo "QA output created by $seq"
> > +
> > +here=`pwd`
> > +tmp=/tmp/$$
> > +status=1    # failure is the default!
> > +trap "_cleanup; exit \$status" 0 1 2 3 15
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	rm -f $tmp.*
> > +}
> > +
> > +# get standard environment, filters and checks
> > +. ./common/rc
> > +. ./common/filter
> > +
> > +# real QA test starts here
> > +_supported_fs xfs
> > +_require_xfs_scratch_inobtcount
> > +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> > +_require_xfs_repair_upgrade inobtcount
> > +
> > +rm -f $seqres.full
> > +
> > +# Make sure we can't format a filesystem with inobtcount and not finobt.
> > +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> > +	echo "Should not be able to format with inobtcount but not finobt."
> > +
> > +# Make sure we can't upgrade a V4 filesystem
> > +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_check_scratch_xfs_features INOBTCNT
> > +
> > +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> > +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_check_scratch_xfs_features INOBTCNT
> > +
> > +# Format V5 filesystem without inode btree counter support and upgrade it.
> > +# Inject failure into repair and make sure that the only path forward is
> > +# to re-run repair on the filesystem.
> > +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> > +echo "Fail partway through upgrading"
> > +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> > +test $? -eq 137 || echo "repair should have been killed??"
> > +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> > +_try_scratch_mount &> $tmp.mount
> > +res=$?
> > +_filter_scratch < $tmp.mount
> > +if [ $res -eq 0 ]; then
> > +	echo "needsrepair should have prevented mount"
> > +	_scratch_unmount
> > +fi
> > +
> > +echo "Re-run repair to finish upgrade"
> > +_scratch_xfs_repair 2>> $seqres.full
> > +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> > +
> > +echo "Filesystem should be usable again"
> > +_try_scratch_mount &> $tmp.mount
> > +res=$?
> > +_filter_scratch < $tmp.mount
> > +if [ $res -eq 0 ]; then
> > +	_scratch_unmount
> > +else
> > +	echo "mount should succeed after second repair"
> > +fi
> > +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> > +
> 
> As you're probably aware I'm not a huge fan of sprinkling these kind of
> repair/mount checks throughout the feature level tests. I don't think
> it's reasonable to expect this to become a consistent pattern, so over
> time this will likely just obfuscate the purpose of some of these tests.
> That said, it seems to be harmless here so I'll just note that as my
> .02.

Ok.  I think inobtcount is the only feature upgrade test that checks it.

> > +# Format V5 filesystem without inode btree counter support and upgrade it.
> > +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> > +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' >> $seqres.full
> > +_scratch_mount >> $seqres.full
> > +
> > +echo moo > $SCRATCH_MNT/urk
> > +
> > +_scratch_unmount
> > +_check_scratch_fs
> > +
> > +# Now upgrade to inobtcount support
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_check_scratch_xfs_features INOBTCNT
> > +_check_scratch_fs
> > +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
> > +
> 
> This looks nearly the same as the previous test with the exception of
> not doing repair failure injection and instead creating a file and
> checking the resulting counters. Could we combine these two sequences
> and simultaneously run a slightly more randomized test? E.g., a logical
> sequence along the lines of:
> 
> mkfs inobtcount=0
> mount
> run fsstress -n 500 or otherwise (quickly) populate w/ some randomized content
> umount
> repair w/ fail injection
> repair w/o fail injection
> check resulting counters
> 
> Hm?

Yes, the two cases can be combined.  Will do, thanks for the feedback!

--D

> 
> Brian
> 
> > +# Make sure we have nonzero counters
> > +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> > +	sed -e 's/= [1-9]*/= NONZERO/g'
> > +
> > +# Mount again, look at our files
> > +_scratch_mount >> $seqres.full
> > +cat $SCRATCH_MNT/urk
> > +
> > +# Make sure we can't re-add inobtcount
> > +_scratch_unmount
> > +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> > +_scratch_mount >> $seqres.full
> > +
> > +status=0
> > +exit
> > diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> > new file mode 100644
> > index 00000000..ed78d88f
> > --- /dev/null
> > +++ b/tests/xfs/910.out
> > @@ -0,0 +1,23 @@
> > +QA output created by 910
> > +Running xfs_repair to upgrade filesystem.
> > +Inode btree count feature only supported on V5 filesystems.
> > +FEATURES: INOBTCNT:NO
> > +Running xfs_repair to upgrade filesystem.
> > +Inode btree count feature requires free inode btree.
> > +FEATURES: INOBTCNT:NO
> > +Fail partway through upgrading
> > +Adding inode btree counts to filesystem.
> > +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> > +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> > +Re-run repair to finish upgrade
> > +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> > +Filesystem should be usable again
> > +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> > +Running xfs_repair to upgrade filesystem.
> > +Adding inode btree counts to filesystem.
> > +FEATURES: INOBTCNT:YES
> > +ino_blocks = NONZERO
> > +fino_blocks = NONZERO
> > +moo
> > +Running xfs_repair to upgrade filesystem.
> > +Filesystem already has inode btree counts.
> > diff --git a/tests/xfs/group b/tests/xfs/group
> > index 5801471b..0dc8038a 100644
> > --- a/tests/xfs/group
> > +++ b/tests/xfs/group
> > @@ -524,3 +524,4 @@
> >  768 auto quick repair
> >  770 auto repair
> >  773 auto quick repair
> > +910 auto quick inobtcount
> > 
> 

Re: [PATCH 2/2] xfs: test inobtcount upgrade

[Brian Foster]

On Tue, Mar 30, 2021 at 06:08:34PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> Make sure we can actually upgrade filesystems to support inode btree
> counters.
> 
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> ---
>  tests/xfs/910     |  112 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/xfs/910.out |   23 +++++++++++
>  tests/xfs/group   |    1 
>  3 files changed, 136 insertions(+)
>  create mode 100755 tests/xfs/910
>  create mode 100644 tests/xfs/910.out
> 
> 
> diff --git a/tests/xfs/910 b/tests/xfs/910
> new file mode 100755
> index 00000000..5f095324
> --- /dev/null
> +++ b/tests/xfs/910
> @@ -0,0 +1,112 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +# Copyright (c) 2021 Oracle.  All Rights Reserved.
> +#
> +# FS QA Test No. 910
> +#
> +# Check that we can upgrade a filesystem to support inobtcount and that
> +# everything works properly after the upgrade.
> +
> +seq=`basename $0`
> +seqres=$RESULT_DIR/$seq
> +echo "QA output created by $seq"
> +
> +here=`pwd`
> +tmp=/tmp/$$
> +status=1    # failure is the default!
> +trap "_cleanup; exit \$status" 0 1 2 3 15
> +
> +_cleanup()
> +{
> +	cd /
> +	rm -f $tmp.*
> +}
> +
> +# get standard environment, filters and checks
> +. ./common/rc
> +. ./common/filter
> +
> +# real QA test starts here
> +_supported_fs xfs
> +_require_xfs_scratch_inobtcount
> +_require_command "$XFS_ADMIN_PROG" "xfs_admin"
> +_require_xfs_repair_upgrade inobtcount
> +
> +rm -f $seqres.full
> +
> +# Make sure we can't format a filesystem with inobtcount and not finobt.
> +_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
> +	echo "Should not be able to format with inobtcount but not finobt."
> +
> +# Make sure we can't upgrade a V4 filesystem
> +_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Make sure we can't upgrade a filesystem to inobtcount without finobt.
> +_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +
> +# Format V5 filesystem without inode btree counter support and upgrade it.
> +# Inject failure into repair and make sure that the only path forward is
> +# to re-run repair on the filesystem.
> +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> +echo "Fail partway through upgrading"
> +XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
> +test $? -eq 137 || echo "repair should have been killed??"
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +_try_scratch_mount &> $tmp.mount
> +res=$?
> +_filter_scratch < $tmp.mount
> +if [ $res -eq 0 ]; then
> +	echo "needsrepair should have prevented mount"
> +	_scratch_unmount
> +fi
> +
> +echo "Re-run repair to finish upgrade"
> +_scratch_xfs_repair 2>> $seqres.full
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +
> +echo "Filesystem should be usable again"
> +_try_scratch_mount &> $tmp.mount
> +res=$?
> +_filter_scratch < $tmp.mount
> +if [ $res -eq 0 ]; then
> +	_scratch_unmount
> +else
> +	echo "mount should succeed after second repair"
> +fi
> +_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
> +

As you're probably aware I'm not a huge fan of sprinkling these kind of
repair/mount checks throughout the feature level tests. I don't think
it's reasonable to expect this to become a consistent pattern, so over
time this will likely just obfuscate the purpose of some of these tests.
That said, it seems to be harmless here so I'll just note that as my
.02.

> +# Format V5 filesystem without inode btree counter support and upgrade it.
> +_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
> +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' >> $seqres.full
> +_scratch_mount >> $seqres.full
> +
> +echo moo > $SCRATCH_MNT/urk
> +
> +_scratch_unmount
> +_check_scratch_fs
> +
> +# Now upgrade to inobtcount support
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_check_scratch_xfs_features INOBTCNT
> +_check_scratch_fs
> +_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
> +

This looks nearly the same as the previous test with the exception of
not doing repair failure injection and instead creating a file and
checking the resulting counters. Could we combine these two sequences
and simultaneously run a slightly more randomized test? E.g., a logical
sequence along the lines of:

mkfs inobtcount=0
mount
run fsstress -n 500 or otherwise (quickly) populate w/ some randomized content
umount
repair w/ fail injection
repair w/o fail injection
check resulting counters

Hm?

Brian

> +# Make sure we have nonzero counters
> +_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
> +	sed -e 's/= [1-9]*/= NONZERO/g'
> +
> +# Mount again, look at our files
> +_scratch_mount >> $seqres.full
> +cat $SCRATCH_MNT/urk
> +
> +# Make sure we can't re-add inobtcount
> +_scratch_unmount
> +_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
> +_scratch_mount >> $seqres.full
> +
> +status=0
> +exit
> diff --git a/tests/xfs/910.out b/tests/xfs/910.out
> new file mode 100644
> index 00000000..ed78d88f
> --- /dev/null
> +++ b/tests/xfs/910.out
> @@ -0,0 +1,23 @@
> +QA output created by 910
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature only supported on V5 filesystems.
> +FEATURES: INOBTCNT:NO
> +Running xfs_repair to upgrade filesystem.
> +Inode btree count feature requires free inode btree.
> +FEATURES: INOBTCNT:NO
> +Fail partway through upgrading
> +Adding inode btree counts to filesystem.
> +FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
> +mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
> +Re-run repair to finish upgrade
> +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> +Filesystem should be usable again
> +FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
> +Running xfs_repair to upgrade filesystem.
> +Adding inode btree counts to filesystem.
> +FEATURES: INOBTCNT:YES
> +ino_blocks = NONZERO
> +fino_blocks = NONZERO
> +moo
> +Running xfs_repair to upgrade filesystem.
> +Filesystem already has inode btree counts.
> diff --git a/tests/xfs/group b/tests/xfs/group
> index 5801471b..0dc8038a 100644
> --- a/tests/xfs/group
> +++ b/tests/xfs/group
> @@ -524,3 +524,4 @@
>  768 auto quick repair
>  770 auto repair
>  773 auto quick repair
> +910 auto quick inobtcount
> 

[PATCH 2/2] xfs: test inobtcount upgrade

[Darrick J. Wong]

From: Darrick J. Wong <djwong@kernel.org>

Make sure we can actually upgrade filesystems to support inode btree
counters.

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
---
 tests/xfs/910     |  112 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/xfs/910.out |   23 +++++++++++
 tests/xfs/group   |    1 
 3 files changed, 136 insertions(+)
 create mode 100755 tests/xfs/910
 create mode 100644 tests/xfs/910.out


diff --git a/tests/xfs/910 b/tests/xfs/910
new file mode 100755
index 00000000..5f095324
--- /dev/null
+++ b/tests/xfs/910
@@ -0,0 +1,112 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2021 Oracle.  All Rights Reserved.
+#
+# FS QA Test No. 910
+#
+# Check that we can upgrade a filesystem to support inobtcount and that
+# everything works properly after the upgrade.
+
+seq=`basename $0`
+seqres=$RESULT_DIR/$seq
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1    # failure is the default!
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+_cleanup()
+{
+	cd /
+	rm -f $tmp.*
+}
+
+# get standard environment, filters and checks
+. ./common/rc
+. ./common/filter
+
+# real QA test starts here
+_supported_fs xfs
+_require_xfs_scratch_inobtcount
+_require_command "$XFS_ADMIN_PROG" "xfs_admin"
+_require_xfs_repair_upgrade inobtcount
+
+rm -f $seqres.full
+
+# Make sure we can't format a filesystem with inobtcount and not finobt.
+_scratch_mkfs -m crc=1,inobtcount=1,finobt=0 &> $seqres.full && \
+	echo "Should not be able to format with inobtcount but not finobt."
+
+# Make sure we can't upgrade a V4 filesystem
+_scratch_mkfs -m crc=0,inobtcount=0,finobt=0 >> $seqres.full
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_check_scratch_xfs_features INOBTCNT
+
+# Make sure we can't upgrade a filesystem to inobtcount without finobt.
+_scratch_mkfs -m crc=1,inobtcount=0,finobt=0 >> $seqres.full
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_check_scratch_xfs_features INOBTCNT
+
+# Format V5 filesystem without inode btree counter support and upgrade it.
+# Inject failure into repair and make sure that the only path forward is
+# to re-run repair on the filesystem.
+_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
+echo "Fail partway through upgrading"
+XFS_REPAIR_FAIL_AFTER_PHASE=2 _scratch_xfs_repair -c inobtcount=1 2>> $seqres.full
+test $? -eq 137 || echo "repair should have been killed??"
+_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
+_try_scratch_mount &> $tmp.mount
+res=$?
+_filter_scratch < $tmp.mount
+if [ $res -eq 0 ]; then
+	echo "needsrepair should have prevented mount"
+	_scratch_unmount
+fi
+
+echo "Re-run repair to finish upgrade"
+_scratch_xfs_repair 2>> $seqres.full
+_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
+
+echo "Filesystem should be usable again"
+_try_scratch_mount &> $tmp.mount
+res=$?
+_filter_scratch < $tmp.mount
+if [ $res -eq 0 ]; then
+	_scratch_unmount
+else
+	echo "mount should succeed after second repair"
+fi
+_check_scratch_xfs_features NEEDSREPAIR INOBTCNT
+
+# Format V5 filesystem without inode btree counter support and upgrade it.
+_scratch_mkfs -m crc=1,inobtcount=0 >> $seqres.full
+_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' >> $seqres.full
+_scratch_mount >> $seqres.full
+
+echo moo > $SCRATCH_MNT/urk
+
+_scratch_unmount
+_check_scratch_fs
+
+# Now upgrade to inobtcount support
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_check_scratch_xfs_features INOBTCNT
+_check_scratch_fs
+_scratch_xfs_db -c 'version' -c 'sb 0' -c 'p' -c 'agi 0' -c 'p' >> $seqres.full
+
+# Make sure we have nonzero counters
+_scratch_xfs_db -c 'agi 0' -c 'print ino_blocks fino_blocks' | \
+	sed -e 's/= [1-9]*/= NONZERO/g'
+
+# Mount again, look at our files
+_scratch_mount >> $seqres.full
+cat $SCRATCH_MNT/urk
+
+# Make sure we can't re-add inobtcount
+_scratch_unmount
+_scratch_xfs_admin -O inobtcount=1 2>> $seqres.full
+_scratch_mount >> $seqres.full
+
+status=0
+exit
diff --git a/tests/xfs/910.out b/tests/xfs/910.out
new file mode 100644
index 00000000..ed78d88f
--- /dev/null
+++ b/tests/xfs/910.out
@@ -0,0 +1,23 @@
+QA output created by 910
+Running xfs_repair to upgrade filesystem.
+Inode btree count feature only supported on V5 filesystems.
+FEATURES: INOBTCNT:NO
+Running xfs_repair to upgrade filesystem.
+Inode btree count feature requires free inode btree.
+FEATURES: INOBTCNT:NO
+Fail partway through upgrading
+Adding inode btree counts to filesystem.
+FEATURES: NEEDSREPAIR:YES INOBTCNT:YES
+mount: SCRATCH_MNT: mount(2) system call failed: Structure needs cleaning.
+Re-run repair to finish upgrade
+FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
+Filesystem should be usable again
+FEATURES: NEEDSREPAIR:NO INOBTCNT:YES
+Running xfs_repair to upgrade filesystem.
+Adding inode btree counts to filesystem.
+FEATURES: INOBTCNT:YES
+ino_blocks = NONZERO
+fino_blocks = NONZERO
+moo
+Running xfs_repair to upgrade filesystem.
+Filesystem already has inode btree counts.
diff --git a/tests/xfs/group b/tests/xfs/group
index 5801471b..0dc8038a 100644
--- a/tests/xfs/group
+++ b/tests/xfs/group
@@ -524,3 +524,4 @@
 768 auto quick repair
 770 auto repair
 773 auto quick repair
+910 auto quick inobtcount