[PATCH 1/2] testsuite: compress modules if feature is enabled

[PATCH 1/2] testsuite: compress modules if feature is enabled

[Lucas De Marchi]

Since the output needs to be the same, regardless if the module is
compressed, change populate-modules.sh to conditionally compress the
module if that feature is enabled.

This way we can execute the tests with any build-time configuration and
it should still pass.

Suggested-by: Michal Suchánek <msuchanek@suse.de>
---
 Makefile.am                   |  2 +-
 testsuite/populate-modules.sh | 27 ++++++++++++++++++---------
 testsuite/test-depmod.c       |  2 --
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index b29e943..24a586e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -249,7 +249,7 @@ CREATE_ROOTFS = $(AM_V_GEN) ( $(RM) -rf $(ROOTFS) && mkdir -p $(dir $(ROOTFS)) &
 				find $(ROOTFS) -type d -exec chmod +w {} \; && \
 				find $(ROOTFS) -type f -name .gitignore -exec rm -f {} \; && \
 				$(top_srcdir)/testsuite/populate-modules.sh \
-					$(MODULE_PLAYGROUND) $(ROOTFS) ) && \
+					$(MODULE_PLAYGROUND) $(ROOTFS) $(top_builddir)/config.h ) && \
 				touch testsuite/stamp-rootfs
 
 build-module-playground:
diff --git a/testsuite/populate-modules.sh b/testsuite/populate-modules.sh
index b0cc932..ae43884 100755
--- a/testsuite/populate-modules.sh
+++ b/testsuite/populate-modules.sh
@@ -4,6 +4,12 @@ set -e
 
 MODULE_PLAYGROUND=$1
 ROOTFS=$2
+CONFIG_H=$3
+
+feature_enabled() {
+	local feature=$1
+	grep KMOD_FEATURES  $CONFIG_H | head -n 1 | grep -q \+$feature
+}
 
 declare -A map
 map=(
@@ -99,15 +105,18 @@ done
 
 # start poking the final rootfs...
 
-# gzip these modules
-for m in "${gzip_array[@]}"; do
-    gzip "$ROOTFS/$m"
-done
-
-# zstd-compress these modules
-for m in "${zstd_array[@]}"; do
-    zstd --rm $ROOTFS/$m
-done
+# compress modules with each format if feature is enabled
+if feature_enabled ZLIB; then
+	for m in "${gzip_array[@]}"; do
+	    gzip "$ROOTFS/$m"
+	done
+fi
+
+if feature_enabled ZSTD; then
+	for m in "${zstd_array[@]}"; do
+	    zstd --rm $ROOTFS/$m
+	done
+fi
 
 for m in "${attach_sha1_array[@]}"; do
     cat "${MODULE_PLAYGROUND}/dummy.sha1" >>"${ROOTFS}/$m"
diff --git a/testsuite/test-depmod.c b/testsuite/test-depmod.c
index 261559c..d7802d7 100644
--- a/testsuite/test-depmod.c
+++ b/testsuite/test-depmod.c
@@ -25,7 +25,6 @@
 
 #include "testsuite.h"
 
-#ifdef ENABLE_ZLIB
 #define MODULES_ORDER_UNAME "4.4.4"
 #define MODULES_ORDER_ROOTFS TESTSUITE_ROOTFS "test-depmod/modules-order-compressed"
 #define MODULES_ORDER_LIB_MODULES MODULES_ORDER_ROOTFS "/lib/modules/" MODULES_ORDER_UNAME
@@ -57,7 +56,6 @@ DEFINE_TEST(depmod_modules_order_for_compressed,
 			{ }
 		},
 	});
-#endif
 
 #define SEARCH_ORDER_SIMPLE_ROOTFS TESTSUITE_ROOTFS "test-depmod/search-order-simple"
 static noreturn int depmod_search_order_simple(const struct test *t)
-- 
2.30.0

[PATCH 2/2] testsuite: also test xz compression

[Lucas De Marchi]

---
 testsuite/populate-modules.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/testsuite/populate-modules.sh b/testsuite/populate-modules.sh
index ae43884..099f026 100755
--- a/testsuite/populate-modules.sh
+++ b/testsuite/populate-modules.sh
@@ -72,6 +72,9 @@ map=(
 
 gzip_array=(
     "test-depmod/modules-order-compressed/lib/modules/4.4.4/kernel/drivers/block/cciss.ko"
+    )
+
+xz_array=(
     "test-depmod/modules-order-compressed/lib/modules/4.4.4/kernel/drivers/scsi/scsi_mod.ko"
     )
 
@@ -112,6 +115,12 @@ if feature_enabled ZLIB; then
 	done
 fi
 
+if feature_enabled XZ; then
+	for m in "${xz_array[@]}"; do
+	    xz "$ROOTFS/$m"
+	done
+fi
+
 if feature_enabled ZSTD; then
 	for m in "${zstd_array[@]}"; do
 	    zstd --rm $ROOTFS/$m
-- 
2.30.0

Re: [PATCH 1/2] testsuite: compress modules if feature is enabled

[Michal Suchánek]

Hello,

thanks for the patch.

I tested it and it allows me to run tests with zstd disabled.

It also makes the code much clearer.

Reviewed-by: Michal Suchánek <msuchanek@suse.de>
Tested-by: Michal Suchánek <msuchanek@suse.de>

On Fri, Jan 29, 2021 at 06:35:59PM -0800, Lucas De Marchi wrote:
> Since the output needs to be the same, regardless if the module is
> compressed, change populate-modules.sh to conditionally compress the
> module if that feature is enabled.
> 
> This way we can execute the tests with any build-time configuration and
> it should still pass.
> 
> Suggested-by: Michal Suchánek <msuchanek@suse.de>
> ---
>  Makefile.am                   |  2 +-
>  testsuite/populate-modules.sh | 27 ++++++++++++++++++---------
>  testsuite/test-depmod.c       |  2 --
>  3 files changed, 19 insertions(+), 12 deletions(-)
> 
> diff --git a/Makefile.am b/Makefile.am
> index b29e943..24a586e 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -249,7 +249,7 @@ CREATE_ROOTFS = $(AM_V_GEN) ( $(RM) -rf $(ROOTFS) && mkdir -p $(dir $(ROOTFS)) &
>  				find $(ROOTFS) -type d -exec chmod +w {} \; && \
>  				find $(ROOTFS) -type f -name .gitignore -exec rm -f {} \; && \
>  				$(top_srcdir)/testsuite/populate-modules.sh \
> -					$(MODULE_PLAYGROUND) $(ROOTFS) ) && \
> +					$(MODULE_PLAYGROUND) $(ROOTFS) $(top_builddir)/config.h ) && \
>  				touch testsuite/stamp-rootfs
>  
>  build-module-playground:
> diff --git a/testsuite/populate-modules.sh b/testsuite/populate-modules.sh
> index b0cc932..ae43884 100755
> --- a/testsuite/populate-modules.sh
> +++ b/testsuite/populate-modules.sh
> @@ -4,6 +4,12 @@ set -e
>  
>  MODULE_PLAYGROUND=$1
>  ROOTFS=$2
> +CONFIG_H=$3
> +
> +feature_enabled() {
> +	local feature=$1
> +	grep KMOD_FEATURES  $CONFIG_H | head -n 1 | grep -q \+$feature
> +}
>  
>  declare -A map
>  map=(
> @@ -99,15 +105,18 @@ done
>  
>  # start poking the final rootfs...
>  
> -# gzip these modules
> -for m in "${gzip_array[@]}"; do
> -    gzip "$ROOTFS/$m"
> -done
> -
> -# zstd-compress these modules
> -for m in "${zstd_array[@]}"; do
> -    zstd --rm $ROOTFS/$m
> -done
> +# compress modules with each format if feature is enabled
> +if feature_enabled ZLIB; then
> +	for m in "${gzip_array[@]}"; do
> +	    gzip "$ROOTFS/$m"
> +	done
> +fi
> +
> +if feature_enabled ZSTD; then
> +	for m in "${zstd_array[@]}"; do
> +	    zstd --rm $ROOTFS/$m
> +	done
> +fi
>  
>  for m in "${attach_sha1_array[@]}"; do
>      cat "${MODULE_PLAYGROUND}/dummy.sha1" >>"${ROOTFS}/$m"
> diff --git a/testsuite/test-depmod.c b/testsuite/test-depmod.c
> index 261559c..d7802d7 100644
> --- a/testsuite/test-depmod.c
> +++ b/testsuite/test-depmod.c
> @@ -25,7 +25,6 @@
>  
>  #include "testsuite.h"
>  
> -#ifdef ENABLE_ZLIB
>  #define MODULES_ORDER_UNAME "4.4.4"
>  #define MODULES_ORDER_ROOTFS TESTSUITE_ROOTFS "test-depmod/modules-order-compressed"
>  #define MODULES_ORDER_LIB_MODULES MODULES_ORDER_ROOTFS "/lib/modules/" MODULES_ORDER_UNAME
> @@ -57,7 +56,6 @@ DEFINE_TEST(depmod_modules_order_for_compressed,
>  			{ }
>  		},
>  	});
> -#endif
>  
>  #define SEARCH_ORDER_SIMPLE_ROOTFS TESTSUITE_ROOTFS "test-depmod/search-order-simple"
>  static noreturn int depmod_search_order_simple(const struct test *t)
> -- 
> 2.30.0
> 

Re: [PATCH 1/2] testsuite: compress modules if feature is enabled

[Petr Vorel]

Hi,

Reviewed-by: Petr Vorel <pvorel@suse.cz>
Great, thank you!

> +feature_enabled() {
> +	local feature=$1
> +	grep KMOD_FEATURES  $CONFIG_H | head -n 1 | grep -q \+$feature
nit: using grep single time is IMHO enough
grep -q "define KMOD_FEATURES.*\+$feature" $CONFIG_H
> +}

Kind regards,
Petr

Re: [PATCH 2/2] testsuite: also test xz compression

[Petr Vorel]

Hi,

Reviewed-by: Petr Vorel <pvorel@suse.cz>

Thanks!

BTW I still have issues with testsuite/test-depmod locally, but probably my
wrong setup (testing also other configure options than this one):

$ CFLAGS='-g -O2' --enable-debug && make -j`nproc` && make check

depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
TESTSUITE: running depmod_search_order_override, in forked context
TESTSUITE: SKIPPED: depmod_search_order_override
TESTSUITE: ------
TESTSUITE: running depmod_search_order_external_last, in forked context
TESTSUITE: 'depmod_search_order_external_last' [31183] exited with return code 0
TESTSUITE: PASSED: depmod_search_order_external_last
TESTSUITE: ------
TESTSUITE: running depmod_search_order_external_first, in forked context
TESTSUITE: SKIPPED: depmod_search_order_external_first
TESTSUITE: ------
TESTSUITE: running depmod_detect_loop, in forked context
TESTSUITE: SKIPPED: depmod_detect_loop
TESTSUITE: ------
TESTSUITE: running depmod_search_order_same_prefix, in forked context
TESTSUITE: 'depmod_search_order_same_prefix' [31182] exited with return code 0
TESTSUITE: ERR: sizes do not match /src/kmod/testsuite/rootfs/test-depmod/search-order-same-prefix/lib/modules/4.4.4/correct-modules.dep /src/kmod/testsuite/rootfs/test-depmod/search-order-same-prefix/lib/modules/4.4.4/modules.dep
TESTSUITE: ERR: FAILED: exit ok but outputs do not match: depmod_search_order_same_prefix
TESTSUITE: ------
FAIL testsuite/test-depmod (exit status: 1)

Kind regards,
Petr

Re: [PATCH 2/2] testsuite: also test xz compression

[Lucas De Marchi]

On Thu, Feb 4, 2021 at 2:35 AM Petr Vorel <pvorel@suse.cz> wrote:
>
> Hi,
>
> Reviewed-by: Petr Vorel <pvorel@suse.cz>


pushed, thanks.


>
> Thanks!
>
> BTW I still have issues with testsuite/test-depmod locally, but probably my
> wrong setup (testing also other configure options than this one):
>
> $ CFLAGS='-g -O2' --enable-debug && make -j`nproc` && make check

humn...  more things to fixup

Lucas De Marchi
>
> depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.order at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
> depmod: WARNING: could not open modules.builtin at /lib/modules/4.4.4: No such file or directory
> TESTSUITE: running depmod_search_order_override, in forked context
> TESTSUITE: SKIPPED: depmod_search_order_override
> TESTSUITE: ------
> TESTSUITE: running depmod_search_order_external_last, in forked context
> TESTSUITE: 'depmod_search_order_external_last' [31183] exited with return code 0
> TESTSUITE: PASSED: depmod_search_order_external_last
> TESTSUITE: ------
> TESTSUITE: running depmod_search_order_external_first, in forked context
> TESTSUITE: SKIPPED: depmod_search_order_external_first
> TESTSUITE: ------
> TESTSUITE: running depmod_detect_loop, in forked context
> TESTSUITE: SKIPPED: depmod_detect_loop
> TESTSUITE: ------
> TESTSUITE: running depmod_search_order_same_prefix, in forked context
> TESTSUITE: 'depmod_search_order_same_prefix' [31182] exited with return code 0
> TESTSUITE: ERR: sizes do not match /src/kmod/testsuite/rootfs/test-depmod/search-order-same-prefix/lib/modules/4.4.4/correct-modules.dep /src/kmod/testsuite/rootfs/test-depmod/search-order-same-prefix/lib/modules/4.4.4/modules.dep
> TESTSUITE: ERR: FAILED: exit ok but outputs do not match: depmod_search_order_same_prefix
> TESTSUITE: ------
> FAIL testsuite/test-depmod (exit status: 1)
>
> Kind regards,
> Petr

Re: [PATCH 1/2] testsuite: compress modules if feature is enabled

[Lucas De Marchi]

Pushed, thanks for reviews.

Lucas De Marchi

On Sat, Jan 30, 2021 at 1:40 AM Lucas De Marchi
<lucas.demarchi@intel.com> wrote:
>
> Since the output needs to be the same, regardless if the module is
> compressed, change populate-modules.sh to conditionally compress the
> module if that feature is enabled.
>
> This way we can execute the tests with any build-time configuration and
> it should still pass.
>
> Suggested-by: Michal Suchánek <msuchanek@suse.de>
> ---
>  Makefile.am                   |  2 +-
>  testsuite/populate-modules.sh | 27 ++++++++++++++++++---------
>  testsuite/test-depmod.c       |  2 --
>  3 files changed, 19 insertions(+), 12 deletions(-)
>
> diff --git a/Makefile.am b/Makefile.am
> index b29e943..24a586e 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -249,7 +249,7 @@ CREATE_ROOTFS = $(AM_V_GEN) ( $(RM) -rf $(ROOTFS) && mkdir -p $(dir $(ROOTFS)) &
>                                 find $(ROOTFS) -type d -exec chmod +w {} \; && \
>                                 find $(ROOTFS) -type f -name .gitignore -exec rm -f {} \; && \
>                                 $(top_srcdir)/testsuite/populate-modules.sh \
> -                                       $(MODULE_PLAYGROUND) $(ROOTFS) ) && \
> +                                       $(MODULE_PLAYGROUND) $(ROOTFS) $(top_builddir)/config.h ) && \
>                                 touch testsuite/stamp-rootfs
>
>  build-module-playground:
> diff --git a/testsuite/populate-modules.sh b/testsuite/populate-modules.sh
> index b0cc932..ae43884 100755
> --- a/testsuite/populate-modules.sh
> +++ b/testsuite/populate-modules.sh
> @@ -4,6 +4,12 @@ set -e
>
>  MODULE_PLAYGROUND=$1
>  ROOTFS=$2
> +CONFIG_H=$3
> +
> +feature_enabled() {
> +       local feature=$1
> +       grep KMOD_FEATURES  $CONFIG_H | head -n 1 | grep -q \+$feature
> +}
>
>  declare -A map
>  map=(
> @@ -99,15 +105,18 @@ done
>
>  # start poking the final rootfs...
>
> -# gzip these modules
> -for m in "${gzip_array[@]}"; do
> -    gzip "$ROOTFS/$m"
> -done
> -
> -# zstd-compress these modules
> -for m in "${zstd_array[@]}"; do
> -    zstd --rm $ROOTFS/$m
> -done
> +# compress modules with each format if feature is enabled
> +if feature_enabled ZLIB; then
> +       for m in "${gzip_array[@]}"; do
> +           gzip "$ROOTFS/$m"
> +       done
> +fi
> +
> +if feature_enabled ZSTD; then
> +       for m in "${zstd_array[@]}"; do
> +           zstd --rm $ROOTFS/$m
> +       done
> +fi
>
>  for m in "${attach_sha1_array[@]}"; do
>      cat "${MODULE_PLAYGROUND}/dummy.sha1" >>"${ROOTFS}/$m"
> diff --git a/testsuite/test-depmod.c b/testsuite/test-depmod.c
> index 261559c..d7802d7 100644
> --- a/testsuite/test-depmod.c
> +++ b/testsuite/test-depmod.c
> @@ -25,7 +25,6 @@
>
>  #include "testsuite.h"
>
> -#ifdef ENABLE_ZLIB
>  #define MODULES_ORDER_UNAME "4.4.4"
>  #define MODULES_ORDER_ROOTFS TESTSUITE_ROOTFS "test-depmod/modules-order-compressed"
>  #define MODULES_ORDER_LIB_MODULES MODULES_ORDER_ROOTFS "/lib/modules/" MODULES_ORDER_UNAME
> @@ -57,7 +56,6 @@ DEFINE_TEST(depmod_modules_order_for_compressed,
>                         { }
>                 },
>         });
> -#endif
>
>  #define SEARCH_ORDER_SIMPLE_ROOTFS TESTSUITE_ROOTFS "test-depmod/search-order-simple"
>  static noreturn int depmod_search_order_simple(const struct test *t)
> --
> 2.30.0
>