* [RFC PATCH] kbuild: add variables for compression tools
@ 2020-05-14 13:12 Denis Efremov
2020-05-15 2:20 ` Masahiro Yamada
` (4 more replies)
0 siblings, 5 replies; 20+ messages in thread
From: Denis Efremov @ 2020-05-14 13:12 UTC (permalink / raw)
To: Masahiro Yamada
Cc: Denis Efremov, linux-kbuild, linux-kernel, Stephen Rothwell,
Dmitry Vyukov
Allow user to use alternative implementations of compression tools.
For example, multi-threaded tools to speed up the build:
$ make KGZIP=pigz KXZ=pxz
Variable KGZIP is used instead of GZIP because the latter is reserved
by the tool. Other variables are prefixed with 'K' for consistency.
The credit goes to @grsecurity.
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Makefile | 11 +++++++++--
arch/arm/boot/deflate_xip_data.sh | 6 +++++-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 4 ++--
arch/mips/lasat/image/Makefile | 2 +-
arch/parisc/Makefile | 2 +-
| 6 +++++-
scripts/Makefile.lib | 12 ++++++------
scripts/Makefile.package | 6 +++---
scripts/xz_wrap.sh | 6 +++++-
10 files changed, 38 insertions(+), 19 deletions(-)
diff --git a/Makefile b/Makefile
index 11fe9b1535de..9af13cfeed7a 100644
--- a/Makefile
+++ b/Makefile
@@ -447,6 +447,12 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+KGZIP = gzip
+KBZIP2 = bzip2
+KLZMA = lzma
+KLZOP = lzop
+KLZ4 = lz4c
+KXZ = xz
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -496,6 +502,7 @@ export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
+export KGZIP KBZIP2 KLZMA KLZOP KLZ4 KXZ
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
@@ -1005,10 +1012,10 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = gzip -n -f
+ mod_compress_cmd = $(KGZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
- mod_compress_cmd = xz -f
+ mod_compress_cmd = $(KXZ) -f
endif # CONFIG_MODULE_COMPRESS_XZ
endif # CONFIG_MODULE_COMPRESS
export mod_compress_cmd
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 40937248cebe..08dd50e08c17 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -19,6 +19,10 @@ XIPIMAGE="$2"
DD="dd status=none"
+if [ x$KGZIP = "x" ]; then
+ KGZIP=gzip
+fi
+
# Use "make V=1" to debug this script.
case "$KBUILD_VERBOSE" in
*1*)
@@ -56,7 +60,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-gzip -9 >> "$XIPIMAGE.tmp"
+$KGZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index 32240000dc0c..2876a7df1b0a 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 5d9288384096..e6c7c92aa72e 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- bzip2 -1c vmlinux.tmp >vmlinux.bz2
+ $(KBZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- bzip2 -1c vmlinux >vmlinux.bz2
+ $(KBZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
index 78ce4cff1012..617ccb1659d5 100644
--- a/arch/mips/lasat/image/Makefile
+++ b/arch/mips/lasat/image/Makefile
@@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
$(LD) -r -o $@ -b binary $<
$(obj)/%.gz: $(obj)/%.bin
- gzip -cf -9 $< > $@
+ $(KGZIP) -cf -9 $< > $@
$(obj)/kImage.bin: $(KERNEL_IMAGE)
$(OBJCOPY) -O binary -S $^ $@
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 628cd8bb7ad8..412ddec0297d 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @gzip -cf -9 $< > $@
+ @$(KGZIP) -cf -9 $< > $@
endif
install:
--git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
index e13ca842eb7e..f3dfaf9f6647 100755
--- a/kernel/gen_kheaders.sh
+++ b/kernel/gen_kheaders.sh
@@ -9,6 +9,10 @@ outdir="$(pwd)"
tarfile=$1
cpio_dir=$outdir/$tarfile.tmp
+if [ x$KXZ = "x" ]; then
+ KXZ=xz
+fi
+
dir_list="
include/
arch/$SRCARCH/include/
@@ -88,7 +92,7 @@ find $cpio_dir -type f -print0 |
find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
--owner=0 --group=0 --numeric-owner --no-recursion \
- -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
+ -I $KXZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
echo $headers_md5 > kernel/kheaders.md5
echo "$this_file_md5" >> kernel/kheaders.md5
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 4b799737722c..dd38f5ac8d48 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
quiet_cmd_lzma = LZMA $@
- cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
+ cmd_lzma = { cat $(real-prereqs) | $(KLZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
+ cmd_lz4 = { cat $(real-prereqs) | $(KLZ4) -l -c1 stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
@@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
$(size_append); } > $@
quiet_cmd_xzmisc = XZMISC $@
- cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
+ cmd_xzmisc = cat $(real-prereqs) | $(KXZ) --check=crc32 --lzma2=dict=1MiB > $@
# ASM offsets
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 02135d2671a6..1b91fe1bfcdb 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),bzip2, \
-$(if $(findstring gz,$@),gzip, \
-$(if $(findstring xz,$@),xz, \
+$(if $(findstring bz2,$@),$(KBZIP2), \
+$(if $(findstring gz,$@),$(KGZIP), \
+$(if $(findstring xz,$@),$(KXZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index 7a2d372f4885..4922102dbfe7 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -9,6 +9,10 @@
# You can do whatever you want with this file.
#
+if [ x$KXZ = "x" ]; then
+ KXZ=xz
+fi
+
BCJ=
LZMA2OPTS=
@@ -20,4 +24,4 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+exec $KXZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
--
2.25.4
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [RFC PATCH] kbuild: add variables for compression tools
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
@ 2020-05-15 2:20 ` Masahiro Yamada
2020-05-15 9:40 ` Denis Efremov
2020-05-21 12:13 ` [RFC PATCH v2] " Denis Efremov
` (3 subsequent siblings)
4 siblings, 1 reply; 20+ messages in thread
From: Masahiro Yamada @ 2020-05-15 2:20 UTC (permalink / raw)
To: Denis Efremov
Cc: Linux Kbuild mailing list, Linux Kernel Mailing List,
Stephen Rothwell, Dmitry Vyukov
On Thu, May 14, 2020 at 10:14 PM Denis Efremov <efremov@linux.com> wrote:
>
> Allow user to use alternative implementations of compression tools.
> For example, multi-threaded tools to speed up the build:
> $ make KGZIP=pigz KXZ=pxz
>
> Variable KGZIP is used instead of GZIP because the latter is reserved
> by the tool. Other variables are prefixed with 'K' for consistency.
This is unfortunate...
'man gzip' says
The obsolescent environment variable GZIP can hold a set of default op‐
tions for gzip. These options are interpreted first and can be over‐
written by explicit command line parameters. As this can cause prob‐
lems when using scripts, this feature is supported only for options
that are reasonably likely to not cause too much harm, and gzip warns
if it is used. This feature will be removed in a future release of
gzip.
It was deprecated in 2015.
commit 5054e88a7934d5ff5ec14231c8b8676161bb45fa
Author: Paul Eggert <eggert@cs.ucla.edu>
Date: Mon Mar 16 14:25:17 2015 -0700
gzip: make the GZIP env var obsolescent
Some possible options I came up with:
[1] Use KGZIP for now, but BZIP2, XZ, etc. for the others.
(Then, rename KGZIP to GZIP when the time comes)
[2] Do not take this patch
The whole build process is parallelized
by 'make -j $(nproc)'.
If you are still eager to use pigz instead gzip,
use a symbolic link or a wrapper shell script.
$ ln -s /usr/bin/pigz /$HOME/bin/gzip
$ PATH="$HOME/bin:$PATH"
Thought?
>
> The credit goes to @grsecurity.
>
> Cc: Stephen Rothwell <sfr@canb.auug.org.au>
> Cc: Dmitry Vyukov <dvyukov@google.com>
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
> Makefile | 11 +++++++++--
> arch/arm/boot/deflate_xip_data.sh | 6 +++++-
> arch/ia64/Makefile | 2 +-
> arch/m68k/Makefile | 4 ++--
> arch/mips/lasat/image/Makefile | 2 +-
> arch/parisc/Makefile | 2 +-
> kernel/gen_kheaders.sh | 6 +++++-
> scripts/Makefile.lib | 12 ++++++------
> scripts/Makefile.package | 6 +++---
> scripts/xz_wrap.sh | 6 +++++-
> 10 files changed, 38 insertions(+), 19 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index 11fe9b1535de..9af13cfeed7a 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -447,6 +447,12 @@ PYTHON = python
> PYTHON3 = python3
> CHECK = sparse
> BASH = bash
> +KGZIP = gzip
> +KBZIP2 = bzip2
> +KLZMA = lzma
> +KLZOP = lzop
> +KLZ4 = lz4c
> +KXZ = xz
>
> CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
> -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
> @@ -496,6 +502,7 @@ export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD
> export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
> export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
> export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
> +export KGZIP KBZIP2 KLZMA KLZOP KLZ4 KXZ
>
> export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
> export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
> @@ -1005,10 +1012,10 @@ export mod_strip_cmd
> mod_compress_cmd = true
> ifdef CONFIG_MODULE_COMPRESS
> ifdef CONFIG_MODULE_COMPRESS_GZIP
> - mod_compress_cmd = gzip -n -f
> + mod_compress_cmd = $(KGZIP) -n -f
> endif # CONFIG_MODULE_COMPRESS_GZIP
> ifdef CONFIG_MODULE_COMPRESS_XZ
> - mod_compress_cmd = xz -f
> + mod_compress_cmd = $(KXZ) -f
> endif # CONFIG_MODULE_COMPRESS_XZ
> endif # CONFIG_MODULE_COMPRESS
> export mod_compress_cmd
> diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
> index 40937248cebe..08dd50e08c17 100755
> --- a/arch/arm/boot/deflate_xip_data.sh
> +++ b/arch/arm/boot/deflate_xip_data.sh
> @@ -19,6 +19,10 @@ XIPIMAGE="$2"
>
> DD="dd status=none"
>
> +if [ x$KGZIP = "x" ]; then
> + KGZIP=gzip
> +fi
> +
> # Use "make V=1" to debug this script.
> case "$KBUILD_VERBOSE" in
> *1*)
> @@ -56,7 +60,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
> # substitute the data section by a compressed version
> $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
> $DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
> -gzip -9 >> "$XIPIMAGE.tmp"
> +$KGZIP -9 >> "$XIPIMAGE.tmp"
>
> # replace kernel binary
> mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
> diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
> index 32240000dc0c..2876a7df1b0a 100644
> --- a/arch/ia64/Makefile
> +++ b/arch/ia64/Makefile
> @@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
> endif
>
> quiet_cmd_gzip = GZIP $@
> -cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> +cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> quiet_cmd_objcopy = OBJCOPY $@
> cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
> index 5d9288384096..e6c7c92aa72e 100644
> --- a/arch/m68k/Makefile
> +++ b/arch/m68k/Makefile
> @@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - bzip2 -1c vmlinux.tmp >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux.tmp >vmlinux.bz2
> rm vmlinux.tmp
> else
> - bzip2 -1c vmlinux >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux >vmlinux.bz2
> endif
>
> archclean:
> diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
> index 78ce4cff1012..617ccb1659d5 100644
> --- a/arch/mips/lasat/image/Makefile
> +++ b/arch/mips/lasat/image/Makefile
> @@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
> $(LD) -r -o $@ -b binary $<
>
> $(obj)/%.gz: $(obj)/%.bin
> - gzip -cf -9 $< > $@
> + $(KGZIP) -cf -9 $< > $@
>
> $(obj)/kImage.bin: $(KERNEL_IMAGE)
> $(OBJCOPY) -O binary -S $^ $@
> diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
> index 628cd8bb7ad8..412ddec0297d 100644
> --- a/arch/parisc/Makefile
> +++ b/arch/parisc/Makefile
> @@ -162,7 +162,7 @@ vmlinuz: bzImage
> $(OBJCOPY) $(boot)/bzImage $@
> else
> vmlinuz: vmlinux
> - @gzip -cf -9 $< > $@
> + @$(KGZIP) -cf -9 $< > $@
> endif
>
> install:
> diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
> index e13ca842eb7e..f3dfaf9f6647 100755
> --- a/kernel/gen_kheaders.sh
> +++ b/kernel/gen_kheaders.sh
> @@ -9,6 +9,10 @@ outdir="$(pwd)"
> tarfile=$1
> cpio_dir=$outdir/$tarfile.tmp
>
> +if [ x$KXZ = "x" ]; then
> + KXZ=xz
> +fi
> +
> dir_list="
> include/
> arch/$SRCARCH/include/
> @@ -88,7 +92,7 @@ find $cpio_dir -type f -print0 |
> find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
> tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
> --owner=0 --group=0 --numeric-owner --no-recursion \
> - -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
> + -I $KXZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
>
> echo $headers_md5 > kernel/kheaders.md5
> echo "$this_file_md5" >> kernel/kheaders.md5
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index 4b799737722c..dd38f5ac8d48 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> # ---------------------------------------------------------------------------
>
> quiet_cmd_gzip = GZIP $@
> - cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> + cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> # DTC
> # ---------------------------------------------------------------------------
> @@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
> )
>
> quiet_cmd_bzip2 = BZIP2 $@
> - cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
> + cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@
>
> # Lzma
> # ---------------------------------------------------------------------------
>
> quiet_cmd_lzma = LZMA $@
> - cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
> + cmd_lzma = { cat $(real-prereqs) | $(KLZMA) -9; $(size_append); } > $@
>
> quiet_cmd_lzo = LZO $@
> - cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
> + cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
>
> quiet_cmd_lz4 = LZ4 $@
> - cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
> + cmd_lz4 = { cat $(real-prereqs) | $(KLZ4) -l -c1 stdin stdout; \
> $(size_append); } > $@
>
> # U-Boot mkimage
> @@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
> $(size_append); } > $@
>
> quiet_cmd_xzmisc = XZMISC $@
> - cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
> + cmd_xzmisc = cat $(real-prereqs) | $(KXZ) --check=crc32 --lzma2=dict=1MiB > $@
>
> # ASM offsets
> # ---------------------------------------------------------------------------
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index 02135d2671a6..1b91fe1bfcdb 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
> tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
> rm -r $(perf-tar); \
> $(if $(findstring tar-src,$@),, \
> -$(if $(findstring bz2,$@),bzip2, \
> -$(if $(findstring gz,$@),gzip, \
> -$(if $(findstring xz,$@),xz, \
> +$(if $(findstring bz2,$@),$(KBZIP2), \
> +$(if $(findstring gz,$@),$(KGZIP), \
> +$(if $(findstring xz,$@),$(KXZ), \
> $(error unknown target $@)))) \
> -f -9 $(perf-tar).tar)
>
> diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
> index 7a2d372f4885..4922102dbfe7 100755
> --- a/scripts/xz_wrap.sh
> +++ b/scripts/xz_wrap.sh
> @@ -9,6 +9,10 @@
> # You can do whatever you want with this file.
> #
>
> +if [ x$KXZ = "x" ]; then
> + KXZ=xz
> +fi
> +
> BCJ=
> LZMA2OPTS=
>
> @@ -20,4 +24,4 @@ case $SRCARCH in
> sparc) BCJ=--sparc ;;
> esac
>
> -exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
> +exec $KXZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
> --
> 2.25.4
>
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [RFC PATCH] kbuild: add variables for compression tools
2020-05-15 2:20 ` Masahiro Yamada
@ 2020-05-15 9:40 ` Denis Efremov
2020-05-21 7:20 ` Masahiro Yamada
0 siblings, 1 reply; 20+ messages in thread
From: Denis Efremov @ 2020-05-15 9:40 UTC (permalink / raw)
To: Masahiro Yamada
Cc: Linux Kbuild mailing list, Linux Kernel Mailing List,
Stephen Rothwell, Dmitry Vyukov
It seems that I missed a couple of tar commands in the patch:
scripts/Makefile.package
scripts/package/buildtar
On 5/15/20 5:20 AM, Masahiro Yamada wrote:
> On Thu, May 14, 2020 at 10:14 PM Denis Efremov <efremov@linux.com> wrote:
>>
>
> commit 5054e88a7934d5ff5ec14231c8b8676161bb45fa
> Author: Paul Eggert <eggert@cs.ucla.edu>
> Date: Mon Mar 16 14:25:17 2015 -0700
>
> gzip: make the GZIP env var obsolescent
Other implementations can depend on this.
pigz still parses GZIP env var:
https://github.com/madler/pigz/blob/master/pigz.c#L4346
>
> Some possible options I came up with:
>
>
> [1] Use KGZIP for now, but BZIP2, XZ, etc. for the others.
>
> (Then, rename KGZIP to GZIP when the time comes)
>
>
> [2] Do not take this patch
>
> The whole build process is parallelized
> by 'make -j $(nproc)'.
>
> If you are still eager to use pigz instead gzip,
> use a symbolic link or a wrapper shell script.
>
> $ ln -s /usr/bin/pigz /$HOME/bin/gzip
> $ PATH="$HOME/bin:$PATH"
>
[3] GZIP at frontend, KGZIP or _GZIP internally? Something like:
$ cat Makefile
GZIP=gzip
override KGZIP=$(GZIP) # optional overrdide. Used to force GZIP value
# in case: make KGZIP=test
unexport GZIP
export KGZIP
default:
@env | grep GZIP
$ make GZIP=test
KGZIP=test
Thanks,
Denis
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [RFC PATCH] kbuild: add variables for compression tools
2020-05-15 9:40 ` Denis Efremov
@ 2020-05-21 7:20 ` Masahiro Yamada
0 siblings, 0 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-05-21 7:20 UTC (permalink / raw)
To: Denis Efremov
Cc: Linux Kbuild mailing list, Linux Kernel Mailing List,
Stephen Rothwell, Dmitry Vyukov
On Fri, May 15, 2020 at 6:40 PM Denis Efremov <efremov@linux.com> wrote:
>
> It seems that I missed a couple of tar commands in the patch:
> scripts/Makefile.package
> scripts/package/buildtar
>
> On 5/15/20 5:20 AM, Masahiro Yamada wrote:
> > On Thu, May 14, 2020 at 10:14 PM Denis Efremov <efremov@linux.com> wrote:
> >>
> >
> > commit 5054e88a7934d5ff5ec14231c8b8676161bb45fa
> > Author: Paul Eggert <eggert@cs.ucla.edu>
> > Date: Mon Mar 16 14:25:17 2015 -0700
> >
> > gzip: make the GZIP env var obsolescent
>
> Other implementations can depend on this.
> pigz still parses GZIP env var:
> https://github.com/madler/pigz/blob/master/pigz.c#L4346
>
> >
> > Some possible options I came up with:
> >
> >
> > [1] Use KGZIP for now, but BZIP2, XZ, etc. for the others.
> >
> > (Then, rename KGZIP to GZIP when the time comes)
> >
> >
> > [2] Do not take this patch
> >
> > The whole build process is parallelized
> > by 'make -j $(nproc)'.
> >
> > If you are still eager to use pigz instead gzip,
> > use a symbolic link or a wrapper shell script.
> >
> > $ ln -s /usr/bin/pigz /$HOME/bin/gzip
> > $ PATH="$HOME/bin:$PATH"
> >
>
> [3] GZIP at frontend, KGZIP or _GZIP internally? Something like:
>
> $ cat Makefile
> GZIP=gzip
> override KGZIP=$(GZIP) # optional overrdide. Used to force GZIP value
> # in case: make KGZIP=test
>
> unexport GZIP
The command line option is really strong,
so you cannot negate it by 'unexport GZIP'.
override GZIP :=
does not work either in sub-make.
> export KGZIP
>
> default:
> @env | grep GZIP
>
> $ make GZIP=test
> KGZIP=test
>
> Thanks,
> Denis
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
* [RFC PATCH v2] kbuild: add variables for compression tools
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
2020-05-15 2:20 ` Masahiro Yamada
@ 2020-05-21 12:13 ` Denis Efremov
2020-05-22 8:43 ` Denis Efremov
2020-05-30 13:44 ` [RFC PATCH v3] " Denis Efremov
` (2 subsequent siblings)
4 siblings, 1 reply; 20+ messages in thread
From: Denis Efremov @ 2020-05-21 12:13 UTC (permalink / raw)
To: Masahiro Yamada; +Cc: Denis Efremov, linux-kbuild, linux-kernel
Allow user to use alternative implementations of compression tools.
For example, multi-threaded tools to speed up the build:
$ make GZIP=pigz BZIP2=pbzip2
Variable _GZIP is used internally instead of GZIP because the latter is
reserved by the tool. The use of GZIP in gzip tool is obsolete since
2015. However, alternative implementations (e.g., pigz) can still rely
on it.
The credit goes to @grsecurity.
As a sidenote, for multi-threaded lzma, xz compression one can use:
$ export XZ_OPT="--threads=0"
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Compile tested on x86_64, parisc, arm64, mips, m68k.
Makefile | 16 ++++++++++++++--
arch/arm/boot/deflate_xip_data.sh | 6 +++++-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 8 ++++----
arch/mips/lasat/image/Makefile | 2 +-
arch/parisc/Makefile | 2 +-
| 6 +++++-
scripts/Kbuild.include | 4 ++++
scripts/Makefile.lib | 12 ++++++------
scripts/Makefile.package | 8 ++++----
scripts/package/buildtar | 15 ++++++++++++---
scripts/xz_wrap.sh | 6 +++++-
12 files changed, 62 insertions(+), 25 deletions(-)
diff --git a/Makefile b/Makefile
index 04f5662ae61a..dd5ff189d97e 100644
--- a/Makefile
+++ b/Makefile
@@ -447,6 +447,12 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+GZIP = gzip
+BZIP2 = bzip2
+LZMA = lzma
+LZO = lzop
+LZ4 = lz4c
+XZ = xz
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -492,10 +498,16 @@ KBUILD_LDFLAGS :=
GCC_PLUGINS_CFLAGS :=
CLANG_FLAGS :=
+# GZIP env var is used by old (<= 2015) versions of the tool
+# and alternative implementations for additional arguments
+override _GZIP=$(GZIP)
+unexport GZIP
+
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
+export _GZIP BZIP2 LZMA LZO LZ4 XZ
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
@@ -1005,10 +1017,10 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = gzip -n -f
+ mod_compress_cmd = $(_GZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
- mod_compress_cmd = xz -f
+ mod_compress_cmd = $(XZ) -f
endif # CONFIG_MODULE_COMPRESS_XZ
endif # CONFIG_MODULE_COMPRESS
export mod_compress_cmd
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 40937248cebe..81253142d2ae 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -19,6 +19,10 @@ XIPIMAGE="$2"
DD="dd status=none"
+if [ x$_GZIP = "x" ]; then
+ _GZIP=gzip
+fi
+
# Use "make V=1" to debug this script.
case "$KBUILD_VERBOSE" in
*1*)
@@ -56,7 +60,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-gzip -9 >> "$XIPIMAGE.tmp"
+$_GZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index 32240000dc0c..f817f3d5e758 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 5d9288384096..7f259a723753 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- gzip -9c vmlinux.tmp >vmlinux.gz
+ $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
rm vmlinux.tmp
else
- gzip -9c vmlinux >vmlinux.gz
+ $(_GZIP) -9c vmlinux >vmlinux.gz
endif
bzImage: vmlinux.bz2
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- bzip2 -1c vmlinux.tmp >vmlinux.bz2
+ $(BZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- bzip2 -1c vmlinux >vmlinux.bz2
+ $(BZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
index 78ce4cff1012..24d8dbde1109 100644
--- a/arch/mips/lasat/image/Makefile
+++ b/arch/mips/lasat/image/Makefile
@@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
$(LD) -r -o $@ -b binary $<
$(obj)/%.gz: $(obj)/%.bin
- gzip -cf -9 $< > $@
+ $(_GZIP) -cf -9 $< > $@
$(obj)/kImage.bin: $(KERNEL_IMAGE)
$(OBJCOPY) -O binary -S $^ $@
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 628cd8bb7ad8..e1aa514aeb36 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @gzip -cf -9 $< > $@
+ @$(_GZIP) -cf -9 $< > $@
endif
install:
--git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
index e13ca842eb7e..58e05b3702f3 100755
--- a/kernel/gen_kheaders.sh
+++ b/kernel/gen_kheaders.sh
@@ -9,6 +9,10 @@ outdir="$(pwd)"
tarfile=$1
cpio_dir=$outdir/$tarfile.tmp
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
+
dir_list="
include/
arch/$SRCARCH/include/
@@ -88,7 +92,7 @@ find $cpio_dir -type f -print0 |
find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
--owner=0 --group=0 --numeric-owner --no-recursion \
- -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
+ -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
echo $headers_md5 > kernel/kheaders.md5
echo "$this_file_md5" >> kernel/kheaders.md5
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 6cabf20ce66a..d7c25b3cb1a8 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -2,6 +2,10 @@
####
# kbuild: Generic definitions
+# GZIP env var is used by old (<= 2015) versions of the tool
+# and alternative implementations for additional arguments
+unexport GZIP
+
# Convenient variables
comma := ,
quote := "
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 4b799737722c..812f1c0e5beb 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(BZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
quiet_cmd_lzma = LZMA $@
- cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
+ cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(LZO) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
+ cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
@@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
$(size_append); } > $@
quiet_cmd_xzmisc = XZMISC $@
- cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
+ cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
# ASM offsets
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 02135d2671a6..537179828d0c 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
false; \
fi ; \
$(srctree)/scripts/setlocalversion --save-scmversion; \
-tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
+tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
rm -f $(objtree)/.scmversion
@@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),bzip2, \
-$(if $(findstring gz,$@),gzip, \
-$(if $(findstring xz,$@),xz, \
+$(if $(findstring bz2,$@),$(BZIP2), \
+$(if $(findstring gz,$@),$(_GZIP), \
+$(if $(findstring xz,$@),$(XZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/package/buildtar b/scripts/package/buildtar
index 77c7caefede1..8a0ebcc292af 100755
--- a/scripts/package/buildtar
+++ b/scripts/package/buildtar
@@ -19,6 +19,15 @@ set -e
tmpdir="${objtree}/tar-install"
tarball="${objtree}/linux-${KERNELRELEASE}-${ARCH}.tar"
+if [ x$_GZIP = "x" ]; then
+ _GZIP=gzip
+fi
+if [ x$BZIP2 = "x" ]; then
+ BZIP2=bzip2
+fi
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
#
# Figure out how to compress, if requested at all
@@ -28,15 +37,15 @@ case "${1}" in
opts=
;;
targz-pkg)
- opts=--gzip
+ opts="-I ${_GZIP}"
tarball=${tarball}.gz
;;
tarbz2-pkg)
- opts=--bzip2
+ opts="-I ${BZIP2}"
tarball=${tarball}.bz2
;;
tarxz-pkg)
- opts=--xz
+ opts="-I ${XZ}"
tarball=${tarball}.xz
;;
*)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index 7a2d372f4885..b387cd208952 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -9,6 +9,10 @@
# You can do whatever you want with this file.
#
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
+
BCJ=
LZMA2OPTS=
@@ -20,4 +24,4 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
--
2.26.2
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [RFC PATCH v2] kbuild: add variables for compression tools
2020-05-21 12:13 ` [RFC PATCH v2] " Denis Efremov
@ 2020-05-22 8:43 ` Denis Efremov
0 siblings, 0 replies; 20+ messages in thread
From: Denis Efremov @ 2020-05-22 8:43 UTC (permalink / raw)
To: Masahiro Yamada; +Cc: linux-kbuild, linux-kernel
On 5/21/20 3:13 PM, Denis Efremov wrote:
> Allow user to use alternative implementations of compression tools.
> For example, multi-threaded tools to speed up the build:
> $ make GZIP=pigz BZIP2=pbzip2
>
> Variable _GZIP is used internally instead of GZIP because the latter is
> reserved by the tool. The use of GZIP in gzip tool is obsolete since
> 2015. However, alternative implementations (e.g., pigz) can still rely
> on it.
I faced the same problem with BZIP2 (BZIP) env var and LZOP.
I didn't noticed it earlier because pbzip2 doesn't respect BZIP2 env var unlike bzip2.
xz, lzma are ok with XZ, LZMA.
It's possible to handle BZIP2, LZOP vars like GZIP.
>
> The credit goes to @grsecurity.
>
> As a sidenote, for multi-threaded lzma, xz compression one can use:
> $ export XZ_OPT="--threads=0"
>
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
>
> Compile tested on x86_64, parisc, arm64, mips, m68k.
>
> Makefile | 16 ++++++++++++++--
> arch/arm/boot/deflate_xip_data.sh | 6 +++++-
> arch/ia64/Makefile | 2 +-
> arch/m68k/Makefile | 8 ++++----
> arch/mips/lasat/image/Makefile | 2 +-
> arch/parisc/Makefile | 2 +-
> kernel/gen_kheaders.sh | 6 +++++-
> scripts/Kbuild.include | 4 ++++
> scripts/Makefile.lib | 12 ++++++------
> scripts/Makefile.package | 8 ++++----
> scripts/package/buildtar | 15 ++++++++++++---
> scripts/xz_wrap.sh | 6 +++++-
> 12 files changed, 62 insertions(+), 25 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index 04f5662ae61a..dd5ff189d97e 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -447,6 +447,12 @@ PYTHON = python
> PYTHON3 = python3
> CHECK = sparse
> BASH = bash
> +GZIP = gzip
> +BZIP2 = bzip2
> +LZMA = lzma
> +LZO = lzop
> +LZ4 = lz4c
> +XZ = xz
>
> CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
> -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
> @@ -492,10 +498,16 @@ KBUILD_LDFLAGS :=
> GCC_PLUGINS_CFLAGS :=
> CLANG_FLAGS :=
>
> +# GZIP env var is used by old (<= 2015) versions of the tool
> +# and alternative implementations for additional arguments
> +override _GZIP=$(GZIP)
> +unexport GZIP
> +
> export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
> export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
> export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
> export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
> +export _GZIP BZIP2 LZMA LZO LZ4 XZ
>
> export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
> export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
> @@ -1005,10 +1017,10 @@ export mod_strip_cmd
> mod_compress_cmd = true
> ifdef CONFIG_MODULE_COMPRESS
> ifdef CONFIG_MODULE_COMPRESS_GZIP
> - mod_compress_cmd = gzip -n -f
> + mod_compress_cmd = $(_GZIP) -n -f
> endif # CONFIG_MODULE_COMPRESS_GZIP
> ifdef CONFIG_MODULE_COMPRESS_XZ
> - mod_compress_cmd = xz -f
> + mod_compress_cmd = $(XZ) -f
> endif # CONFIG_MODULE_COMPRESS_XZ
> endif # CONFIG_MODULE_COMPRESS
> export mod_compress_cmd
> diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
> index 40937248cebe..81253142d2ae 100755
> --- a/arch/arm/boot/deflate_xip_data.sh
> +++ b/arch/arm/boot/deflate_xip_data.sh
> @@ -19,6 +19,10 @@ XIPIMAGE="$2"
>
> DD="dd status=none"
>
> +if [ x$_GZIP = "x" ]; then
> + _GZIP=gzip
> +fi
> +
> # Use "make V=1" to debug this script.
> case "$KBUILD_VERBOSE" in
> *1*)
> @@ -56,7 +60,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
> # substitute the data section by a compressed version
> $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
> $DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
> -gzip -9 >> "$XIPIMAGE.tmp"
> +$_GZIP -9 >> "$XIPIMAGE.tmp"
>
> # replace kernel binary
> mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
> diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
> index 32240000dc0c..f817f3d5e758 100644
> --- a/arch/ia64/Makefile
> +++ b/arch/ia64/Makefile
> @@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
> endif
>
> quiet_cmd_gzip = GZIP $@
> -cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> +cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
>
> quiet_cmd_objcopy = OBJCOPY $@
> cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
> index 5d9288384096..7f259a723753 100644
> --- a/arch/m68k/Makefile
> +++ b/arch/m68k/Makefile
> @@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - gzip -9c vmlinux.tmp >vmlinux.gz
> + $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
> rm vmlinux.tmp
> else
> - gzip -9c vmlinux >vmlinux.gz
> + $(_GZIP) -9c vmlinux >vmlinux.gz
> endif
>
> bzImage: vmlinux.bz2
> @@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - bzip2 -1c vmlinux.tmp >vmlinux.bz2
> + $(BZIP2) -1c vmlinux.tmp >vmlinux.bz2
> rm vmlinux.tmp
> else
> - bzip2 -1c vmlinux >vmlinux.bz2
> + $(BZIP2) -1c vmlinux >vmlinux.bz2
> endif
>
> archclean:
> diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
> index 78ce4cff1012..24d8dbde1109 100644
> --- a/arch/mips/lasat/image/Makefile
> +++ b/arch/mips/lasat/image/Makefile
> @@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
> $(LD) -r -o $@ -b binary $<
>
> $(obj)/%.gz: $(obj)/%.bin
> - gzip -cf -9 $< > $@
> + $(_GZIP) -cf -9 $< > $@
>
> $(obj)/kImage.bin: $(KERNEL_IMAGE)
> $(OBJCOPY) -O binary -S $^ $@
> diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
> index 628cd8bb7ad8..e1aa514aeb36 100644
> --- a/arch/parisc/Makefile
> +++ b/arch/parisc/Makefile
> @@ -162,7 +162,7 @@ vmlinuz: bzImage
> $(OBJCOPY) $(boot)/bzImage $@
> else
> vmlinuz: vmlinux
> - @gzip -cf -9 $< > $@
> + @$(_GZIP) -cf -9 $< > $@
> endif
>
> install:
> diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
> index e13ca842eb7e..58e05b3702f3 100755
> --- a/kernel/gen_kheaders.sh
> +++ b/kernel/gen_kheaders.sh
> @@ -9,6 +9,10 @@ outdir="$(pwd)"
> tarfile=$1
> cpio_dir=$outdir/$tarfile.tmp
>
> +if [ x$XZ = "x" ]; then
> + XZ=xz
> +fi
> +
> dir_list="
> include/
> arch/$SRCARCH/include/
> @@ -88,7 +92,7 @@ find $cpio_dir -type f -print0 |
> find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
> tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
> --owner=0 --group=0 --numeric-owner --no-recursion \
> - -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
> + -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
>
> echo $headers_md5 > kernel/kheaders.md5
> echo "$this_file_md5" >> kernel/kheaders.md5
> diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> index 6cabf20ce66a..d7c25b3cb1a8 100644
> --- a/scripts/Kbuild.include
> +++ b/scripts/Kbuild.include
> @@ -2,6 +2,10 @@
> ####
> # kbuild: Generic definitions
>
> +# GZIP env var is used by old (<= 2015) versions of the tool
> +# and alternative implementations for additional arguments
> +unexport GZIP
> +
> # Convenient variables
> comma := ,
> quote := "
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index 4b799737722c..812f1c0e5beb 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> # ---------------------------------------------------------------------------
>
> quiet_cmd_gzip = GZIP $@
> - cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> + cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
>
> # DTC
> # ---------------------------------------------------------------------------
> @@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
> )
>
> quiet_cmd_bzip2 = BZIP2 $@
> - cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
> + cmd_bzip2 = { cat $(real-prereqs) | $(BZIP2) -9; $(size_append); } > $@
>
> # Lzma
> # ---------------------------------------------------------------------------
>
> quiet_cmd_lzma = LZMA $@
> - cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
> + cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
>
> quiet_cmd_lzo = LZO $@
> - cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
> + cmd_lzo = { cat $(real-prereqs) | $(LZO) -9; $(size_append); } > $@
>
> quiet_cmd_lz4 = LZ4 $@
> - cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
> + cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
> $(size_append); } > $@
>
> # U-Boot mkimage
> @@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
> $(size_append); } > $@
>
> quiet_cmd_xzmisc = XZMISC $@
> - cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
> + cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
>
> # ASM offsets
> # ---------------------------------------------------------------------------
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index 02135d2671a6..537179828d0c 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
> false; \
> fi ; \
> $(srctree)/scripts/setlocalversion --save-scmversion; \
> -tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> +tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
> rm -f $(objtree)/.scmversion
>
> @@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
> tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
> rm -r $(perf-tar); \
> $(if $(findstring tar-src,$@),, \
> -$(if $(findstring bz2,$@),bzip2, \
> -$(if $(findstring gz,$@),gzip, \
> -$(if $(findstring xz,$@),xz, \
> +$(if $(findstring bz2,$@),$(BZIP2), \
> +$(if $(findstring gz,$@),$(_GZIP), \
> +$(if $(findstring xz,$@),$(XZ), \
> $(error unknown target $@)))) \
> -f -9 $(perf-tar).tar)
>
> diff --git a/scripts/package/buildtar b/scripts/package/buildtar
> index 77c7caefede1..8a0ebcc292af 100755
> --- a/scripts/package/buildtar
> +++ b/scripts/package/buildtar
> @@ -19,6 +19,15 @@ set -e
> tmpdir="${objtree}/tar-install"
> tarball="${objtree}/linux-${KERNELRELEASE}-${ARCH}.tar"
>
> +if [ x$_GZIP = "x" ]; then
> + _GZIP=gzip
> +fi
> +if [ x$BZIP2 = "x" ]; then
> + BZIP2=bzip2
> +fi
> +if [ x$XZ = "x" ]; then
> + XZ=xz
> +fi
>
> #
> # Figure out how to compress, if requested at all
> @@ -28,15 +37,15 @@ case "${1}" in
> opts=
> ;;
> targz-pkg)
> - opts=--gzip
> + opts="-I ${_GZIP}"
> tarball=${tarball}.gz
> ;;
> tarbz2-pkg)
> - opts=--bzip2
> + opts="-I ${BZIP2}"
> tarball=${tarball}.bz2
> ;;
> tarxz-pkg)
> - opts=--xz
> + opts="-I ${XZ}"
> tarball=${tarball}.xz
> ;;
> *)
> diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
> index 7a2d372f4885..b387cd208952 100755
> --- a/scripts/xz_wrap.sh
> +++ b/scripts/xz_wrap.sh
> @@ -9,6 +9,10 @@
> # You can do whatever you want with this file.
> #
>
> +if [ x$XZ = "x" ]; then
> + XZ=xz
> +fi
> +
> BCJ=
> LZMA2OPTS=
>
> @@ -20,4 +24,4 @@ case $SRCARCH in
> sparc) BCJ=--sparc ;;
> esac
>
> -exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
> +exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
>
^ permalink raw reply [flat|nested] 20+ messages in thread
* [RFC PATCH v3] kbuild: add variables for compression tools
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
2020-05-15 2:20 ` Masahiro Yamada
2020-05-21 12:13 ` [RFC PATCH v2] " Denis Efremov
@ 2020-05-30 13:44 ` Denis Efremov
2020-06-01 12:45 ` Masahiro Yamada
2020-06-03 9:20 ` [RFC PATCH v4] " Denis Efremov
2020-06-05 7:39 ` [PATCH v5] " Denis Efremov
4 siblings, 1 reply; 20+ messages in thread
From: Denis Efremov @ 2020-05-30 13:44 UTC (permalink / raw)
To: Masahiro Yamada; +Cc: Denis Efremov, linux-kbuild, linux-kernel
Allow user to use alternative implementations of compression tools,
such as pigz, pbzip2, pxz. For example, multi-threaded tools to
speed up the build:
$ make GZIP=pigz BZIP2=pbzip2
Variables _GZIP, _BZIP2, _LZOP are used internally because original env
vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
since 2015. However, alternative implementations (e.g., pigz) still rely
on it. BZIP2, BZIP, LZOP vars are not obsolescent.
The credit goes to @grsecurity.
As a sidenote, for multi-threaded lzma, xz compression one can use:
$ export XZ_OPT="--threads=0"
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Changes in v2:
- _GZIP used instead of GZIP
- tar commands altered to use tools from the vars
Changes in v3:
- _BZIP2 used instead of BZIP2
- _LZOP used instead of LZOP
Makefile | 19 +++++++++++++++++--
arch/arm/boot/deflate_xip_data.sh | 6 +++++-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 8 ++++----
arch/mips/lasat/image/Makefile | 2 +-
arch/parisc/Makefile | 2 +-
| 6 +++++-
scripts/Kbuild.include | 5 +++++
scripts/Makefile.lib | 12 ++++++------
scripts/Makefile.package | 8 ++++----
scripts/package/buildtar | 15 ++++++++++++---
scripts/xz_wrap.sh | 6 +++++-
12 files changed, 66 insertions(+), 25 deletions(-)
diff --git a/Makefile b/Makefile
index f0d118b86287..2ac2dc87f903 100644
--- a/Makefile
+++ b/Makefile
@@ -447,6 +447,12 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+GZIP = gzip
+BZIP2 = bzip2
+LZOP = lzop
+LZMA = lzma
+LZ4 = lz4c
+XZ = xz
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -492,10 +498,19 @@ KBUILD_LDFLAGS :=
GCC_PLUGINS_CFLAGS :=
CLANG_FLAGS :=
+# GZIP, BZIP2, LZOP env vars are used by the tools
+override _GZIP=$(GZIP)
+override _BZIP2=$(BZIP2)
+override _LZOP=$(LZOP)
+unexport GZIP
+unexport BZIP2
+unexport LZOP
+
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
+export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
@@ -1005,10 +1020,10 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = gzip -n -f
+ mod_compress_cmd = $(_GZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
- mod_compress_cmd = xz -f
+ mod_compress_cmd = $(XZ) -f
endif # CONFIG_MODULE_COMPRESS_XZ
endif # CONFIG_MODULE_COMPRESS
export mod_compress_cmd
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 40937248cebe..81253142d2ae 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -19,6 +19,10 @@ XIPIMAGE="$2"
DD="dd status=none"
+if [ x$_GZIP = "x" ]; then
+ _GZIP=gzip
+fi
+
# Use "make V=1" to debug this script.
case "$KBUILD_VERBOSE" in
*1*)
@@ -56,7 +60,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-gzip -9 >> "$XIPIMAGE.tmp"
+$_GZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index 32240000dc0c..f817f3d5e758 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 5d9288384096..ce6db5e5a5a3 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- gzip -9c vmlinux.tmp >vmlinux.gz
+ $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
rm vmlinux.tmp
else
- gzip -9c vmlinux >vmlinux.gz
+ $(_GZIP) -9c vmlinux >vmlinux.gz
endif
bzImage: vmlinux.bz2
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- bzip2 -1c vmlinux.tmp >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- bzip2 -1c vmlinux >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
index 78ce4cff1012..24d8dbde1109 100644
--- a/arch/mips/lasat/image/Makefile
+++ b/arch/mips/lasat/image/Makefile
@@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
$(LD) -r -o $@ -b binary $<
$(obj)/%.gz: $(obj)/%.bin
- gzip -cf -9 $< > $@
+ $(_GZIP) -cf -9 $< > $@
$(obj)/kImage.bin: $(KERNEL_IMAGE)
$(OBJCOPY) -O binary -S $^ $@
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 628cd8bb7ad8..e1aa514aeb36 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @gzip -cf -9 $< > $@
+ @$(_GZIP) -cf -9 $< > $@
endif
install:
--git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
index e13ca842eb7e..58e05b3702f3 100755
--- a/kernel/gen_kheaders.sh
+++ b/kernel/gen_kheaders.sh
@@ -9,6 +9,10 @@ outdir="$(pwd)"
tarfile=$1
cpio_dir=$outdir/$tarfile.tmp
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
+
dir_list="
include/
arch/$SRCARCH/include/
@@ -88,7 +92,7 @@ find $cpio_dir -type f -print0 |
find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
--owner=0 --group=0 --numeric-owner --no-recursion \
- -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
+ -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
echo $headers_md5 > kernel/kheaders.md5
echo "$this_file_md5" >> kernel/kheaders.md5
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 6cabf20ce66a..b3b49fe7f25f 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -2,6 +2,11 @@
####
# kbuild: Generic definitions
+# GZIP, BZIP2, LZOP env vars are used by the tools
+unexport GZIP
+unexport BZIP2
+unexport LZOP
+
# Convenient variables
comma := ,
quote := "
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 4b799737722c..a1c1424e6b52 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
quiet_cmd_lzma = LZMA $@
- cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
+ cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
+ cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
@@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
$(size_append); } > $@
quiet_cmd_xzmisc = XZMISC $@
- cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
+ cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
# ASM offsets
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 02135d2671a6..b2b6153af63a 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
false; \
fi ; \
$(srctree)/scripts/setlocalversion --save-scmversion; \
-tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
+tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
rm -f $(objtree)/.scmversion
@@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),bzip2, \
-$(if $(findstring gz,$@),gzip, \
-$(if $(findstring xz,$@),xz, \
+$(if $(findstring bz2,$@),$(_BZIP2), \
+$(if $(findstring gz,$@),$(_GZIP), \
+$(if $(findstring xz,$@),$(XZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/package/buildtar b/scripts/package/buildtar
index 77c7caefede1..165826c12da9 100755
--- a/scripts/package/buildtar
+++ b/scripts/package/buildtar
@@ -19,6 +19,15 @@ set -e
tmpdir="${objtree}/tar-install"
tarball="${objtree}/linux-${KERNELRELEASE}-${ARCH}.tar"
+if [ x$_GZIP = "x" ]; then
+ _GZIP=gzip
+fi
+if [ x$_BZIP2 = "x" ]; then
+ _BZIP2=bzip2
+fi
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
#
# Figure out how to compress, if requested at all
@@ -28,15 +37,15 @@ case "${1}" in
opts=
;;
targz-pkg)
- opts=--gzip
+ opts="-I ${_GZIP}"
tarball=${tarball}.gz
;;
tarbz2-pkg)
- opts=--bzip2
+ opts="-I ${_BZIP2}"
tarball=${tarball}.bz2
;;
tarxz-pkg)
- opts=--xz
+ opts="-I ${XZ}"
tarball=${tarball}.xz
;;
*)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index 7a2d372f4885..b387cd208952 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -9,6 +9,10 @@
# You can do whatever you want with this file.
#
+if [ x$XZ = "x" ]; then
+ XZ=xz
+fi
+
BCJ=
LZMA2OPTS=
@@ -20,4 +24,4 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
--
2.26.2
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [RFC PATCH v3] kbuild: add variables for compression tools
2020-05-30 13:44 ` [RFC PATCH v3] " Denis Efremov
@ 2020-06-01 12:45 ` Masahiro Yamada
0 siblings, 0 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-06-01 12:45 UTC (permalink / raw)
To: Denis Efremov; +Cc: Linux Kbuild mailing list, Linux Kernel Mailing List
On Sat, May 30, 2020 at 10:44 PM Denis Efremov <efremov@linux.com> wrote:
>
> Allow user to use alternative implementations of compression tools,
> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
> speed up the build:
> $ make GZIP=pigz BZIP2=pbzip2
>
> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
> since 2015. However, alternative implementations (e.g., pigz) still rely
> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>
> The credit goes to @grsecurity.
>
> As a sidenote, for multi-threaded lzma, xz compression one can use:
> $ export XZ_OPT="--threads=0"
>
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
> Changes in v2:
> - _GZIP used instead of GZIP
> - tar commands altered to use tools from the vars
> Changes in v3:
> - _BZIP2 used instead of BZIP2
> - _LZOP used instead of LZOP
>
> diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
> index 6cabf20ce66a..b3b49fe7f25f 100644
> --- a/scripts/Kbuild.include
> +++ b/scripts/Kbuild.include
> @@ -2,6 +2,11 @@
> ####
> # kbuild: Generic definitions
>
> +# GZIP, BZIP2, LZOP env vars are used by the tools
> +unexport GZIP
> +unexport BZIP2
> +unexport LZOP
> +
> # Convenient variables
> comma := ,
> quote := "
I do not like unexport in Kbuild.include
One idea is to use MAKEOVERRIDES to
implement this all in top Makefile.
diff --git a/Makefile b/Makefile
index e0aeeedbef55..4b7e7496c904 100644
--- a/Makefile
+++ b/Makefile
@@ -458,6 +458,26 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+GZIP = gzip
+BZIP2 = bzip2
+LZMA = lzma
+LZO = lzop
+LZ4 = lz4c
+XZ = xz
+
+# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
+# line interface, but use _GZIP, _BZIP2, _LZOP internally.
+_GZIP := $(GZIP)
+_BZIP2 := $(BZIP2)
+_LZOP := $(LZOP)
+
+# Reset GZIP, BZIP2, LZOP in this Makefile
+override GZIP=
+override BZIP2=
+override LZOP=
+
+# Reset GZIP, BZIP2, LZOP in recursive invocations
+MAKEOVERRIDES += GZIP= BZIP2= LZOP=
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -506,6 +526,7 @@ CLANG_FLAGS :=
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS
CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX
YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
+export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
Another solution is to use 'KGZIP' etc. as in v1.
> diff --git a/scripts/package/buildtar b/scripts/package/buildtar
> index 77c7caefede1..165826c12da9 100755
> --- a/scripts/package/buildtar
> +++ b/scripts/package/buildtar
> @@ -19,6 +19,15 @@ set -e
> tmpdir="${objtree}/tar-install"
> tarball="${objtree}/linux-${KERNELRELEASE}-${ARCH}.tar"
>
> +if [ x$_GZIP = "x" ]; then
> + _GZIP=gzip
> +fi
> +if [ x$_BZIP2 = "x" ]; then
> + _BZIP2=bzip2
> +fi
> +if [ x$XZ = "x" ]; then
> + XZ=xz
> +fi
>
Is this necessary?
These shell scripts are not intended to be run
stand-alone.
--
Best Regards
Masahiro Yamada
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [RFC PATCH v4] kbuild: add variables for compression tools
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
` (2 preceding siblings ...)
2020-05-30 13:44 ` [RFC PATCH v3] " Denis Efremov
@ 2020-06-03 9:20 ` Denis Efremov
2020-06-04 0:12 ` Masahiro Yamada
2020-06-05 7:39 ` [PATCH v5] " Denis Efremov
4 siblings, 1 reply; 20+ messages in thread
From: Denis Efremov @ 2020-06-03 9:20 UTC (permalink / raw)
To: Masahiro Yamada; +Cc: Denis Efremov, linux-kbuild, linux-kernel
Allow user to use alternative implementations of compression tools,
such as pigz, pbzip2, pxz. For example, multi-threaded tools to
speed up the build:
$ make GZIP=pigz BZIP2=pbzip2
Variables _GZIP, _BZIP2, _LZOP are used internally because original env
vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
since 2015. However, alternative implementations (e.g., pigz) still rely
on it. BZIP2, BZIP, LZOP vars are not obsolescent.
The credit goes to @grsecurity.
As a sidenote, for multi-threaded lzma, xz compression one can use:
$ export XZ_OPT="--threads=0"
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Changes in v2:
- _GZIP used instead of GZIP
- tar commands altered to use tools from the vars
Changes in v3:
- _BZIP2 used instead of BZIP2
- _LZOP used instead of LZOP
Changes in v4:
- Unexports removed from Kbuild.include
- MAKEOVERRIDES used in top Makefile
- All variables checks removed from scripts
Makefile | 25 +++++++++++++++++++++++--
arch/arm/boot/deflate_xip_data.sh | 2 +-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 8 ++++----
arch/mips/lasat/image/Makefile | 2 +-
arch/parisc/Makefile | 2 +-
| 2 +-
scripts/Makefile.lib | 12 ++++++------
scripts/Makefile.package | 8 ++++----
scripts/package/buildtar | 6 +++---
scripts/xz_wrap.sh | 2 +-
11 files changed, 46 insertions(+), 25 deletions(-)
diff --git a/Makefile b/Makefile
index b668725a2a62..9a9cce1ed83d 100644
--- a/Makefile
+++ b/Makefile
@@ -447,6 +447,26 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+GZIP = gzip
+BZIP2 = bzip2
+LZOP = lzop
+LZMA = lzma
+LZ4 = lz4c
+XZ = xz
+
+# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
+# line interface, but use _GZIP, _BZIP2, _LZOP internally.
+_GZIP := $(GZIP)
+_BZIP2 := $(BZIP2)
+_LZOP := $(LZOP)
+
+# Reset GZIP, BZIP2, LZOP in this Makefile
+override GZIP=
+override BZIP2=
+override LZOP=
+
+# Reset GZIP, BZIP2, LZOP in recursive invocations
+MAKEOVERRIDES += GZIP= BZIP2= LZOP=
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -495,6 +515,7 @@ CLANG_FLAGS :=
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
+export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
@@ -1005,10 +1026,10 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = gzip -n -f
+ mod_compress_cmd = $(_GZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
- mod_compress_cmd = xz -f
+ mod_compress_cmd = $(XZ) -f
endif # CONFIG_MODULE_COMPRESS_XZ
endif # CONFIG_MODULE_COMPRESS
export mod_compress_cmd
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 40937248cebe..739f0464321e 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-gzip -9 >> "$XIPIMAGE.tmp"
+$_GZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index 32240000dc0c..f817f3d5e758 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 5d9288384096..ce6db5e5a5a3 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- gzip -9c vmlinux.tmp >vmlinux.gz
+ $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
rm vmlinux.tmp
else
- gzip -9c vmlinux >vmlinux.gz
+ $(_GZIP) -9c vmlinux >vmlinux.gz
endif
bzImage: vmlinux.bz2
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- bzip2 -1c vmlinux.tmp >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- bzip2 -1c vmlinux >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
index 78ce4cff1012..24d8dbde1109 100644
--- a/arch/mips/lasat/image/Makefile
+++ b/arch/mips/lasat/image/Makefile
@@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
$(LD) -r -o $@ -b binary $<
$(obj)/%.gz: $(obj)/%.bin
- gzip -cf -9 $< > $@
+ $(_GZIP) -cf -9 $< > $@
$(obj)/kImage.bin: $(KERNEL_IMAGE)
$(OBJCOPY) -O binary -S $^ $@
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 628cd8bb7ad8..e1aa514aeb36 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @gzip -cf -9 $< > $@
+ @$(_GZIP) -cf -9 $< > $@
endif
install:
--git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
index e13ca842eb7e..c1510f0ab3ea 100755
--- a/kernel/gen_kheaders.sh
+++ b/kernel/gen_kheaders.sh
@@ -88,7 +88,7 @@ find $cpio_dir -type f -print0 |
find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
--owner=0 --group=0 --numeric-owner --no-recursion \
- -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
+ -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
echo $headers_md5 > kernel/kheaders.md5
echo "$this_file_md5" >> kernel/kheaders.md5
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 4b799737722c..a1c1424e6b52 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
quiet_cmd_lzma = LZMA $@
- cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
+ cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
+ cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
@@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
$(size_append); } > $@
quiet_cmd_xzmisc = XZMISC $@
- cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
+ cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
# ASM offsets
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 02135d2671a6..b2b6153af63a 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
false; \
fi ; \
$(srctree)/scripts/setlocalversion --save-scmversion; \
-tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
+tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
rm -f $(objtree)/.scmversion
@@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),bzip2, \
-$(if $(findstring gz,$@),gzip, \
-$(if $(findstring xz,$@),xz, \
+$(if $(findstring bz2,$@),$(_BZIP2), \
+$(if $(findstring gz,$@),$(_GZIP), \
+$(if $(findstring xz,$@),$(XZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/package/buildtar b/scripts/package/buildtar
index 77c7caefede1..ad62c6879622 100755
--- a/scripts/package/buildtar
+++ b/scripts/package/buildtar
@@ -28,15 +28,15 @@ case "${1}" in
opts=
;;
targz-pkg)
- opts=--gzip
+ opts="-I ${_GZIP}"
tarball=${tarball}.gz
;;
tarbz2-pkg)
- opts=--bzip2
+ opts="-I ${_BZIP2}"
tarball=${tarball}.bz2
;;
tarxz-pkg)
- opts=--xz
+ opts="-I ${XZ}"
tarball=${tarball}.xz
;;
*)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index 7a2d372f4885..76e9cbcfbeab 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -20,4 +20,4 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
--
2.26.2
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [RFC PATCH v4] kbuild: add variables for compression tools
2020-06-03 9:20 ` [RFC PATCH v4] " Denis Efremov
@ 2020-06-04 0:12 ` Masahiro Yamada
0 siblings, 0 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-06-04 0:12 UTC (permalink / raw)
To: Denis Efremov; +Cc: Linux Kbuild mailing list, Linux Kernel Mailing List
On Wed, Jun 3, 2020 at 6:19 PM Denis Efremov <efremov@linux.com> wrote:
>
> Allow user to use alternative implementations of compression tools,
> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
> speed up the build:
> $ make GZIP=pigz BZIP2=pbzip2
>
> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
> since 2015. However, alternative implementations (e.g., pigz) still rely
> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>
> The credit goes to @grsecurity.
>
> As a sidenote, for multi-threaded lzma, xz compression one can use:
> $ export XZ_OPT="--threads=0"
>
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
Applied to linux-kbuild.
Thanks.
> Changes in v2:
> - _GZIP used instead of GZIP
> - tar commands altered to use tools from the vars
> Changes in v3:
> - _BZIP2 used instead of BZIP2
> - _LZOP used instead of LZOP
> Changes in v4:
> - Unexports removed from Kbuild.include
> - MAKEOVERRIDES used in top Makefile
> - All variables checks removed from scripts
>
> Makefile | 25 +++++++++++++++++++++++--
> arch/arm/boot/deflate_xip_data.sh | 2 +-
> arch/ia64/Makefile | 2 +-
> arch/m68k/Makefile | 8 ++++----
> arch/mips/lasat/image/Makefile | 2 +-
> arch/parisc/Makefile | 2 +-
> kernel/gen_kheaders.sh | 2 +-
> scripts/Makefile.lib | 12 ++++++------
> scripts/Makefile.package | 8 ++++----
> scripts/package/buildtar | 6 +++---
> scripts/xz_wrap.sh | 2 +-
> 11 files changed, 46 insertions(+), 25 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index b668725a2a62..9a9cce1ed83d 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -447,6 +447,26 @@ PYTHON = python
> PYTHON3 = python3
> CHECK = sparse
> BASH = bash
> +GZIP = gzip
> +BZIP2 = bzip2
> +LZOP = lzop
> +LZMA = lzma
> +LZ4 = lz4c
> +XZ = xz
> +
> +# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
> +# line interface, but use _GZIP, _BZIP2, _LZOP internally.
> +_GZIP := $(GZIP)
> +_BZIP2 := $(BZIP2)
> +_LZOP := $(LZOP)
> +
> +# Reset GZIP, BZIP2, LZOP in this Makefile
> +override GZIP=
> +override BZIP2=
> +override LZOP=
> +
> +# Reset GZIP, BZIP2, LZOP in recursive invocations
> +MAKEOVERRIDES += GZIP= BZIP2= LZOP=
>
> CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
> -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
> @@ -495,6 +515,7 @@ CLANG_FLAGS :=
> export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
> export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
> export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
> +export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
> export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
>
> export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
> @@ -1005,10 +1026,10 @@ export mod_strip_cmd
> mod_compress_cmd = true
> ifdef CONFIG_MODULE_COMPRESS
> ifdef CONFIG_MODULE_COMPRESS_GZIP
> - mod_compress_cmd = gzip -n -f
> + mod_compress_cmd = $(_GZIP) -n -f
> endif # CONFIG_MODULE_COMPRESS_GZIP
> ifdef CONFIG_MODULE_COMPRESS_XZ
> - mod_compress_cmd = xz -f
> + mod_compress_cmd = $(XZ) -f
> endif # CONFIG_MODULE_COMPRESS_XZ
> endif # CONFIG_MODULE_COMPRESS
> export mod_compress_cmd
> diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
> index 40937248cebe..739f0464321e 100755
> --- a/arch/arm/boot/deflate_xip_data.sh
> +++ b/arch/arm/boot/deflate_xip_data.sh
> @@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
> # substitute the data section by a compressed version
> $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
> $DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
> -gzip -9 >> "$XIPIMAGE.tmp"
> +$_GZIP -9 >> "$XIPIMAGE.tmp"
>
> # replace kernel binary
> mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
> diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
> index 32240000dc0c..f817f3d5e758 100644
> --- a/arch/ia64/Makefile
> +++ b/arch/ia64/Makefile
> @@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
> endif
>
> quiet_cmd_gzip = GZIP $@
> -cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> +cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
>
> quiet_cmd_objcopy = OBJCOPY $@
> cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
> index 5d9288384096..ce6db5e5a5a3 100644
> --- a/arch/m68k/Makefile
> +++ b/arch/m68k/Makefile
> @@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - gzip -9c vmlinux.tmp >vmlinux.gz
> + $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
> rm vmlinux.tmp
> else
> - gzip -9c vmlinux >vmlinux.gz
> + $(_GZIP) -9c vmlinux >vmlinux.gz
> endif
>
> bzImage: vmlinux.bz2
> @@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - bzip2 -1c vmlinux.tmp >vmlinux.bz2
> + $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
> rm vmlinux.tmp
> else
> - bzip2 -1c vmlinux >vmlinux.bz2
> + $(_BZIP2) -1c vmlinux >vmlinux.bz2
> endif
>
> archclean:
> diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile
> index 78ce4cff1012..24d8dbde1109 100644
> --- a/arch/mips/lasat/image/Makefile
> +++ b/arch/mips/lasat/image/Makefile
> @@ -44,7 +44,7 @@ $(obj)/%.o: $(obj)/%.gz
> $(LD) -r -o $@ -b binary $<
>
> $(obj)/%.gz: $(obj)/%.bin
> - gzip -cf -9 $< > $@
> + $(_GZIP) -cf -9 $< > $@
>
> $(obj)/kImage.bin: $(KERNEL_IMAGE)
> $(OBJCOPY) -O binary -S $^ $@
> diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
> index 628cd8bb7ad8..e1aa514aeb36 100644
> --- a/arch/parisc/Makefile
> +++ b/arch/parisc/Makefile
> @@ -162,7 +162,7 @@ vmlinuz: bzImage
> $(OBJCOPY) $(boot)/bzImage $@
> else
> vmlinuz: vmlinux
> - @gzip -cf -9 $< > $@
> + @$(_GZIP) -cf -9 $< > $@
> endif
>
> install:
> diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
> index e13ca842eb7e..c1510f0ab3ea 100755
> --- a/kernel/gen_kheaders.sh
> +++ b/kernel/gen_kheaders.sh
> @@ -88,7 +88,7 @@ find $cpio_dir -type f -print0 |
> find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
> tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
> --owner=0 --group=0 --numeric-owner --no-recursion \
> - -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
> + -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
>
> echo $headers_md5 > kernel/kheaders.md5
> echo "$this_file_md5" >> kernel/kheaders.md5
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index 4b799737722c..a1c1424e6b52 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> # ---------------------------------------------------------------------------
>
> quiet_cmd_gzip = GZIP $@
> - cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
> + cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
>
> # DTC
> # ---------------------------------------------------------------------------
> @@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
> )
>
> quiet_cmd_bzip2 = BZIP2 $@
> - cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
> + cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
>
> # Lzma
> # ---------------------------------------------------------------------------
>
> quiet_cmd_lzma = LZMA $@
> - cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
> + cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
>
> quiet_cmd_lzo = LZO $@
> - cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
> + cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
>
> quiet_cmd_lz4 = LZ4 $@
> - cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
> + cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
> $(size_append); } > $@
>
> # U-Boot mkimage
> @@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
> $(size_append); } > $@
>
> quiet_cmd_xzmisc = XZMISC $@
> - cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
> + cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
>
> # ASM offsets
> # ---------------------------------------------------------------------------
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index 02135d2671a6..b2b6153af63a 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
> false; \
> fi ; \
> $(srctree)/scripts/setlocalversion --save-scmversion; \
> -tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> +tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
> rm -f $(objtree)/.scmversion
>
> @@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
> tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
> rm -r $(perf-tar); \
> $(if $(findstring tar-src,$@),, \
> -$(if $(findstring bz2,$@),bzip2, \
> -$(if $(findstring gz,$@),gzip, \
> -$(if $(findstring xz,$@),xz, \
> +$(if $(findstring bz2,$@),$(_BZIP2), \
> +$(if $(findstring gz,$@),$(_GZIP), \
> +$(if $(findstring xz,$@),$(XZ), \
> $(error unknown target $@)))) \
> -f -9 $(perf-tar).tar)
>
> diff --git a/scripts/package/buildtar b/scripts/package/buildtar
> index 77c7caefede1..ad62c6879622 100755
> --- a/scripts/package/buildtar
> +++ b/scripts/package/buildtar
> @@ -28,15 +28,15 @@ case "${1}" in
> opts=
> ;;
> targz-pkg)
> - opts=--gzip
> + opts="-I ${_GZIP}"
> tarball=${tarball}.gz
> ;;
> tarbz2-pkg)
> - opts=--bzip2
> + opts="-I ${_BZIP2}"
> tarball=${tarball}.bz2
> ;;
> tarxz-pkg)
> - opts=--xz
> + opts="-I ${XZ}"
> tarball=${tarball}.xz
> ;;
> *)
> diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
> index 7a2d372f4885..76e9cbcfbeab 100755
> --- a/scripts/xz_wrap.sh
> +++ b/scripts/xz_wrap.sh
> @@ -20,4 +20,4 @@ case $SRCARCH in
> sparc) BCJ=--sparc ;;
> esac
>
> -exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
> +exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
> --
> 2.26.2
>
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
* [PATCH v5] kbuild: add variables for compression tools
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
` (3 preceding siblings ...)
2020-06-03 9:20 ` [RFC PATCH v4] " Denis Efremov
@ 2020-06-05 7:39 ` Denis Efremov
2020-06-06 14:43 ` Masahiro Yamada
2020-06-08 1:30 ` Guenter Roeck
4 siblings, 2 replies; 20+ messages in thread
From: Denis Efremov @ 2020-06-05 7:39 UTC (permalink / raw)
To: Masahiro Yamada
Cc: Denis Efremov, Stephen Rothwell, linux-kbuild, linux-kernel
Allow user to use alternative implementations of compression tools,
such as pigz, pbzip2, pxz. For example, multi-threaded tools to
speed up the build:
$ make GZIP=pigz BZIP2=pbzip2
Variables _GZIP, _BZIP2, _LZOP are used internally because original env
vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
since 2015. However, alternative implementations (e.g., pigz) still rely
on it. BZIP2, BZIP, LZOP vars are not obsolescent.
The credit goes to @grsecurity.
As a sidenote, for multi-threaded lzma, xz compression one can use:
$ export XZ_OPT="--threads=0"
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Changes in v2:
- _GZIP used instead of GZIP
- tar commands altered to use tools from the vars
Changes in v3:
- _BZIP2 used instead of BZIP2
- _LZOP used instead of LZOP
Changes in v4:
- Unexports removed from Kbuild.include
- MAKEOVERRIDES used in top Makefile
- All variables checks removed from scripts
Changes in v5:
- Conflict with removed LASAT resolved
Makefile | 25 +++++++++++++++++++++++--
arch/arm/boot/deflate_xip_data.sh | 2 +-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 8 ++++----
arch/parisc/Makefile | 2 +-
| 2 +-
scripts/Makefile.lib | 12 ++++++------
scripts/Makefile.package | 8 ++++----
scripts/package/buildtar | 6 +++---
scripts/xz_wrap.sh | 2 +-
10 files changed, 45 insertions(+), 24 deletions(-)
diff --git a/Makefile b/Makefile
index a7bc91cbac8f..7920404950c3 100644
--- a/Makefile
+++ b/Makefile
@@ -447,6 +447,26 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
+GZIP = gzip
+BZIP2 = bzip2
+LZOP = lzop
+LZMA = lzma
+LZ4 = lz4c
+XZ = xz
+
+# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
+# line interface, but use _GZIP, _BZIP2, _LZOP internally.
+_GZIP := $(GZIP)
+_BZIP2 := $(BZIP2)
+_LZOP := $(LZOP)
+
+# Reset GZIP, BZIP2, LZOP in this Makefile
+override GZIP=
+override BZIP2=
+override LZOP=
+
+# Reset GZIP, BZIP2, LZOP in recursive invocations
+MAKEOVERRIDES += GZIP= BZIP2= LZOP=
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -495,6 +515,7 @@ CLANG_FLAGS :=
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
+export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
@@ -1011,10 +1032,10 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = gzip -n -f
+ mod_compress_cmd = $(_GZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
- mod_compress_cmd = xz -f
+ mod_compress_cmd = $(XZ) -f
endif # CONFIG_MODULE_COMPRESS_XZ
endif # CONFIG_MODULE_COMPRESS
export mod_compress_cmd
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 40937248cebe..739f0464321e 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-gzip -9 >> "$XIPIMAGE.tmp"
+$_GZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index 32240000dc0c..f817f3d5e758 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index 5d9288384096..ce6db5e5a5a3 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- gzip -9c vmlinux.tmp >vmlinux.gz
+ $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
rm vmlinux.tmp
else
- gzip -9c vmlinux >vmlinux.gz
+ $(_GZIP) -9c vmlinux >vmlinux.gz
endif
bzImage: vmlinux.bz2
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- bzip2 -1c vmlinux.tmp >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- bzip2 -1c vmlinux >vmlinux.bz2
+ $(_BZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index fadbbd010337..182a5bca3e2c 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @gzip -cf -9 $< > $@
+ @$(_GZIP) -cf -9 $< > $@
endif
install:
--git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh
index e13ca842eb7e..c1510f0ab3ea 100755
--- a/kernel/gen_kheaders.sh
+++ b/kernel/gen_kheaders.sh
@@ -88,7 +88,7 @@ find $cpio_dir -type f -print0 |
find $cpio_dir -printf "./%P\n" | LC_ALL=C sort | \
tar "${KBUILD_BUILD_TIMESTAMP:+--mtime=$KBUILD_BUILD_TIMESTAMP}" \
--owner=0 --group=0 --numeric-owner --no-recursion \
- -Jcf $tarfile -C $cpio_dir/ -T - > /dev/null
+ -I $XZ -cf $tarfile -C $cpio_dir/ -T - > /dev/null
echo $headers_md5 > kernel/kheaders.md5
echo "$this_file_md5" >> kernel/kheaders.md5
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 4b799737722c..a1c1424e6b52 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -241,7 +241,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | gzip -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -334,19 +334,19 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | bzip2 -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
quiet_cmd_lzma = LZMA $@
- cmd_lzma = { cat $(real-prereqs) | lzma -9; $(size_append); } > $@
+ cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | lzop -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
- cmd_lz4 = { cat $(real-prereqs) | lz4c -l -c1 stdin stdout; \
+ cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
$(size_append); } > $@
# U-Boot mkimage
@@ -393,7 +393,7 @@ quiet_cmd_xzkern = XZKERN $@
$(size_append); } > $@
quiet_cmd_xzmisc = XZMISC $@
- cmd_xzmisc = cat $(real-prereqs) | xz --check=crc32 --lzma2=dict=1MiB > $@
+ cmd_xzmisc = cat $(real-prereqs) | $(XZ) --check=crc32 --lzma2=dict=1MiB > $@
# ASM offsets
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 02135d2671a6..b2b6153af63a 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
false; \
fi ; \
$(srctree)/scripts/setlocalversion --save-scmversion; \
-tar -cz $(RCS_TAR_IGNORE) -f $(2).tar.gz \
+tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
rm -f $(objtree)/.scmversion
@@ -127,9 +127,9 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),bzip2, \
-$(if $(findstring gz,$@),gzip, \
-$(if $(findstring xz,$@),xz, \
+$(if $(findstring bz2,$@),$(_BZIP2), \
+$(if $(findstring gz,$@),$(_GZIP), \
+$(if $(findstring xz,$@),$(XZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/package/buildtar b/scripts/package/buildtar
index 77c7caefede1..ad62c6879622 100755
--- a/scripts/package/buildtar
+++ b/scripts/package/buildtar
@@ -28,15 +28,15 @@ case "${1}" in
opts=
;;
targz-pkg)
- opts=--gzip
+ opts="-I ${_GZIP}"
tarball=${tarball}.gz
;;
tarbz2-pkg)
- opts=--bzip2
+ opts="-I ${_BZIP2}"
tarball=${tarball}.bz2
;;
tarxz-pkg)
- opts=--xz
+ opts="-I ${XZ}"
tarball=${tarball}.xz
;;
*)
diff --git a/scripts/xz_wrap.sh b/scripts/xz_wrap.sh
index 7a2d372f4885..76e9cbcfbeab 100755
--- a/scripts/xz_wrap.sh
+++ b/scripts/xz_wrap.sh
@@ -20,4 +20,4 @@ case $SRCARCH in
sparc) BCJ=--sparc ;;
esac
-exec xz --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
+exec $XZ --check=crc32 $BCJ --lzma2=$LZMA2OPTS,dict=32MiB
--
2.26.2
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [PATCH v5] kbuild: add variables for compression tools
2020-06-05 7:39 ` [PATCH v5] " Denis Efremov
@ 2020-06-06 14:43 ` Masahiro Yamada
2020-06-08 1:30 ` Guenter Roeck
1 sibling, 0 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-06-06 14:43 UTC (permalink / raw)
To: Denis Efremov
Cc: Stephen Rothwell, Linux Kbuild mailing list, Linux Kernel Mailing List
On Fri, Jun 5, 2020 at 4:40 PM Denis Efremov <efremov@linux.com> wrote:
>
> Allow user to use alternative implementations of compression tools,
> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
> speed up the build:
> $ make GZIP=pigz BZIP2=pbzip2
>
> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
> since 2015. However, alternative implementations (e.g., pigz) still rely
> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>
> The credit goes to @grsecurity.
>
> As a sidenote, for multi-threaded lzma, xz compression one can use:
> $ export XZ_OPT="--threads=0"
>
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
> Changes in v2:
> - _GZIP used instead of GZIP
> - tar commands altered to use tools from the vars
> Changes in v3:
> - _BZIP2 used instead of BZIP2
> - _LZOP used instead of LZOP
> Changes in v4:
> - Unexports removed from Kbuild.include
> - MAKEOVERRIDES used in top Makefile
> - All variables checks removed from scripts
> Changes in v5:
> - Conflict with removed LASAT resolved
Replaced. Thanks.
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v5] kbuild: add variables for compression tools
2020-06-05 7:39 ` [PATCH v5] " Denis Efremov
2020-06-06 14:43 ` Masahiro Yamada
@ 2020-06-08 1:30 ` Guenter Roeck
2020-06-08 4:59 ` Masahiro Yamada
1 sibling, 1 reply; 20+ messages in thread
From: Guenter Roeck @ 2020-06-08 1:30 UTC (permalink / raw)
To: Denis Efremov
Cc: Masahiro Yamada, Stephen Rothwell, linux-kbuild, linux-kernel
Hi,
On Fri, Jun 05, 2020 at 10:39:55AM +0300, Denis Efremov wrote:
> Allow user to use alternative implementations of compression tools,
> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
> speed up the build:
> $ make GZIP=pigz BZIP2=pbzip2
>
> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
> since 2015. However, alternative implementations (e.g., pigz) still rely
> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>
When building mips:defconfig, this patch results in:
Building mips:defconfig ... failed
--------------
Error log:
/bin/sh: -n: command not found
make[3]: *** [kernel/config_data.gz] Error 127
make[3]: *** Deleting file 'kernel/config_data.gz'
make[3]: *** Waiting for unfinished jobs....
make[2]: *** [kernel] Error 2
make[2]: *** Waiting for unfinished jobs....
make[1]: *** [autoksyms_recursive] Error 2
make: *** [__sub-make] Error 2
Reverting this patch fixes the problem. Bisect log is attached.
Guenter
---
# bad: [cf0c97f148e9e50aa5a7ddd1984a604dd2bde4af] Merge tag 'pinctrl-v5.8-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl
# good: [aaa2faab4ed8e5fe0111e04d6e168c028fe2987f] Merge tag 'for-linus-5.8-ofs1' of git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux
git bisect start 'HEAD' 'aaa2faab4ed8'
# good: [77f55d1305c11fb729b88f2c3f7881ba0831fa6f] staging: rtl8723bs: Use common packet header constants
git bisect good 77f55d1305c11fb729b88f2c3f7881ba0831fa6f
# bad: [e611c0fe318c6d6827ee2bba660fbc23cf73f7dc] Merge tag 'usb-5.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb
git bisect bad e611c0fe318c6d6827ee2bba660fbc23cf73f7dc
# bad: [cff11abeca78aa782378401ca2800bd2194aa14e] Merge tag 'kbuild-v5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild
git bisect bad cff11abeca78aa782378401ca2800bd2194aa14e
# good: [2bd81cd04a3f5eb873cc81fa16c469377be3b092] Merge branch 'remotes/lorenzo/pci/vmd'
git bisect good 2bd81cd04a3f5eb873cc81fa16c469377be3b092
# good: [269a535ca931b754a40dda3ab60514e68773c759] modpost: generate vmlinux.symvers and reuse it for the second modpost
git bisect good 269a535ca931b754a40dda3ab60514e68773c759
# good: [e542e0dc3ee3eafc46dd8e3073388079d69cace0] Merge branch 'dmi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging
git bisect good e542e0dc3ee3eafc46dd8e3073388079d69cace0
# good: [4de7b62936122570408357417f21072e78292926] modpost: remove is_vmlinux() helper
git bisect good 4de7b62936122570408357417f21072e78292926
# good: [1ee18de92927f37e6948d5a6fc73cbf89f806905] Merge tag 'dma-mapping-5.8' of git://git.infradead.org/users/hch/dma-mapping
git bisect good 1ee18de92927f37e6948d5a6fc73cbf89f806905
# bad: [8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294] kbuild: add variables for compression tools
git bisect bad 8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294
# good: [c0901577e1dcc8d1c0fd1a11c8d571f650df845f] kbuild: doc: rename LDFLAGS to KBUILD_LDFLAGS
git bisect good c0901577e1dcc8d1c0fd1a11c8d571f650df845f
# good: [e0b250b57dcf403529081e5898a9de717f96b76b] Makefile: install modules.builtin even if CONFIG_MODULES=n
git bisect good e0b250b57dcf403529081e5898a9de717f96b76b
# first bad commit: [8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294] kbuild: add variables for compression tools
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v5] kbuild: add variables for compression tools
2020-06-08 1:30 ` Guenter Roeck
@ 2020-06-08 4:59 ` Masahiro Yamada
2020-06-08 9:59 ` [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables Denis Efremov
2020-06-08 10:28 ` [PATCH v5] kbuild: add variables for compression tools Denis Efremov
0 siblings, 2 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-06-08 4:59 UTC (permalink / raw)
To: Guenter Roeck
Cc: Denis Efremov, Stephen Rothwell, Linux Kbuild mailing list,
Linux Kernel Mailing List
On Mon, Jun 8, 2020 at 10:30 AM Guenter Roeck <linux@roeck-us.net> wrote:
>
> Hi,
>
> On Fri, Jun 05, 2020 at 10:39:55AM +0300, Denis Efremov wrote:
> > Allow user to use alternative implementations of compression tools,
> > such as pigz, pbzip2, pxz. For example, multi-threaded tools to
> > speed up the build:
> > $ make GZIP=pigz BZIP2=pbzip2
> >
> > Variables _GZIP, _BZIP2, _LZOP are used internally because original env
> > vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
> > since 2015. However, alternative implementations (e.g., pigz) still rely
> > on it. BZIP2, BZIP, LZOP vars are not obsolescent.
> >
>
> When building mips:defconfig, this patch results in:
>
> Building mips:defconfig ... failed
> --------------
> Error log:
> /bin/sh: -n: command not found
> make[3]: *** [kernel/config_data.gz] Error 127
> make[3]: *** Deleting file 'kernel/config_data.gz'
> make[3]: *** Waiting for unfinished jobs....
> make[2]: *** [kernel] Error 2
> make[2]: *** Waiting for unfinished jobs....
> make[1]: *** [autoksyms_recursive] Error 2
> make: *** [__sub-make] Error 2
>
> Reverting this patch fixes the problem. Bisect log is attached.
>
> Guenter
Agh, this is because of CONFIG_TRIM_UNUSED_KSYMS.
Also, the distro package builds are broken
e.g. make GZIP=gzip bindeb-pkg
Denis,
I think we should go back to the original
KGZIP, KBZIP2, KLZOP.
>
> ---
> # bad: [cf0c97f148e9e50aa5a7ddd1984a604dd2bde4af] Merge tag 'pinctrl-v5.8-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl
> # good: [aaa2faab4ed8e5fe0111e04d6e168c028fe2987f] Merge tag 'for-linus-5.8-ofs1' of git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux
> git bisect start 'HEAD' 'aaa2faab4ed8'
> # good: [77f55d1305c11fb729b88f2c3f7881ba0831fa6f] staging: rtl8723bs: Use common packet header constants
> git bisect good 77f55d1305c11fb729b88f2c3f7881ba0831fa6f
> # bad: [e611c0fe318c6d6827ee2bba660fbc23cf73f7dc] Merge tag 'usb-5.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb
> git bisect bad e611c0fe318c6d6827ee2bba660fbc23cf73f7dc
> # bad: [cff11abeca78aa782378401ca2800bd2194aa14e] Merge tag 'kbuild-v5.8' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild
> git bisect bad cff11abeca78aa782378401ca2800bd2194aa14e
> # good: [2bd81cd04a3f5eb873cc81fa16c469377be3b092] Merge branch 'remotes/lorenzo/pci/vmd'
> git bisect good 2bd81cd04a3f5eb873cc81fa16c469377be3b092
> # good: [269a535ca931b754a40dda3ab60514e68773c759] modpost: generate vmlinux.symvers and reuse it for the second modpost
> git bisect good 269a535ca931b754a40dda3ab60514e68773c759
> # good: [e542e0dc3ee3eafc46dd8e3073388079d69cace0] Merge branch 'dmi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging
> git bisect good e542e0dc3ee3eafc46dd8e3073388079d69cace0
> # good: [4de7b62936122570408357417f21072e78292926] modpost: remove is_vmlinux() helper
> git bisect good 4de7b62936122570408357417f21072e78292926
> # good: [1ee18de92927f37e6948d5a6fc73cbf89f806905] Merge tag 'dma-mapping-5.8' of git://git.infradead.org/users/hch/dma-mapping
> git bisect good 1ee18de92927f37e6948d5a6fc73cbf89f806905
> # bad: [8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294] kbuild: add variables for compression tools
> git bisect bad 8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294
> # good: [c0901577e1dcc8d1c0fd1a11c8d571f650df845f] kbuild: doc: rename LDFLAGS to KBUILD_LDFLAGS
> git bisect good c0901577e1dcc8d1c0fd1a11c8d571f650df845f
> # good: [e0b250b57dcf403529081e5898a9de717f96b76b] Makefile: install modules.builtin even if CONFIG_MODULES=n
> git bisect good e0b250b57dcf403529081e5898a9de717f96b76b
> # first bad commit: [8dfb61dcbaceb19a5ded5e9c9dcf8d05acc32294] kbuild: add variables for compression tools
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
* [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables
2020-06-08 4:59 ` Masahiro Yamada
@ 2020-06-08 9:59 ` Denis Efremov
2020-06-08 15:36 ` Adam Borowski
2020-06-09 1:03 ` Masahiro Yamada
2020-06-08 10:28 ` [PATCH v5] kbuild: add variables for compression tools Denis Efremov
1 sibling, 2 replies; 20+ messages in thread
From: Denis Efremov @ 2020-06-08 9:59 UTC (permalink / raw)
To: Masahiro Yamada, Guenter Roeck
Cc: Denis Efremov, Linux Kernel Mailing List, Linux Kbuild mailing list
Redefine GZIP, BZIP2, LZOP variables as KGZIP, KBZIP2, KLZOP resp.
GZIP, BZIP2, LZOP env variables are reserved by the tools. The original
attempt to redefine them internally doesn't work in makefiles/scripts
intercall scenarios, e.g., "make GZIP=gzip bindeb-pkg" and results in
broken builds. There can be other broken build commands because of this,
so the universal solution is to use non-reserved env variables for the
compression tools.
Fixes: 8dfb61dcbace ("kbuild: add variables for compression tools")
Signed-off-by: Denis Efremov <efremov@linux.com>
---
Makefile | 24 +++++-------------------
arch/arm/boot/deflate_xip_data.sh | 2 +-
arch/ia64/Makefile | 2 +-
arch/m68k/Makefile | 8 ++++----
arch/parisc/Makefile | 2 +-
scripts/Makefile.lib | 6 +++---
scripts/Makefile.package | 6 +++---
scripts/package/buildtar | 4 ++--
8 files changed, 20 insertions(+), 34 deletions(-)
diff --git a/Makefile b/Makefile
index 839f9fee22cb..e43d193bb3b2 100644
--- a/Makefile
+++ b/Makefile
@@ -458,27 +458,13 @@ PYTHON = python
PYTHON3 = python3
CHECK = sparse
BASH = bash
-GZIP = gzip
-BZIP2 = bzip2
-LZOP = lzop
+KGZIP = gzip
+KBZIP2 = bzip2
+KLZOP = lzop
LZMA = lzma
LZ4 = lz4c
XZ = xz
-# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
-# line interface, but use _GZIP, _BZIP2, _LZOP internally.
-_GZIP := $(GZIP)
-_BZIP2 := $(BZIP2)
-_LZOP := $(LZOP)
-
-# Reset GZIP, BZIP2, LZOP in this Makefile
-override GZIP=
-override BZIP2=
-override LZOP=
-
-# Reset GZIP, BZIP2, LZOP in recursive invocations
-MAKEOVERRIDES += GZIP= BZIP2= LZOP=
-
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
NOSTDINC_FLAGS :=
@@ -526,7 +512,7 @@ CLANG_FLAGS :=
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
-export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
+export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
@@ -1047,7 +1033,7 @@ export mod_strip_cmd
mod_compress_cmd = true
ifdef CONFIG_MODULE_COMPRESS
ifdef CONFIG_MODULE_COMPRESS_GZIP
- mod_compress_cmd = $(_GZIP) -n -f
+ mod_compress_cmd = $(KGZIP) -n -f
endif # CONFIG_MODULE_COMPRESS_GZIP
ifdef CONFIG_MODULE_COMPRESS_XZ
mod_compress_cmd = $(XZ) -f
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 739f0464321e..304495c3c2c5 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
# substitute the data section by a compressed version
$DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
$DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
-$_GZIP -9 >> "$XIPIMAGE.tmp"
+$KGZIP -9 >> "$XIPIMAGE.tmp"
# replace kernel binary
mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index f817f3d5e758..2876a7df1b0a 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
endif
quiet_cmd_gzip = GZIP $@
-cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
+cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
quiet_cmd_objcopy = OBJCOPY $@
cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index ce6db5e5a5a3..0415d28dbe4f 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
+ $(KGZIP) -9c vmlinux.tmp >vmlinux.gz
rm vmlinux.tmp
else
- $(_GZIP) -9c vmlinux >vmlinux.gz
+ $(KGZIP) -9c vmlinux >vmlinux.gz
endif
bzImage: vmlinux.bz2
@@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
ifndef CONFIG_KGDB
cp vmlinux vmlinux.tmp
$(STRIP) vmlinux.tmp
- $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
+ $(KBZIP2) -1c vmlinux.tmp >vmlinux.bz2
rm vmlinux.tmp
else
- $(_BZIP2) -1c vmlinux >vmlinux.bz2
+ $(KBZIP2) -1c vmlinux >vmlinux.bz2
endif
archclean:
diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
index 182a5bca3e2c..5140c602207f 100644
--- a/arch/parisc/Makefile
+++ b/arch/parisc/Makefile
@@ -162,7 +162,7 @@ vmlinuz: bzImage
$(OBJCOPY) $(boot)/bzImage $@
else
vmlinuz: vmlinux
- @$(_GZIP) -cf -9 $< > $@
+ @$(KGZIP) -cf -9 $< > $@
endif
install:
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 127f2a7e3ced..94eeddb2e599 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -244,7 +244,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
# ---------------------------------------------------------------------------
quiet_cmd_gzip = GZIP $@
- cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
+ cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
# DTC
# ---------------------------------------------------------------------------
@@ -337,7 +337,7 @@ printf "%08x\n" $$dec_size | \
)
quiet_cmd_bzip2 = BZIP2 $@
- cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
+ cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@
# Lzma
# ---------------------------------------------------------------------------
@@ -346,7 +346,7 @@ quiet_cmd_lzma = LZMA $@
cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
quiet_cmd_lzo = LZO $@
- cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
+ cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
quiet_cmd_lz4 = LZ4 $@
cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index b2b6153af63a..f952fb64789d 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
false; \
fi ; \
$(srctree)/scripts/setlocalversion --save-scmversion; \
-tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
+tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
rm -f $(objtree)/.scmversion
@@ -127,8 +127,8 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
rm -r $(perf-tar); \
$(if $(findstring tar-src,$@),, \
-$(if $(findstring bz2,$@),$(_BZIP2), \
-$(if $(findstring gz,$@),$(_GZIP), \
+$(if $(findstring bz2,$@),$(KBZIP2), \
+$(if $(findstring gz,$@),$(KGZIP), \
$(if $(findstring xz,$@),$(XZ), \
$(error unknown target $@)))) \
-f -9 $(perf-tar).tar)
diff --git a/scripts/package/buildtar b/scripts/package/buildtar
index ad62c6879622..fb1578e72ab9 100755
--- a/scripts/package/buildtar
+++ b/scripts/package/buildtar
@@ -28,11 +28,11 @@ case "${1}" in
opts=
;;
targz-pkg)
- opts="-I ${_GZIP}"
+ opts="-I ${KGZIP}"
tarball=${tarball}.gz
;;
tarbz2-pkg)
- opts="-I ${_BZIP2}"
+ opts="-I ${KBZIP2}"
tarball=${tarball}.bz2
;;
tarxz-pkg)
--
2.26.2
^ permalink raw reply related [flat|nested] 20+ messages in thread
* Re: [PATCH v5] kbuild: add variables for compression tools
2020-06-08 4:59 ` Masahiro Yamada
2020-06-08 9:59 ` [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables Denis Efremov
@ 2020-06-08 10:28 ` Denis Efremov
2020-06-08 13:52 ` Guenter Roeck
1 sibling, 1 reply; 20+ messages in thread
From: Denis Efremov @ 2020-06-08 10:28 UTC (permalink / raw)
To: Masahiro Yamada, Guenter Roeck
Cc: Stephen Rothwell, Linux Kbuild mailing list, Linux Kernel Mailing List
On 6/8/20 7:59 AM, Masahiro Yamada wrote:
> On Mon, Jun 8, 2020 at 10:30 AM Guenter Roeck <linux@roeck-us.net> wrote:
>>
>> Hi,
>>
>> On Fri, Jun 05, 2020 at 10:39:55AM +0300, Denis Efremov wrote:
>>> Allow user to use alternative implementations of compression tools,
>>> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
>>> speed up the build:
>>> $ make GZIP=pigz BZIP2=pbzip2
>>>
>>> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
>>> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
>>> since 2015. However, alternative implementations (e.g., pigz) still rely
>>> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>>>
>>
>> When building mips:defconfig, this patch results in:
>>
>> Building mips:defconfig ... failed
>> --------------
>> Error log:
>> /bin/sh: -n: command not found
>> make[3]: *** [kernel/config_data.gz] Error 127
>> make[3]: *** Deleting file 'kernel/config_data.gz'
>> make[3]: *** Waiting for unfinished jobs....
>> make[2]: *** [kernel] Error 2
>> make[2]: *** Waiting for unfinished jobs....
>> make[1]: *** [autoksyms_recursive] Error 2
>> make: *** [__sub-make] Error 2
>>
>> Reverting this patch fixes the problem. Bisect log is attached.
>>
>> Guenter
>
I tried to reproduce it with cross-compilation on Fedora32.
$ export ARCH=mips
$ export CROSS_COMPILE=mips64-linux-gnu-
$ make defconfig
$ make -j12
And the kernel builds successfully. Could you please provide details about your
compilation steps and environment, esp. what "env | grep ZIP" shows,
"gzip --version", "sh --version", "bash --version"? This will be very helpful.
Additionally:
$ make GZIP=gzip -j12 # works
$ make GZIP=pigz -j12 # works
$ make GZIP=nosuchcommand -j12 # fails, as expected
>
> Agh, this is because of CONFIG_TRIM_UNUSED_KSYMS.
>
Hmm, it somehow works on my machine. But yes, this call looks like a problem
for these env vars:
autoksyms_recursive: descend modules.order
$(Q)$(CONFIG_SHELL) $(srctree)/scripts/adjust_autoksyms.sh \
"$(MAKE) -f $(srctree)/Makefile vmlinux"
> Also, the distro package builds are broken
> e.g. make GZIP=gzip bindeb-pkg
>
Yes, thanks.
> Denis,
>
> I think we should go back to the original
> KGZIP, KBZIP2, KLZOP.
>
Given that the original patch is already in the Linus tree,
I've sent a hotfix. Commit message is not perfect, because I
didn't have enough time to deeply debug it. I just hope that
the original patch didn't broke too many builds. Maybe later
I will try to prepare a patch with GZIP again when I will fully
debug these corner cases.
Thanks,
Denis
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH v5] kbuild: add variables for compression tools
2020-06-08 10:28 ` [PATCH v5] kbuild: add variables for compression tools Denis Efremov
@ 2020-06-08 13:52 ` Guenter Roeck
0 siblings, 0 replies; 20+ messages in thread
From: Guenter Roeck @ 2020-06-08 13:52 UTC (permalink / raw)
To: Denis Efremov, Masahiro Yamada
Cc: Stephen Rothwell, Linux Kbuild mailing list, Linux Kernel Mailing List
On 6/8/20 3:28 AM, Denis Efremov wrote:
>
>
> On 6/8/20 7:59 AM, Masahiro Yamada wrote:
>> On Mon, Jun 8, 2020 at 10:30 AM Guenter Roeck <linux@roeck-us.net> wrote:
>>>
>>> Hi,
>>>
>>> On Fri, Jun 05, 2020 at 10:39:55AM +0300, Denis Efremov wrote:
>>>> Allow user to use alternative implementations of compression tools,
>>>> such as pigz, pbzip2, pxz. For example, multi-threaded tools to
>>>> speed up the build:
>>>> $ make GZIP=pigz BZIP2=pbzip2
>>>>
>>>> Variables _GZIP, _BZIP2, _LZOP are used internally because original env
>>>> vars are reserved by the tools. The use of GZIP in gzip tool is obsolete
>>>> since 2015. However, alternative implementations (e.g., pigz) still rely
>>>> on it. BZIP2, BZIP, LZOP vars are not obsolescent.
>>>>
>>>
>>> When building mips:defconfig, this patch results in:
>>>
>>> Building mips:defconfig ... failed
>>> --------------
>>> Error log:
>>> /bin/sh: -n: command not found
>>> make[3]: *** [kernel/config_data.gz] Error 127
>>> make[3]: *** Deleting file 'kernel/config_data.gz'
>>> make[3]: *** Waiting for unfinished jobs....
>>> make[2]: *** [kernel] Error 2
>>> make[2]: *** Waiting for unfinished jobs....
>>> make[1]: *** [autoksyms_recursive] Error 2
>>> make: *** [__sub-make] Error 2
>>>
>>> Reverting this patch fixes the problem. Bisect log is attached.
>>>
>>> Guenter
>>
>
> I tried to reproduce it with cross-compilation on Fedora32.
> $ export ARCH=mips
> $ export CROSS_COMPILE=mips64-linux-gnu-
> $ make defconfig
> $ make -j12
>
> And the kernel builds successfully. Could you please provide details about your
> compilation steps and environment, esp. what "env | grep ZIP" shows,
> "gzip --version", "sh --version", "bash --version"? This will be very helpful.
>
> Additionally:
> $ make GZIP=gzip -j12 # works
> $ make GZIP=pigz -j12 # works
> $ make GZIP=nosuchcommand -j12 # fails, as expected
>
I use
make -j30 ARCH=mips CROSS_COMPILE=mips64-linux- ...
ie I don't use environment variables. Using environment variables indeed makes
the problem "disappear". Also, it is important to run "make mrproper" first
to ensure that the generated file does not already exist.
make -j30 ARCH=mips CROSS_COMPILE=mips64-linux- GZIP=gzip ...
does not work for me.
Guenter
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables
2020-06-08 9:59 ` [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables Denis Efremov
@ 2020-06-08 15:36 ` Adam Borowski
2020-06-08 16:23 ` Denis Efremov
2020-06-09 1:03 ` Masahiro Yamada
1 sibling, 1 reply; 20+ messages in thread
From: Adam Borowski @ 2020-06-08 15:36 UTC (permalink / raw)
To: Denis Efremov
Cc: Masahiro Yamada, Guenter Roeck, Linux Kernel Mailing List,
Linux Kbuild mailing list
On Mon, Jun 08, 2020 at 12:59:44PM +0300, Denis Efremov wrote:
> Redefine GZIP, BZIP2, LZOP variables as KGZIP, KBZIP2, KLZOP resp.
> GZIP, BZIP2, LZOP env variables are reserved by the tools. The original
> attempt to redefine them internally doesn't work in makefiles/scripts
> intercall scenarios, e.g., "make GZIP=gzip bindeb-pkg" and results in
> broken builds. There can be other broken build commands because of this,
> so the universal solution is to use non-reserved env variables for the
> compression tools.
>
> Fixes: 8dfb61dcbace ("kbuild: add variables for compression tools")
Same said my bisect before I noticed your fix. :)
> Signed-off-by: Denis Efremov <efremov@linux.com>
However, I run just basic "make bindeb-pkg" without forcing any variables,
thus the commit message is wrong.
Yet, your patch fixes the functionality. Thanks!
> ---
> Makefile | 24 +++++-------------------
> arch/arm/boot/deflate_xip_data.sh | 2 +-
> arch/ia64/Makefile | 2 +-
> arch/m68k/Makefile | 8 ++++----
> arch/parisc/Makefile | 2 +-
> scripts/Makefile.lib | 6 +++---
> scripts/Makefile.package | 6 +++---
> scripts/package/buildtar | 4 ++--
> 8 files changed, 20 insertions(+), 34 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index 839f9fee22cb..e43d193bb3b2 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -458,27 +458,13 @@ PYTHON = python
> PYTHON3 = python3
> CHECK = sparse
> BASH = bash
> -GZIP = gzip
> -BZIP2 = bzip2
> -LZOP = lzop
> +KGZIP = gzip
> +KBZIP2 = bzip2
> +KLZOP = lzop
> LZMA = lzma
> LZ4 = lz4c
> XZ = xz
>
> -# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
> -# line interface, but use _GZIP, _BZIP2, _LZOP internally.
> -_GZIP := $(GZIP)
> -_BZIP2 := $(BZIP2)
> -_LZOP := $(LZOP)
> -
> -# Reset GZIP, BZIP2, LZOP in this Makefile
> -override GZIP=
> -override BZIP2=
> -override LZOP=
> -
> -# Reset GZIP, BZIP2, LZOP in recursive invocations
> -MAKEOVERRIDES += GZIP= BZIP2= LZOP=
> -
> CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
> -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
> NOSTDINC_FLAGS :=
> @@ -526,7 +512,7 @@ CLANG_FLAGS :=
> export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
> export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
> export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
> -export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
> +export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
> export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
>
> export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
> @@ -1047,7 +1033,7 @@ export mod_strip_cmd
> mod_compress_cmd = true
> ifdef CONFIG_MODULE_COMPRESS
> ifdef CONFIG_MODULE_COMPRESS_GZIP
> - mod_compress_cmd = $(_GZIP) -n -f
> + mod_compress_cmd = $(KGZIP) -n -f
> endif # CONFIG_MODULE_COMPRESS_GZIP
> ifdef CONFIG_MODULE_COMPRESS_XZ
> mod_compress_cmd = $(XZ) -f
> diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
> index 739f0464321e..304495c3c2c5 100755
> --- a/arch/arm/boot/deflate_xip_data.sh
> +++ b/arch/arm/boot/deflate_xip_data.sh
> @@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
> # substitute the data section by a compressed version
> $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
> $DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
> -$_GZIP -9 >> "$XIPIMAGE.tmp"
> +$KGZIP -9 >> "$XIPIMAGE.tmp"
>
> # replace kernel binary
> mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
> diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
> index f817f3d5e758..2876a7df1b0a 100644
> --- a/arch/ia64/Makefile
> +++ b/arch/ia64/Makefile
> @@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
> endif
>
> quiet_cmd_gzip = GZIP $@
> -cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
> +cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> quiet_cmd_objcopy = OBJCOPY $@
> cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
> index ce6db5e5a5a3..0415d28dbe4f 100644
> --- a/arch/m68k/Makefile
> +++ b/arch/m68k/Makefile
> @@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
> + $(KGZIP) -9c vmlinux.tmp >vmlinux.gz
> rm vmlinux.tmp
> else
> - $(_GZIP) -9c vmlinux >vmlinux.gz
> + $(KGZIP) -9c vmlinux >vmlinux.gz
> endif
>
> bzImage: vmlinux.bz2
> @@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux.tmp >vmlinux.bz2
> rm vmlinux.tmp
> else
> - $(_BZIP2) -1c vmlinux >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux >vmlinux.bz2
> endif
>
> archclean:
> diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
> index 182a5bca3e2c..5140c602207f 100644
> --- a/arch/parisc/Makefile
> +++ b/arch/parisc/Makefile
> @@ -162,7 +162,7 @@ vmlinuz: bzImage
> $(OBJCOPY) $(boot)/bzImage $@
> else
> vmlinuz: vmlinux
> - @$(_GZIP) -cf -9 $< > $@
> + @$(KGZIP) -cf -9 $< > $@
> endif
>
> install:
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index 127f2a7e3ced..94eeddb2e599 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -244,7 +244,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> # ---------------------------------------------------------------------------
>
> quiet_cmd_gzip = GZIP $@
> - cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
> + cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> # DTC
> # ---------------------------------------------------------------------------
> @@ -337,7 +337,7 @@ printf "%08x\n" $$dec_size | \
> )
>
> quiet_cmd_bzip2 = BZIP2 $@
> - cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
> + cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@
>
> # Lzma
> # ---------------------------------------------------------------------------
> @@ -346,7 +346,7 @@ quiet_cmd_lzma = LZMA $@
> cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
>
> quiet_cmd_lzo = LZO $@
> - cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
> + cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
>
> quiet_cmd_lz4 = LZ4 $@
> cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index b2b6153af63a..f952fb64789d 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
> false; \
> fi ; \
> $(srctree)/scripts/setlocalversion --save-scmversion; \
> -tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> +tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
> rm -f $(objtree)/.scmversion
>
> @@ -127,8 +127,8 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
> tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
> rm -r $(perf-tar); \
> $(if $(findstring tar-src,$@),, \
> -$(if $(findstring bz2,$@),$(_BZIP2), \
> -$(if $(findstring gz,$@),$(_GZIP), \
> +$(if $(findstring bz2,$@),$(KBZIP2), \
> +$(if $(findstring gz,$@),$(KGZIP), \
> $(if $(findstring xz,$@),$(XZ), \
> $(error unknown target $@)))) \
> -f -9 $(perf-tar).tar)
> diff --git a/scripts/package/buildtar b/scripts/package/buildtar
> index ad62c6879622..fb1578e72ab9 100755
> --- a/scripts/package/buildtar
> +++ b/scripts/package/buildtar
> @@ -28,11 +28,11 @@ case "${1}" in
> opts=
> ;;
> targz-pkg)
> - opts="-I ${_GZIP}"
> + opts="-I ${KGZIP}"
> tarball=${tarball}.gz
> ;;
> tarbz2-pkg)
> - opts="-I ${_BZIP2}"
> + opts="-I ${KBZIP2}"
> tarball=${tarball}.bz2
> ;;
> tarxz-pkg)
> --
> 2.26.2
>
>
--
⢀⣴⠾⠻⢶⣦⠀
⣾⠁⢠⠒⠀⣿⡁ in the beginning was the boot and root floppies and they were good.
⢿⡄⠘⠷⠚⠋⠀ -- <willmore> on #linux-sunxi
⠈⠳⣄⠀⠀⠀⠀
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables
2020-06-08 15:36 ` Adam Borowski
@ 2020-06-08 16:23 ` Denis Efremov
0 siblings, 0 replies; 20+ messages in thread
From: Denis Efremov @ 2020-06-08 16:23 UTC (permalink / raw)
To: Adam Borowski
Cc: Masahiro Yamada, Guenter Roeck, Linux Kernel Mailing List,
Linux Kbuild mailing list
On 6/8/20 6:36 PM, Adam Borowski wrote:
> On Mon, Jun 08, 2020 at 12:59:44PM +0300, Denis Efremov wrote:
>> Redefine GZIP, BZIP2, LZOP variables as KGZIP, KBZIP2, KLZOP resp.
>> GZIP, BZIP2, LZOP env variables are reserved by the tools. The original
>> attempt to redefine them internally doesn't work in makefiles/scripts
>> intercall scenarios, e.g., "make GZIP=gzip bindeb-pkg" and results in
>> broken builds. There can be other broken build commands because of this,
>> so the universal solution is to use non-reserved env variables for the
>> compression tools.
>>
>> Fixes: 8dfb61dcbace ("kbuild: add variables for compression tools")
>
> Same said my bisect before I noticed your fix. :)
>
>> Signed-off-by: Denis Efremov <efremov@linux.com>
>
> However, I run just basic "make bindeb-pkg" without forcing any variables,
> thus the commit message is wrong.
I would not say it's fully wrong. At least in my case "make bindeb-pkg" builds
successfully (Fedora32/Debian10). I just added to the commit's message the
command "make GZIP=gzip bindeb-pkg" as an example that I think will "reliably"
break the build in any environment.
Thanks,
Denis
^ permalink raw reply [flat|nested] 20+ messages in thread
* Re: [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables
2020-06-08 9:59 ` [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables Denis Efremov
2020-06-08 15:36 ` Adam Borowski
@ 2020-06-09 1:03 ` Masahiro Yamada
1 sibling, 0 replies; 20+ messages in thread
From: Masahiro Yamada @ 2020-06-09 1:03 UTC (permalink / raw)
To: Denis Efremov
Cc: Guenter Roeck, Linux Kernel Mailing List, Linux Kbuild mailing list
On Mon, Jun 8, 2020 at 7:00 PM Denis Efremov <efremov@linux.com> wrote:
>
> Redefine GZIP, BZIP2, LZOP variables as KGZIP, KBZIP2, KLZOP resp.
> GZIP, BZIP2, LZOP env variables are reserved by the tools. The original
> attempt to redefine them internally doesn't work in makefiles/scripts
> intercall scenarios, e.g., "make GZIP=gzip bindeb-pkg" and results in
> broken builds. There can be other broken build commands because of this,
> so the universal solution is to use non-reserved env variables for the
> compression tools.
>
> Fixes: 8dfb61dcbace ("kbuild: add variables for compression tools")
> Signed-off-by: Denis Efremov <efremov@linux.com>
> ---
Applied to linux-kbuild.
Thanks.
> Makefile | 24 +++++-------------------
> arch/arm/boot/deflate_xip_data.sh | 2 +-
> arch/ia64/Makefile | 2 +-
> arch/m68k/Makefile | 8 ++++----
> arch/parisc/Makefile | 2 +-
> scripts/Makefile.lib | 6 +++---
> scripts/Makefile.package | 6 +++---
> scripts/package/buildtar | 4 ++--
> 8 files changed, 20 insertions(+), 34 deletions(-)
>
> diff --git a/Makefile b/Makefile
> index 839f9fee22cb..e43d193bb3b2 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -458,27 +458,13 @@ PYTHON = python
> PYTHON3 = python3
> CHECK = sparse
> BASH = bash
> -GZIP = gzip
> -BZIP2 = bzip2
> -LZOP = lzop
> +KGZIP = gzip
> +KBZIP2 = bzip2
> +KLZOP = lzop
> LZMA = lzma
> LZ4 = lz4c
> XZ = xz
>
> -# GZIP, BZIP2, LZOP env vars are used by the tools. Support them as the command
> -# line interface, but use _GZIP, _BZIP2, _LZOP internally.
> -_GZIP := $(GZIP)
> -_BZIP2 := $(BZIP2)
> -_LZOP := $(LZOP)
> -
> -# Reset GZIP, BZIP2, LZOP in this Makefile
> -override GZIP=
> -override BZIP2=
> -override LZOP=
> -
> -# Reset GZIP, BZIP2, LZOP in recursive invocations
> -MAKEOVERRIDES += GZIP= BZIP2= LZOP=
> -
> CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
> -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
> NOSTDINC_FLAGS :=
> @@ -526,7 +512,7 @@ CLANG_FLAGS :=
> export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
> export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
> export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
> -export _GZIP _BZIP2 _LZOP LZMA LZ4 XZ
> +export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
> export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
>
> export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
> @@ -1047,7 +1033,7 @@ export mod_strip_cmd
> mod_compress_cmd = true
> ifdef CONFIG_MODULE_COMPRESS
> ifdef CONFIG_MODULE_COMPRESS_GZIP
> - mod_compress_cmd = $(_GZIP) -n -f
> + mod_compress_cmd = $(KGZIP) -n -f
> endif # CONFIG_MODULE_COMPRESS_GZIP
> ifdef CONFIG_MODULE_COMPRESS_XZ
> mod_compress_cmd = $(XZ) -f
> diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
> index 739f0464321e..304495c3c2c5 100755
> --- a/arch/arm/boot/deflate_xip_data.sh
> +++ b/arch/arm/boot/deflate_xip_data.sh
> @@ -56,7 +56,7 @@ trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
> # substitute the data section by a compressed version
> $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
> $DD if="$XIPIMAGE" skip=$data_start iflag=skip_bytes |
> -$_GZIP -9 >> "$XIPIMAGE.tmp"
> +$KGZIP -9 >> "$XIPIMAGE.tmp"
>
> # replace kernel binary
> mv -f "$XIPIMAGE.tmp" "$XIPIMAGE"
> diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
> index f817f3d5e758..2876a7df1b0a 100644
> --- a/arch/ia64/Makefile
> +++ b/arch/ia64/Makefile
> @@ -40,7 +40,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
> endif
>
> quiet_cmd_gzip = GZIP $@
> -cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
> +cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> quiet_cmd_objcopy = OBJCOPY $@
> cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
> index ce6db5e5a5a3..0415d28dbe4f 100644
> --- a/arch/m68k/Makefile
> +++ b/arch/m68k/Makefile
> @@ -135,10 +135,10 @@ vmlinux.gz: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - $(_GZIP) -9c vmlinux.tmp >vmlinux.gz
> + $(KGZIP) -9c vmlinux.tmp >vmlinux.gz
> rm vmlinux.tmp
> else
> - $(_GZIP) -9c vmlinux >vmlinux.gz
> + $(KGZIP) -9c vmlinux >vmlinux.gz
> endif
>
> bzImage: vmlinux.bz2
> @@ -148,10 +148,10 @@ vmlinux.bz2: vmlinux
> ifndef CONFIG_KGDB
> cp vmlinux vmlinux.tmp
> $(STRIP) vmlinux.tmp
> - $(_BZIP2) -1c vmlinux.tmp >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux.tmp >vmlinux.bz2
> rm vmlinux.tmp
> else
> - $(_BZIP2) -1c vmlinux >vmlinux.bz2
> + $(KBZIP2) -1c vmlinux >vmlinux.bz2
> endif
>
> archclean:
> diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile
> index 182a5bca3e2c..5140c602207f 100644
> --- a/arch/parisc/Makefile
> +++ b/arch/parisc/Makefile
> @@ -162,7 +162,7 @@ vmlinuz: bzImage
> $(OBJCOPY) $(boot)/bzImage $@
> else
> vmlinuz: vmlinux
> - @$(_GZIP) -cf -9 $< > $@
> + @$(KGZIP) -cf -9 $< > $@
> endif
>
> install:
> diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
> index 127f2a7e3ced..94eeddb2e599 100644
> --- a/scripts/Makefile.lib
> +++ b/scripts/Makefile.lib
> @@ -244,7 +244,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
> # ---------------------------------------------------------------------------
>
> quiet_cmd_gzip = GZIP $@
> - cmd_gzip = cat $(real-prereqs) | $(_GZIP) -n -f -9 > $@
> + cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
>
> # DTC
> # ---------------------------------------------------------------------------
> @@ -337,7 +337,7 @@ printf "%08x\n" $$dec_size | \
> )
>
> quiet_cmd_bzip2 = BZIP2 $@
> - cmd_bzip2 = { cat $(real-prereqs) | $(_BZIP2) -9; $(size_append); } > $@
> + cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@
>
> # Lzma
> # ---------------------------------------------------------------------------
> @@ -346,7 +346,7 @@ quiet_cmd_lzma = LZMA $@
> cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@
>
> quiet_cmd_lzo = LZO $@
> - cmd_lzo = { cat $(real-prereqs) | $(_LZOP) -9; $(size_append); } > $@
> + cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@
>
> quiet_cmd_lz4 = LZ4 $@
> cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index b2b6153af63a..f952fb64789d 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -45,7 +45,7 @@ if test "$(objtree)" != "$(srctree)"; then \
> false; \
> fi ; \
> $(srctree)/scripts/setlocalversion --save-scmversion; \
> -tar -I $(_GZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> +tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
> --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3); \
> rm -f $(objtree)/.scmversion
>
> @@ -127,8 +127,8 @@ util/PERF-VERSION-GEN $(CURDIR)/$(perf-tar)/); \
> tar rf $(perf-tar).tar $(perf-tar)/HEAD $(perf-tar)/PERF-VERSION-FILE; \
> rm -r $(perf-tar); \
> $(if $(findstring tar-src,$@),, \
> -$(if $(findstring bz2,$@),$(_BZIP2), \
> -$(if $(findstring gz,$@),$(_GZIP), \
> +$(if $(findstring bz2,$@),$(KBZIP2), \
> +$(if $(findstring gz,$@),$(KGZIP), \
> $(if $(findstring xz,$@),$(XZ), \
> $(error unknown target $@)))) \
> -f -9 $(perf-tar).tar)
> diff --git a/scripts/package/buildtar b/scripts/package/buildtar
> index ad62c6879622..fb1578e72ab9 100755
> --- a/scripts/package/buildtar
> +++ b/scripts/package/buildtar
> @@ -28,11 +28,11 @@ case "${1}" in
> opts=
> ;;
> targz-pkg)
> - opts="-I ${_GZIP}"
> + opts="-I ${KGZIP}"
> tarball=${tarball}.gz
> ;;
> tarbz2-pkg)
> - opts="-I ${_BZIP2}"
> + opts="-I ${KBZIP2}"
> tarball=${tarball}.bz2
> ;;
> tarxz-pkg)
> --
> 2.26.2
>
--
Best Regards
Masahiro Yamada
^ permalink raw reply [flat|nested] 20+ messages in thread
end of thread, other threads:[~2020-06-09 1:04 UTC | newest]
Thread overview: 20+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-05-14 13:12 [RFC PATCH] kbuild: add variables for compression tools Denis Efremov
2020-05-15 2:20 ` Masahiro Yamada
2020-05-15 9:40 ` Denis Efremov
2020-05-21 7:20 ` Masahiro Yamada
2020-05-21 12:13 ` [RFC PATCH v2] " Denis Efremov
2020-05-22 8:43 ` Denis Efremov
2020-05-30 13:44 ` [RFC PATCH v3] " Denis Efremov
2020-06-01 12:45 ` Masahiro Yamada
2020-06-03 9:20 ` [RFC PATCH v4] " Denis Efremov
2020-06-04 0:12 ` Masahiro Yamada
2020-06-05 7:39 ` [PATCH v5] " Denis Efremov
2020-06-06 14:43 ` Masahiro Yamada
2020-06-08 1:30 ` Guenter Roeck
2020-06-08 4:59 ` Masahiro Yamada
2020-06-08 9:59 ` [PATCH] kbuild: fix broken builds because of GZIP,BZIP2,LZOP variables Denis Efremov
2020-06-08 15:36 ` Adam Borowski
2020-06-08 16:23 ` Denis Efremov
2020-06-09 1:03 ` Masahiro Yamada
2020-06-08 10:28 ` [PATCH v5] kbuild: add variables for compression tools Denis Efremov
2020-06-08 13:52 ` Guenter Roeck
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).