From: Sedat Dilek <sedat.dilek@gmail.com>
To: Arvind Sankar <nivedita@alum.mit.edu>
Cc: Thomas Gleixner <tglx@linutronix.de>,
Ingo Molnar <mingo@redhat.com>, Borislav Petkov <bp@alien8.de>,
"H. Peter Anvin" <hpa@zytor.com>,
x86@kernel.org, Nick Desaulniers <ndesaulniers@google.com>,
Fangrui Song <maskray@google.com>,
Dmitry Golovin <dima@golovin.in>,
Clang-Built-Linux ML <clang-built-linux@googlegroups.com>,
Ard Biesheuvel <ardb@kernel.org>,
Masahiro Yamada <masahiroy@kernel.org>,
Daniel Kiper <daniel.kiper@oracle.com>,
Kees Cook <keescook@chromium.org>,
Nathan Chancellor <natechancellor@gmail.com>,
Arnd Bergmann <arnd@arndb.de>, "H . J . Lu" <hjl@sourceware.org>,
linux-kernel@vger.kernel.org
Subject: Re: [PATCH v4 0/7] x86/boot: Remove runtime relocations from compressed kernel
Date: Tue, 14 Jul 2020 22:17:33 +0200 [thread overview]
Message-ID: <CA+icZUV45bAExLaUAPkiJEC-kOYw2vMWUrDpy_9XNPeK7WB4qQ@mail.gmail.com> (raw)
In-Reply-To: <20200714201451.GA902932@rani.riverdale.lan>
[-- Attachment #1: Type: text/plain, Size: 1292 bytes --]
On Tue, Jul 14, 2020 at 10:14 PM Arvind Sankar <nivedita@alum.mit.edu> wrote:
>
> On Tue, Jul 14, 2020 at 10:10:14PM +0200, Sedat Dilek wrote:
> > On Tue, Jul 14, 2020 at 10:07 PM Arvind Sankar <nivedita@alum.mit.edu> wrote:
> > >
> > > On Tue, Jul 14, 2020 at 09:53:19PM +0200, Sedat Dilek wrote:
> > > >
> > > > Hmm, you might be right with moving to LDFLAGS_vmlinux.
> > > >
> > > > Attached are my linux-config and dmesg-output.
> > > >
> > > > - Sedat -
> > >
> > > Which tree are you building against? I notice you have KERNEL_ZSTD
> > > enabled, which hasn't been merged yet.
> > >
> > > Are you using Nick's series [v7]?
> > >
> > > Also from the naming -- are you using LLVM_IAS=1 with some patches to
> > > make it work?
> > >
> > > [v7] https://lore.kernel.org/lkml/20200708185024.2767937-1-nickrterrell@gmail.com/
> >
> > Sorry for not telling you the full story.
> > Yes, I have some additional patches like Nick T. "zstd-v7" which
> > should IMHO not touch this area.
> >
> > - Sedat -
>
> That series does touch boot/compressed, since the point is to add
> support for zstd-compressed kernels. I'll need to test it out, the
> __DISABLE_EXPORTS he's now using must not be working for some reason.
I have seen this in the between zstd-v6 and zstd-v7 - diff attached.
- Sedat -
[-- Attachment #2: zstd-initramfs-terrelln-v6-VS-zstd-initramfs-terrelln-v7.diff --]
[-- Type: text/x-patch, Size: 7050 bytes --]
diff --git a/Makefile b/Makefile
index fe0164a654c7..d14f9b7038ba 100644
--- a/Makefile
+++ b/Makefile
@@ -464,6 +464,7 @@ KLZOP = lzop
LZMA = lzma
LZ4 = lz4c
XZ = xz
+ZSTD = zstd
CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
@@ -512,7 +513,7 @@ CLANG_FLAGS :=
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
export CPP AR NM STRIP OBJCOPY OBJDUMP OBJSIZE READELF PAHOLE LEX YACC AWK INSTALLKERNEL
export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
-export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ
+export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 471e61400a2e..3498cd990869 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -146,7 +146,7 @@ $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
$(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
$(call if_changed,lz4)
$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
- $(call if_changed,zstd)
+ $(call if_changed,zstd22)
suffix-$(CONFIG_KERNEL_GZIP) := gz
suffix-$(CONFIG_KERNEL_BZIP2) := bz2
diff --git a/lib/decompress_unzstd.c b/lib/decompress_unzstd.c
index f317afab502f..053834f24b8c 100644
--- a/lib/decompress_unzstd.c
+++ b/lib/decompress_unzstd.c
@@ -59,12 +59,15 @@
* zstd's only source dependeny is xxhash, which has no source
* dependencies.
*
- * zstd and xxhash avoid declaring themselves as modules
- * when ZSTD_PREBOOT and XXH_PREBOOT are defined.
+ * When UNZSTD_PREBOOT is defined we declare __decompress(), which is
+ * used for kernel decompression, instead of unzstd().
+ *
+ * __DISABLE_EXPORTS stops zstd and xxhash from declaring themselves
+ * as modules by disabling the EXPORT_SYMBOL macro.
*/
#ifdef STATIC
-# define ZSTD_PREBOOT
-# define XXH_PREBOOT
+# define UNZSTD_PREBOOT
+# define __DISABLE_EXPORTS
# include "xxhash.c"
# include "zstd/entropy_common.c"
# include "zstd/fse_decompress.c"
@@ -319,7 +322,7 @@ static int INIT __unzstd(unsigned char *in_buf, long in_len,
return err;
}
-#ifndef ZSTD_PREBOOT
+#ifndef UNZSTD_PREBOOT
STATIC int INIT unzstd(unsigned char *buf, long len,
long (*fill)(void*, unsigned long),
long (*flush)(void*, unsigned long),
diff --git a/lib/xxhash.c b/lib/xxhash.c
index b4364e011392..aa61e2a3802f 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -80,11 +80,13 @@ void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
{
memcpy(dst, src, sizeof(*dst));
}
+EXPORT_SYMBOL(xxh32_copy_state);
void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
{
memcpy(dst, src, sizeof(*dst));
}
+EXPORT_SYMBOL(xxh64_copy_state);
/*-***************************
* Simple Hash Functions
@@ -149,6 +151,7 @@ uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
return h32;
}
+EXPORT_SYMBOL(xxh32);
static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
{
@@ -231,6 +234,7 @@ uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
return h64;
}
+EXPORT_SYMBOL(xxh64);
/*-**************************************************
* Advanced Hash Functions
@@ -247,6 +251,7 @@ void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
state.v4 = seed - PRIME32_1;
memcpy(statePtr, &state, sizeof(state));
}
+EXPORT_SYMBOL(xxh32_reset);
void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
{
@@ -260,6 +265,7 @@ void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
state.v4 = seed - PRIME64_1;
memcpy(statePtr, &state, sizeof(state));
}
+EXPORT_SYMBOL(xxh64_reset);
int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
{
@@ -328,6 +334,7 @@ int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
return 0;
}
+EXPORT_SYMBOL(xxh32_update);
uint32_t xxh32_digest(const struct xxh32_state *state)
{
@@ -365,6 +372,7 @@ uint32_t xxh32_digest(const struct xxh32_state *state)
return h32;
}
+EXPORT_SYMBOL(xxh32_digest);
int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
{
@@ -431,6 +439,7 @@ int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
return 0;
}
+EXPORT_SYMBOL(xxh64_update);
uint64_t xxh64_digest(const struct xxh64_state *state)
{
@@ -485,19 +494,7 @@ uint64_t xxh64_digest(const struct xxh64_state *state)
return h64;
}
-
-#ifndef XXH_PREBOOT
-EXPORT_SYMBOL(xxh32_copy_state);
-EXPORT_SYMBOL(xxh64_copy_state);
-EXPORT_SYMBOL(xxh32);
-EXPORT_SYMBOL(xxh64);
-EXPORT_SYMBOL(xxh32_reset);
-EXPORT_SYMBOL(xxh64_reset);
-EXPORT_SYMBOL(xxh32_update);
-EXPORT_SYMBOL(xxh32_digest);
-EXPORT_SYMBOL(xxh64_update);
EXPORT_SYMBOL(xxh64_digest);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("xxHash");
-#endif
diff --git a/lib/zstd/decompress.c b/lib/zstd/decompress.c
index 73ded63278cf..269ee9a796c1 100644
--- a/lib/zstd/decompress.c
+++ b/lib/zstd/decompress.c
@@ -2490,7 +2490,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream *zds, ZSTD_outBuffer *output, ZSTD_inB
}
}
-#ifndef ZSTD_PREBOOT
EXPORT_SYMBOL(ZSTD_DCtxWorkspaceBound);
EXPORT_SYMBOL(ZSTD_initDCtx);
EXPORT_SYMBOL(ZSTD_decompressDCtx);
@@ -2530,4 +2529,3 @@ EXPORT_SYMBOL(ZSTD_insertBlock);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Decompressor");
-#endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index d960f8815f87..54f7b7eb580b 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -419,14 +419,21 @@ quiet_cmd_xzmisc = XZMISC $@
# format has the size information available at the beginning of the file too,
# but it's in a more complex format and it's good to avoid changing the part
# of the boot code that reads the uncompressed size.
+#
# Note that the bytes added by size_append will make the zstd tool think that
# the file is corrupt. This is expected.
+#
+# zstd uses a maximum window size of 8 MB. zstd22 uses a maximum window size of
+# 128 MB. zstd22 is used for kernel compression because it is decompressed in a
+# single pass, so zstd doesn't need to allocate a window buffer. When streaming
+# decompression is used, like initramfs decompression, zstd22 should likely not
+# be used because it would require zstd to allocate a 128 MB buffer.
quiet_cmd_zstd = ZSTD $@
-cmd_zstd = (cat $(filter-out FORCE,$^) | \
- zstd -19 && \
- $(call size_append, $(filter-out FORCE,$^))) > $@ || \
- (rm -f $@ ; false)
+ cmd_zstd = { cat $(real-prereqs) | $(ZSTD) -19; $(size_append); } > $@
+
+quiet_cmd_zstd22 = ZSTD22 $@
+ cmd_zstd22 = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@
# ASM offsets
# ---------------------------------------------------------------------------
next prev parent reply other threads:[~2020-07-14 20:17 UTC|newest]
Thread overview: 113+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-29 14:09 [PATCH v3 0/7] x86/boot: Remove runtime relocations from compressed kernel Arvind Sankar
2020-06-29 14:09 ` [PATCH v3 1/7] x86/boot/compressed: Move .got.plt entries out of the .got section Arvind Sankar
2020-06-29 15:48 ` Kees Cook
2020-06-29 15:50 ` Arvind Sankar
2020-06-29 15:51 ` Ard Biesheuvel
2020-06-29 16:10 ` Kees Cook
2020-06-29 14:09 ` [PATCH v3 2/7] x86/boot/compressed: Force hidden visibility for all symbol references Arvind Sankar
2020-06-29 15:50 ` Kees Cook
2020-07-14 9:20 ` Sedat Dilek
2020-07-14 9:47 ` Ard Biesheuvel
2020-06-29 14:09 ` [PATCH v3 3/7] x86/boot/compressed: Get rid of GOT fixup code Arvind Sankar
2020-06-29 15:53 ` Kees Cook
2020-06-29 14:09 ` [PATCH v3 4/7] x86/boot: Add .text.* to setup.ld Arvind Sankar
2020-06-29 15:55 ` Kees Cook
2020-06-29 14:09 ` [PATCH v3 5/7] x86/boot: Remove run-time relocations from .head.text code Arvind Sankar
2020-06-29 16:04 ` Kees Cook
2020-06-29 17:01 ` Arvind Sankar
2020-07-14 13:20 ` Sedat Dilek
2020-06-29 14:09 ` [PATCH v3 6/7] x86/boot: Remove runtime relocations from head_{32,64}.S Arvind Sankar
2020-06-29 16:06 ` Kees Cook
2020-06-29 16:52 ` Arvind Sankar
2020-06-29 14:09 ` [PATCH v3 7/7] x86/boot: Check that there are no runtime relocations Arvind Sankar
2020-06-29 16:09 ` Kees Cook
2020-06-29 16:11 ` Ard Biesheuvel
2020-06-29 16:20 ` Kees Cook
2020-06-29 16:56 ` Arvind Sankar
2020-06-29 17:37 ` Fangrui Song
2020-06-29 18:11 ` Ard Biesheuvel
2020-06-29 23:34 ` Fangrui Song
2020-06-30 16:26 ` Ard Biesheuvel
2020-06-30 17:54 ` Arvind Sankar
2020-06-30 22:00 ` Fangrui Song
2020-06-30 23:27 ` Arvind Sankar
2020-07-01 6:44 ` Ard Biesheuvel
2020-07-01 14:42 ` Arvind Sankar
2020-06-29 18:43 ` Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 0/7] x86/boot: Remove runtime relocations from compressed kernel Arvind Sankar
2020-07-14 13:15 ` Sedat Dilek
2020-07-14 14:15 ` Arvind Sankar
2020-07-14 18:13 ` Sedat Dilek
2020-07-14 18:30 ` Sedat Dilek
2020-07-14 18:33 ` Sedat Dilek
2020-07-14 19:21 ` Sedat Dilek
2020-07-14 19:29 ` Arvind Sankar
2020-07-14 19:53 ` Sedat Dilek
2020-07-14 20:07 ` Arvind Sankar
2020-07-14 20:10 ` Sedat Dilek
2020-07-14 20:14 ` Arvind Sankar
2020-07-14 20:17 ` Sedat Dilek [this message]
2020-07-14 20:08 ` Sedat Dilek
2020-07-14 20:21 ` Arvind Sankar
2020-07-14 20:24 ` Sedat Dilek
2020-07-14 20:27 ` Sedat Dilek
2020-07-14 20:35 ` Arvind Sankar
2020-07-14 20:43 ` Sedat Dilek
2020-07-14 21:07 ` Arvind Sankar
2020-07-14 20:33 ` Arvind Sankar
2020-07-15 0:41 ` [PATCH v5 0/7] x86/boot: Remove run-time " Arvind Sankar
2020-07-15 1:46 ` Sedat Dilek
2020-07-15 7:11 ` Sedat Dilek
2020-07-17 13:46 ` Arvind Sankar
2020-07-17 18:16 ` Nick Desaulniers
2020-07-17 18:21 ` Sedat Dilek
2020-07-17 20:17 ` [PATCH-next " Arvind Sankar
2020-07-17 23:46 ` Nick Desaulniers
2020-07-29 22:04 ` Kees Cook
2020-07-29 22:23 ` Arvind Sankar
2020-07-30 2:38 ` Kees Cook
2020-07-17 20:17 ` [PATCH-next v5 1/7] x86/boot/compressed: Move .got.plt entries out of the .got section Arvind Sankar
2020-07-17 20:17 ` [PATCH-next v5 2/7] x86/boot/compressed: Force hidden visibility for all symbol references Arvind Sankar
2020-07-17 20:17 ` [PATCH-next v5 3/7] x86/boot/compressed: Get rid of GOT fixup code Arvind Sankar
2020-07-17 20:17 ` [PATCH-next v5 4/7] x86/boot: Add .text.* to setup.ld Arvind Sankar
2020-07-17 20:17 ` [PATCH-next v5 5/7] x86/boot: Remove run-time relocations from .head.text code Arvind Sankar
2020-07-17 20:18 ` [PATCH-next v5 6/7] x86/boot: Remove run-time relocations from head_{32,64}.S Arvind Sankar
2020-07-17 20:18 ` [PATCH-next v5 7/7] x86/boot: Check that there are no run-time relocations Arvind Sankar
2020-07-18 5:44 ` [PATCH v5 0/7] x86/boot: Remove run-time relocations from compressed kernel Ard Biesheuvel
2020-07-18 7:01 ` Sedat Dilek
2020-07-24 23:25 ` Kees Cook
2020-07-31 14:53 ` Arvind Sankar
2020-07-31 14:58 ` Sedat Dilek
2020-07-31 20:27 ` [PATCH v6 " Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 1/7] x86/boot/compressed: Move .got.plt entries out of the .got section Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 2/7] x86/boot/compressed: Force hidden visibility for all symbol references Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 3/7] x86/boot/compressed: Get rid of GOT fixup code Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 4/7] x86/boot: Add .text.* to setup.ld Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 5/7] x86/boot: Remove run-time relocations from .head.text code Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 6/7] x86/boot: Remove run-time relocations from head_{32,64}.S Arvind Sankar
2020-07-31 20:27 ` [PATCH v6 7/7] x86/boot: Check that there are no run-time relocations Arvind Sankar
2020-07-31 23:15 ` [PATCH v6 0/7] x86/boot: Remove run-time relocations from compressed kernel Kees Cook
2020-07-15 0:41 ` [PATCH v5 1/7] x86/boot/compressed: Move .got.plt entries out of the .got section Arvind Sankar
2020-07-15 8:52 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 2/7] x86/boot/compressed: Force hidden visibility for all symbol references Arvind Sankar
2020-07-15 8:54 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 3/7] x86/boot/compressed: Get rid of GOT fixup code Arvind Sankar
2020-07-15 8:54 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 4/7] x86/boot: Add .text.* to setup.ld Arvind Sankar
2020-07-15 8:55 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 5/7] x86/boot: Remove run-time relocations from .head.text code Arvind Sankar
2020-07-15 8:56 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 6/7] x86/boot: Remove run-time relocations from head_{32,64}.S Arvind Sankar
2020-07-15 8:58 ` Sedat Dilek
2020-07-15 9:03 ` Ard Biesheuvel
2020-07-15 9:10 ` Sedat Dilek
2020-07-15 0:41 ` [PATCH v5 7/7] x86/boot: Check that there are no run-time relocations Arvind Sankar
2020-07-15 9:00 ` Sedat Dilek
2020-07-15 9:12 ` Sedat Dilek
2020-07-14 2:38 ` [PATCH v4 1/7] x86/boot/compressed: Move .got.plt entries out of the .got section Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 2/7] x86/boot/compressed: Force hidden visibility for all symbol references Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 3/7] x86/boot/compressed: Get rid of GOT fixup code Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 4/7] x86/boot: Add .text.* to setup.ld Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 5/7] x86/boot: Remove run-time relocations from .head.text code Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 6/7] x86/boot: Remove runtime relocations from head_{32,64}.S Arvind Sankar
2020-07-14 2:38 ` [PATCH v4 7/7] x86/boot: Check that there are no runtime relocations Arvind Sankar
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=CA+icZUV45bAExLaUAPkiJEC-kOYw2vMWUrDpy_9XNPeK7WB4qQ@mail.gmail.com \
--to=sedat.dilek@gmail.com \
--cc=ardb@kernel.org \
--cc=arnd@arndb.de \
--cc=bp@alien8.de \
--cc=clang-built-linux@googlegroups.com \
--cc=daniel.kiper@oracle.com \
--cc=dima@golovin.in \
--cc=hjl@sourceware.org \
--cc=hpa@zytor.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=masahiroy@kernel.org \
--cc=maskray@google.com \
--cc=mingo@redhat.com \
--cc=natechancellor@gmail.com \
--cc=ndesaulniers@google.com \
--cc=nivedita@alum.mit.edu \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).