From: Kees Cook <keescook@chromium.org>
To: Thomas Gleixner <tglx@linutronix.de>
Cc: Xi Ruoyao <xry111@mengyan1223.wang>,
Peter Zijlstra <peterz@infradead.org>,
Jiri Kosina <jikos@kernel.org>,
Linus Torvalds <torvalds@linux-foundation.org>,
Ingo Molnar <mingo@kernel.org>,
Linux List Kernel Mailing <linux-kernel@vger.kernel.org>,
Borislav Petkov <bp@alien8.de>, Len Brown <lenb@kernel.org>,
Andrew Morton <akpm@linux-foundation.org>,
"Rafael J. Wysocki" <rafael.j.wysocki@intel.com>,
Tony Luck <tony.luck@intel.com>,
Bob Moore <robert.moore@intel.com>,
Erik Schmauss <erik.schmauss@intel.com>,
Josh Poimboeuf <jpoimboe@redhat.com>,
Daniel Bristot de Oliveira <bristot@redhat.com>,
Juergen Gross <jgross@suse.com>
Subject: Re: [PATCH] x86/asm: Move native_write_cr0/3() out of line
Date: Wed, 10 Jul 2019 12:59:41 -0700 [thread overview]
Message-ID: <201907101258.FE97AEC86@keescook> (raw)
In-Reply-To: <alpine.DEB.2.21.1907102140340.1758@nanos.tec.linutronix.de>
On Wed, Jul 10, 2019 at 09:42:46PM +0200, Thomas Gleixner wrote:
> The pinning of sensitive CR0 and CR4 bits caused a boot crash when loading
> the kvm_intel module on a kernel compiled with CONFIG_PARAVIRT=n.
>
> The reason is that the static key which controls the pinning is marked RO
> after init. The kvm_intel module contains a CR4 write which requires to
> update the static key entry list. That obviously does not work when the key
> is in a RO section.
>
> With CONFIG_PARAVIRT enabled this does not happen because the CR4 write
> uses the paravirt indirection and the actual write function is built in.
>
> As the key is intended to be immutable after init, move
> native_write_cr0/3() out of line.
>
> While at it consolidate the update of the cr4 shadow variable and store the
> value right away when the pinning is initialized on a booting CPU. No point
> in reading it back 20 instructions later. This allows to confine the static
> key and the pinning variable to cpu/common and allows to mark them static.
>
> Fixes: 8dbec27a242c ("x86/asm: Pin sensitive CR0 bits")
> Fixes: 873d50d58f67 ("x86/asm: Pin sensitive CR4 bits")
> Reported-by: Linus Torvalds <torvalds@linux-foundation.org>
> Reported-by: Xi Ruoyao <xry111@mengyan1223.wang>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Tested-by: Xi Ruoyao <xry111@mengyan1223.wang>
Thank you for tracking this down and solving it!
Nit: should be "cr0/4()" in Subject and in paragraph 4.
Acked-by: Kees Cook <keescook@chromium.org>
-Kees
> ---
> arch/x86/include/asm/processor.h | 1
> arch/x86/include/asm/special_insns.h | 41 -------------------
> arch/x86/kernel/cpu/common.c | 72 +++++++++++++++++++++++++++--------
> arch/x86/kernel/smpboot.c | 14 ------
> arch/x86/xen/smp_pv.c | 1
> 5 files changed, 61 insertions(+), 68 deletions(-)
>
> --- a/arch/x86/include/asm/processor.h
> +++ b/arch/x86/include/asm/processor.h
> @@ -741,6 +741,7 @@ extern void load_direct_gdt(int);
> extern void load_fixmap_gdt(int);
> extern void load_percpu_segment(int);
> extern void cpu_init(void);
> +extern void cr4_init(void);
>
> static inline unsigned long get_debugctlmsr(void)
> {
> --- a/arch/x86/include/asm/special_insns.h
> +++ b/arch/x86/include/asm/special_insns.h
> @@ -18,9 +18,7 @@
> */
> extern unsigned long __force_order;
>
> -/* Starts false and gets enabled once CPU feature detection is done. */
> -DECLARE_STATIC_KEY_FALSE(cr_pinning);
> -extern unsigned long cr4_pinned_bits;
> +void native_write_cr0(unsigned long val);
>
> static inline unsigned long native_read_cr0(void)
> {
> @@ -29,24 +27,6 @@ static inline unsigned long native_read_
> return val;
> }
>
> -static inline void native_write_cr0(unsigned long val)
> -{
> - unsigned long bits_missing = 0;
> -
> -set_register:
> - asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order));
> -
> - if (static_branch_likely(&cr_pinning)) {
> - if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
> - bits_missing = X86_CR0_WP;
> - val |= bits_missing;
> - goto set_register;
> - }
> - /* Warn after we've set the missing bits. */
> - WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
> - }
> -}
> -
> static inline unsigned long native_read_cr2(void)
> {
> unsigned long val;
> @@ -91,24 +71,7 @@ static inline unsigned long native_read_
> return val;
> }
>
> -static inline void native_write_cr4(unsigned long val)
> -{
> - unsigned long bits_missing = 0;
> -
> -set_register:
> - asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits));
> -
> - if (static_branch_likely(&cr_pinning)) {
> - if (unlikely((val & cr4_pinned_bits) != cr4_pinned_bits)) {
> - bits_missing = ~val & cr4_pinned_bits;
> - val |= bits_missing;
> - goto set_register;
> - }
> - /* Warn after we've set the missing bits. */
> - WARN_ONCE(bits_missing, "CR4 bits went missing: %lx!?\n",
> - bits_missing);
> - }
> -}
> +void native_write_cr4(unsigned long val);
>
> #ifdef CONFIG_X86_64
> static inline unsigned long native_read_cr8(void)
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -366,10 +366,62 @@ static __always_inline void setup_umip(s
> cr4_clear_bits(X86_CR4_UMIP);
> }
>
> -DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
> -EXPORT_SYMBOL(cr_pinning);
> -unsigned long cr4_pinned_bits __ro_after_init;
> -EXPORT_SYMBOL(cr4_pinned_bits);
> +static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
> +static unsigned long cr4_pinned_bits __ro_after_init;
> +
> +void native_write_cr0(unsigned long val)
> +{
> + unsigned long bits_missing = 0;
> +
> +set_register:
> + asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order));
> +
> + if (static_branch_likely(&cr_pinning)) {
> + if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
> + bits_missing = X86_CR0_WP;
> + val |= bits_missing;
> + goto set_register;
> + }
> + /* Warn after we've set the missing bits. */
> + WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
> + }
> +}
> +EXPORT_SYMBOL(native_write_cr0);
> +
> +void native_write_cr4(unsigned long val)
> +{
> + unsigned long bits_missing = 0;
> +
> +set_register:
> + asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits));
> +
> + if (static_branch_likely(&cr_pinning)) {
> + if (unlikely((val & cr4_pinned_bits) != cr4_pinned_bits)) {
> + bits_missing = ~val & cr4_pinned_bits;
> + val |= bits_missing;
> + goto set_register;
> + }
> + /* Warn after we've set the missing bits. */
> + WARN_ONCE(bits_missing, "CR4 bits went missing: %lx!?\n",
> + bits_missing);
> + }
> +}
> +EXPORT_SYMBOL(native_write_cr4);
> +
> +void cr4_init(void)
> +{
> + unsigned long cr4 = __read_cr4();
> +
> + if (boot_cpu_has(X86_FEATURE_PCID))
> + cr4 |= X86_CR4_PCIDE;
> + if (static_branch_likely(&cr_pinning))
> + cr4 |= cr4_pinned_bits;
> +
> + __write_cr4(cr4);
> +
> + /* Initialize cr4 shadow for this CPU. */
> + this_cpu_write(cpu_tlbstate.cr4, cr4);
> +}
>
> /*
> * Once CPU feature detection is finished (and boot params have been
> @@ -1723,12 +1775,6 @@ void cpu_init(void)
>
> wait_for_master_cpu(cpu);
>
> - /*
> - * Initialize the CR4 shadow before doing anything that could
> - * try to read it.
> - */
> - cr4_init_shadow();
> -
> if (cpu)
> load_ucode_ap();
>
> @@ -1823,12 +1869,6 @@ void cpu_init(void)
>
> wait_for_master_cpu(cpu);
>
> - /*
> - * Initialize the CR4 shadow before doing anything that could
> - * try to read it.
> - */
> - cr4_init_shadow();
> -
> show_ucode_info_early();
>
> pr_info("Initializing CPU#%d\n", cpu);
> --- a/arch/x86/kernel/smpboot.c
> +++ b/arch/x86/kernel/smpboot.c
> @@ -210,28 +210,16 @@ static int enable_start_cpu0;
> */
> static void notrace start_secondary(void *unused)
> {
> - unsigned long cr4 = __read_cr4();
> -
> /*
> * Don't put *anything* except direct CPU state initialization
> * before cpu_init(), SMP booting is too fragile that we want to
> * limit the things done here to the most necessary things.
> */
> - if (boot_cpu_has(X86_FEATURE_PCID))
> - cr4 |= X86_CR4_PCIDE;
> - if (static_branch_likely(&cr_pinning))
> - cr4 |= cr4_pinned_bits;
> -
> - __write_cr4(cr4);
> + cr4_init();
>
> #ifdef CONFIG_X86_32
> /* switch away from the initial page table */
> load_cr3(swapper_pg_dir);
> - /*
> - * Initialize the CR4 shadow before doing anything that could
> - * try to read it.
> - */
> - cr4_init_shadow();
> __flush_tlb_all();
> #endif
> load_current_idt();
> --- a/arch/x86/xen/smp_pv.c
> +++ b/arch/x86/xen/smp_pv.c
> @@ -58,6 +58,7 @@ static void cpu_bringup(void)
> {
> int cpu;
>
> + cr4_init();
> cpu_init();
> touch_softlockup_watchdog();
> preempt_disable();
--
Kees Cook
next prev parent reply other threads:[~2019-07-10 19:59 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-08 16:27 [GIT PULL] x86/topology changes for v5.3 Ingo Molnar
2019-07-09 1:45 ` pr-tracker-bot
2019-07-09 21:20 ` Linus Torvalds
2019-07-09 21:26 ` Linus Torvalds
2019-07-09 21:45 ` Linus Torvalds
2019-07-09 22:00 ` Linus Torvalds
2019-07-09 22:07 ` Linus Torvalds
2019-07-09 22:27 ` Thomas Gleixner
2019-07-09 23:00 ` Thomas Gleixner
2019-07-09 23:17 ` Thomas Gleixner
2019-07-10 0:31 ` Kees Cook
2019-07-10 11:27 ` Xi Ruoyao
2019-07-10 12:01 ` Xi Ruoyao
2019-07-10 12:19 ` Thomas Gleixner
2019-07-10 12:31 ` Jiri Kosina
2019-07-10 13:21 ` Peter Zijlstra
2019-07-10 13:27 ` Jiri Kosina
2019-07-10 13:28 ` Jiri Kosina
2019-07-10 13:31 ` Xi Ruoyao
2019-07-10 13:25 ` Xi Ruoyao
2019-07-10 13:44 ` Peter Zijlstra
2019-07-10 14:03 ` Thomas Gleixner
2019-07-10 14:26 ` Thomas Gleixner
2019-07-10 14:22 ` Jiri Kosina
2019-07-10 14:26 ` Peter Zijlstra
2019-07-10 15:13 ` Thomas Gleixner
2019-07-10 15:58 ` Xi Ruoyao
2019-07-10 19:42 ` [PATCH] x86/asm: Move native_write_cr0/3() out of line Thomas Gleixner
2019-07-10 19:59 ` Kees Cook [this message]
2019-07-10 20:00 ` Thomas Gleixner
2019-07-10 20:02 ` Peter Zijlstra
2019-07-10 20:19 ` [tip:x86/urgent] x86/asm: Move native_write_cr0/4() " tip-bot for Thomas Gleixner
2019-07-10 14:44 ` [GIT PULL] x86/topology changes for v5.3 Xi Ruoyao
2019-07-11 7:11 ` Nadav Amit
2019-07-11 7:16 ` Thomas Gleixner
2019-07-11 8:01 ` Peter Zijlstra
2019-07-11 15:08 ` Kees Cook
2019-07-11 17:09 ` Nadav Amit
2019-07-10 0:59 ` Linus Torvalds
2019-07-10 1:08 ` Linus Torvalds
2019-07-10 3:21 ` Linus Torvalds
2019-07-10 5:15 ` Linus Torvalds
2019-07-10 5:33 ` Kees Cook
2019-07-10 18:40 ` Linus Torvalds
2019-07-10 10:03 ` Rafael J. Wysocki
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=201907101258.FE97AEC86@keescook \
--to=keescook@chromium.org \
--cc=akpm@linux-foundation.org \
--cc=bp@alien8.de \
--cc=bristot@redhat.com \
--cc=erik.schmauss@intel.com \
--cc=jgross@suse.com \
--cc=jikos@kernel.org \
--cc=jpoimboe@redhat.com \
--cc=lenb@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=rafael.j.wysocki@intel.com \
--cc=robert.moore@intel.com \
--cc=tglx@linutronix.de \
--cc=tony.luck@intel.com \
--cc=torvalds@linux-foundation.org \
--cc=xry111@mengyan1223.wang \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).