All of lore.kernel.org
 help / color / mirror / Atom feed
From: Youquan Song <youquan.song@intel.com>
To: stable@vger.kernel.org, gregkh@linuxfoundation.org
Cc: tim.c.chen@linux.intel.com, ashok.raj@intel.com,
	dave.hansen@intel.com, yi.y.sun@linux.intel.com,
	youquan.song@intel.com, youquan.song@linux.intel.com,
	David Woodhouse <dwmw@amazon.co.uk>,
	Linus Torvalds <torvalds@linux-foundation.org>,
	Peter Zijlstra <peterz@infradead.org>,
	arjan.van.de.ven@intel.com, bp@alien8.de, jmattson@google.com,
	karahmed@amazon.de, kvm@vger.kernel.org, pbonzini@redhat.com,
	rkrcmar@redhat.com
Subject: [PATCH 13/24] x86/speculation: Use IBRS if available before calling into firmware
Date: Tue, 17 Apr 2018 00:27:09 -0400	[thread overview]
Message-ID: <1523939240-16508-13-git-send-email-youquan.song@intel.com> (raw)
In-Reply-To: <1523939240-16508-1-git-send-email-youquan.song@intel.com>

From: David Woodhouse <dwmw@amazon.co.uk>

(cherry picked from commit dd84441a797150dcc49298ec95c459a8891d8bb1)

Retpoline means the kernel is safe because it has no indirect branches.
But firmware isn't, so use IBRS for firmware calls if it's available.

Block preemption while IBRS is set, although in practice the call sites
already had to be doing that.

Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware
code, from an NMI handler. I don't want to touch that with a bargepole.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: arjan.van.de.ven@intel.com
Cc: bp@alien8.de
Cc: dave.hansen@intel.com
Cc: jmattson@google.com
Cc: karahmed@amazon.de
Cc: kvm@vger.kernel.org
Cc: pbonzini@redhat.com
Cc: rkrcmar@redhat.com
Link: http://lkml.kernel.org/r/1519037457-7643-2-git-send-email-dwmw@amazon.co.uk
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Youquan Song <youquan.song@linux.intel.com> [v4.4 backport]
---
 arch/x86/include/asm/apm.h           |  6 ++++++
 arch/x86/include/asm/cpufeature.h    |  1 +
 arch/x86/include/asm/efi.h           |  7 +++++++
 arch/x86/include/asm/nospec-branch.h | 39 +++++++++++++++++++++++++++---------
 arch/x86/kernel/cpu/bugs.c           | 12 ++++++++++-
 5 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 20370c6..3d1ec41 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -6,6 +6,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include <asm/nospec-branch.h>
+
 #ifdef APM_ZERO_SEGS
 #	define APM_DO_ZERO_SEGS \
 		"pushl %%ds\n\t" \
@@ -31,6 +33,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
 	 * N.B. We do NOT need a cld after the BIOS call
 	 * because we always save and restore the flags.
 	 */
+	firmware_restrict_branch_speculation_start();
 	__asm__ __volatile__(APM_DO_ZERO_SEGS
 		"pushl %%edi\n\t"
 		"pushl %%ebp\n\t"
@@ -43,6 +46,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
 		  "=S" (*esi)
 		: "a" (func), "b" (ebx_in), "c" (ecx_in)
 		: "memory", "cc");
+	firmware_restrict_branch_speculation_end();
 }
 
 static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -55,6 +59,7 @@ static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
 	 * N.B. We do NOT need a cld after the BIOS call
 	 * because we always save and restore the flags.
 	 */
+	firmware_restrict_branch_speculation_start();
 	__asm__ __volatile__(APM_DO_ZERO_SEGS
 		"pushl %%edi\n\t"
 		"pushl %%ebp\n\t"
@@ -67,6 +72,7 @@ static inline u8 apm_bios_call_simple_asm(u32 func, u32 ebx_in,
 		  "=S" (si)
 		: "a" (func), "b" (ebx_in), "c" (ecx_in)
 		: "memory", "cc");
+	firmware_restrict_branch_speculation_end();
 	return error;
 }
 
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 13a53a6..8677d45 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -207,6 +207,7 @@
 #define X86_FEATURE_KAISER	( 7*32+31) /* CONFIG_PAGE_TABLE_ISOLATION w/o nokaiser */
 
 #define X86_FEATURE_USE_IBPB	( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW	( 7*32+22) /* "" Use IBRS during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW  ( 8*32+ 0) /* Intel TPR Shadow */
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 0010c78..7e5a2ff 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -3,6 +3,7 @@
 
 #include <asm/fpu/api.h>
 #include <asm/pgtable.h>
+#include <asm/nospec-branch.h>
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -39,8 +40,10 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...);
 ({									\
 	efi_status_t __s;						\
 	kernel_fpu_begin();						\
+	firmware_restrict_branch_speculation_start();			\
 	__s = ((efi_##f##_t __attribute__((regparm(0)))*)		\
 		efi.systab->runtime->f)(args);				\
+	firmware_restrict_branch_speculation_end();			\
 	kernel_fpu_end();						\
 	__s;								\
 })
@@ -49,8 +52,10 @@ extern unsigned long asmlinkage efi_call_phys(void *, ...);
 #define __efi_call_virt(f, args...) \
 ({									\
 	kernel_fpu_begin();						\
+	firmware_restrict_branch_speculation_start();			\
 	((efi_##f##_t __attribute__((regparm(0)))*)			\
 		efi.systab->runtime->f)(args);				\
+	firmware_restrict_branch_speculation_end();			\
 	kernel_fpu_end();						\
 })
 
@@ -71,7 +76,9 @@ extern u64 asmlinkage efi_call(void *fp, ...);
 	efi_sync_low_kernel_mappings();					\
 	preempt_disable();						\
 	__kernel_fpu_begin();						\
+	firmware_restrict_branch_speculation_start();			\
 	__s = efi_call((void *)efi.systab->runtime->f, __VA_ARGS__);	\
+	firmware_restrict_branch_speculation_end();			\
 	__kernel_fpu_end();						\
 	preempt_enable();						\
 	__s;								\
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 5de59b4..27582aa 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -195,17 +195,38 @@ static inline void vmexit_fill_RSB(void)
 #endif
 }
 
+#define alternative_msr_write(_msr, _val, _feature)		\
+	asm volatile(ALTERNATIVE("",				\
+				 "movl %[msr], %%ecx\n\t"	\
+				 "movl %[val], %%eax\n\t"	\
+				 "movl $0, %%edx\n\t"		\
+				 "wrmsr",			\
+				 _feature)			\
+		     : : [msr] "i" (_msr), [val] "i" (_val)	\
+		     : "eax", "ecx", "edx", "memory")
+
 static inline void indirect_branch_prediction_barrier(void)
 {
-	asm volatile(ALTERNATIVE("",
-				 "movl %[msr], %%ecx\n\t"
-				 "movl %[val], %%eax\n\t"
-				 "movl $0, %%edx\n\t"
-				 "wrmsr",
-				 X86_FEATURE_USE_IBPB)
-		     : : [msr] "i" (MSR_IA32_PRED_CMD),
-			 [val] "i" (PRED_CMD_IBPB)
-		     : "eax", "ecx", "edx", "memory");
+	alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
+			      X86_FEATURE_USE_IBPB);
+}
+
+/*
+ * With retpoline, we must use IBRS to restrict branch prediction
+ * before calling into firmware.
+ */
+static inline void firmware_restrict_branch_speculation_start(void)
+{
+	preempt_disable();
+	alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,
+			      X86_FEATURE_USE_IBRS_FW);
+}
+
+static inline void firmware_restrict_branch_speculation_end(void)
+{
+	alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,
+			      X86_FEATURE_USE_IBRS_FW);
+	preempt_enable();
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 478ac10..56724b9 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -302,6 +302,15 @@ retpoline_auto:
 		setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 		pr_info("Enabling Indirect Branch Prediction Barrier\n");
 	}
+
+	/*
+	 * Retpoline means the kernel is safe because it has no indirect
+	 * branches. But firmware isn't, so use IBRS to protect that.
+	 */
+	if (boot_cpu_has(X86_FEATURE_IBRS)) {
+		setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
+		pr_info("Enabling Restricted Speculation for firmware calls\n");
+	}
 }
 
 #undef pr_fmt
@@ -331,8 +340,9 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 		return sprintf(buf, "Not affected\n");
 
-	return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+	return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 			boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
+			boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
 			spectre_v2_module_string());
 }
 #endif
-- 
1.8.3.1

  parent reply	other threads:[~2018-04-16 16:58 UTC|newest]

Thread overview: 26+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-04-17  4:26 [PATCH 01/24] x86/cpufeatures: Add CPUID_7_EDX CPUID leaf Youquan Song
2018-04-16 18:27 ` Greg KH
2018-04-17  4:26 ` [PATCH 02/24] x86/cpufeatures: Add Intel feature bits for Speculation Control Youquan Song
2018-04-17  4:26 ` [PATCH 03/24] x86/cpufeatures: Add AMD " Youquan Song
2018-04-17  4:27 ` [PATCH 04/24] x86/msr: Add definitions for new speculation control MSRs Youquan Song
2018-04-17  4:27 ` [PATCH 05/24] x86/pti: Do not enable PTI on CPUs which are not vulnerable to Meltdown Youquan Song
2018-04-17  4:27 ` [PATCH 06/24] x86/cpufeature: Blacklist SPEC_CTRL/PRED_CMD on early Spectre v2 microcodes Youquan Song
2018-04-17  4:27 ` [PATCH 07/24] x86/speculation: Add basic IBPB (Indirect Branch Prediction Barrier) support Youquan Song
2018-04-17  4:27 ` [PATCH 08/24] x86/cpufeatures: Clean up Spectre v2 related CPUID flags Youquan Song
2018-04-17  4:27 ` [PATCH 09/24] x86/cpuid: Fix up "virtual" IBRS/IBPB/STIBP feature bits on Intel Youquan Song
2018-04-17  4:27 ` [PATCH 10/24] x86/speculation: Add <asm/msr-index.h> dependency Youquan Song
2018-04-17  4:27 ` [PATCH 11/24] x86/mm: Give each mm TLB flush generation a unique ID Youquan Song
2018-04-17  4:27 ` [PATCH 12/24] x86/speculation: Use Indirect Branch Prediction Barrier in context switch Youquan Song
2018-04-17  4:27 ` Youquan Song [this message]
2018-04-17  4:27 ` [PATCH 14/24] x86/speculation: Move firmware_restrict_branch_speculation_*() from C to CPP Youquan Song
2018-04-17  4:27 ` [PATCH 15/24] KVM: nVMX: Eliminate vmcs02 pool Youquan Song
2018-04-17  4:27 ` [PATCH 16/24] KVM: VMX: introduce alloc_loaded_vmcs Youquan Song
2018-04-17  4:27 ` [PATCH 17/24] KVM: VMX: make MSR bitmaps per-VCPU Youquan Song
2018-04-17  4:27 ` [PATCH 18/24] KVM/x86: Add IBPB support Youquan Song
2018-04-17  4:27 ` [PATCH 19/24] KVM/VMX: Emulate MSR_IA32_ARCH_CAPABILITIES Youquan Song
2018-04-17  4:27 ` [PATCH 20/24] KVM/VMX: Allow direct access to MSR_IA32_SPEC_CTRL Youquan Song
2018-04-17  4:27 ` [PATCH 21/24] KVM/SVM: " Youquan Song
2018-04-17  4:27 ` [PATCH 22/24] KVM/x86: Remove indirect MSR op calls from SPEC_CTRL Youquan Song
2018-04-17  4:27 ` [PATCH 23/24] KVM/VMX: Optimize vmx_vcpu_run() and svm_vcpu_run() by marking the RDMSR path as unlikely() Youquan Song
2018-04-17  4:27 ` [PATCH 24/24] x86/spectre_v2: Don't check microcode versions when running under hypervisors Youquan Song
2018-04-18  3:18 [PATCH 00/24] Backport Speculation Control support for 4.4 Youquan Song
2018-04-18  3:18 ` [PATCH 13/24] x86/speculation: Use IBRS if available before calling into firmware Youquan Song

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1523939240-16508-13-git-send-email-youquan.song@intel.com \
    --to=youquan.song@intel.com \
    --cc=arjan.van.de.ven@intel.com \
    --cc=ashok.raj@intel.com \
    --cc=bp@alien8.de \
    --cc=dave.hansen@intel.com \
    --cc=dwmw@amazon.co.uk \
    --cc=gregkh@linuxfoundation.org \
    --cc=jmattson@google.com \
    --cc=karahmed@amazon.de \
    --cc=kvm@vger.kernel.org \
    --cc=pbonzini@redhat.com \
    --cc=peterz@infradead.org \
    --cc=rkrcmar@redhat.com \
    --cc=stable@vger.kernel.org \
    --cc=tim.c.chen@linux.intel.com \
    --cc=torvalds@linux-foundation.org \
    --cc=yi.y.sun@linux.intel.com \
    --cc=youquan.song@linux.intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.