From: tip-bot for David Woodhouse <tipbot@zytor.com>
To: linux-tip-commits@vger.kernel.org
Cc: pjt@google.com, tim.c.chen@linux.intel.com, jikos@kernel.org,
gregkh@linux-foundation.org, dave.hansen@intel.com,
dwmw@amazon.co.uk, riel@redhat.com, luto@amacapital.net,
mingo@kernel.org, torvalds@linux-foundation.org,
ak@linux.intel.com, tglx@linutronix.de, hpa@zytor.com,
linux-kernel@vger.kernel.org, jpoimboe@redhat.com,
keescook@google.com, peterz@infradead.org
Subject: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit
Date: Fri, 12 Jan 2018 03:37:49 -0800 [thread overview]
Message-ID: <tip-117cc7a908c83697b0b737d15ae1eb5943afe35b@git.kernel.org> (raw)
In-Reply-To: <1515755487-8524-1-git-send-email-dwmw@amazon.co.uk>
Commit-ID: 117cc7a908c83697b0b737d15ae1eb5943afe35b
Gitweb: https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
Author: David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Fri, 12 Jan 2018 11:11:27 +0000
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 12:33:37 +0100
x86/retpoline: Fill return stack buffer on vmexit
In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.
[ak: numbers again for the RSB stuffing labels]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515755487-8524-1-git-send-email-dwmw@amazon.co.uk
---
arch/x86/include/asm/nospec-branch.h | 78 +++++++++++++++++++++++++++++++++++-
arch/x86/kvm/svm.c | 4 ++
arch/x86/kvm/vmx.c | 4 ++
3 files changed, 85 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
#include <asm/alternative-asm.h>
#include <asm/cpufeatures.h>
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS 16 /* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp) \
+ mov $(nr/2), reg; \
+771: \
+ call 772f; \
+773: /* speculation trap */ \
+ pause; \
+ jmp 773b; \
+772: \
+ call 774f; \
+775: /* speculation trap */ \
+ pause; \
+ jmp 775b; \
+774: \
+ dec reg; \
+ jnz 771b; \
+ add $(BITS_PER_LONG/8) * nr, sp;
+
#ifdef __ASSEMBLY__
/*
@@ -76,6 +118,20 @@
#endif
.endm
+ /*
+ * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+ * monstrosity above, manually.
+ */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+ ANNOTATE_NOSPEC_ALTERNATIVE
+ ALTERNATIVE "jmp .Lskip_rsb_\@", \
+ __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
+ \ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
#else /* __ASSEMBLY__ */
#define ANNOTATE_NOSPEC_ALTERNATIVE \
@@ -119,7 +175,7 @@
X86_FEATURE_RETPOLINE)
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
# define CALL_NOSPEC "call *%[thunk_target]\n"
# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
#endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
SPECTRE_V2_IBRS,
};
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+ unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+ asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+ ALTERNATIVE("jmp 910f",
+ __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+ X86_FEATURE_RETPOLINE)
+ "910:"
+ : "=&r" (loops), ASM_CALL_CONSTRAINT
+ : "r" (loops) : "memory" );
+#endif
+}
#endif /* __ASSEMBLY__ */
#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
#include <asm/debugreg.h>
#include <asm/kvm_para.h>
#include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
#include <asm/virtext.h>
#include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
#endif
);
+ /* Eliminate branch target predictions from guest mode */
+ vmexit_fill_RSB();
+
#ifdef CONFIG_X86_64
wrmsrl(MSR_GS_BASE, svm->host.gs_base);
#else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
#include <asm/apic.h>
#include <asm/irq_remapping.h>
#include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
#include "trace.h"
#include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
#endif
);
+ /* Eliminate branch target predictions from guest mode */
+ vmexit_fill_RSB();
+
/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
if (debugctlmsr)
update_debugctlmsr(debugctlmsr);
next prev parent reply other threads:[~2018-01-12 11:42 UTC|newest]
Thread overview: 85+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-01-11 21:46 [PATCH v8 00/12] Retpoline: Avoid speculative indirect calls in kernel David Woodhouse
2018-01-11 21:46 ` [PATCH v8 01/12] objtool: Detect jumps to retpoline thunks David Woodhouse
2018-01-11 23:22 ` [tip:x86/pti] " tip-bot for Josh Poimboeuf
2018-01-11 21:46 ` [PATCH v8 02/12] objtool: Allow alternatives to be ignored David Woodhouse
2018-01-11 23:22 ` [tip:x86/pti] " tip-bot for Josh Poimboeuf
2018-01-18 19:09 ` [v8,02/12] " Guenter Roeck
2018-01-18 19:33 ` Josh Poimboeuf
2018-01-18 19:41 ` Guenter Roeck
2018-01-22 19:34 ` David Woodhouse
2018-01-22 20:25 ` Guenter Roeck
2018-01-22 20:27 ` David Woodhouse
2018-01-28 21:06 ` Josh Poimboeuf
2018-01-29 1:26 ` Guenter Roeck
2018-01-29 17:15 ` Guenter Roeck
2018-01-29 17:30 ` Josh Poimboeuf
2018-01-22 19:27 ` Guenter Roeck
2018-01-11 21:46 ` [PATCH v8 03/12] x86/retpoline: Add initial retpoline support David Woodhouse
2018-01-11 23:23 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 23:58 ` [PATCH v8 03/12] " Tom Lendacky
2018-01-12 10:28 ` David Woodhouse
2018-01-12 14:02 ` Tom Lendacky
2018-01-14 15:02 ` Borislav Petkov
2018-01-14 15:53 ` Josh Poimboeuf
2018-01-14 15:59 ` Borislav Petkov
2018-01-11 21:46 ` [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation David Woodhouse
2018-01-11 23:23 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-23 22:40 ` [PATCH v8 04/12] " Borislav Petkov
2018-01-23 22:53 ` David Woodhouse
2018-01-23 23:05 ` Andi Kleen
2018-01-23 22:55 ` Jiri Kosina
2018-01-23 23:05 ` Borislav Petkov
2018-01-24 0:32 ` Kees Cook
2018-01-24 9:58 ` Borislav Petkov
2018-01-23 23:06 ` Jiri Kosina
2018-01-23 23:21 ` Andi Kleen
2018-01-23 23:24 ` Jiri Kosina
2018-01-23 23:45 ` Andi Kleen
2018-01-23 23:49 ` Jiri Kosina
2018-01-24 4:26 ` Greg Kroah-Hartman
2018-01-24 9:56 ` Jiri Kosina
2018-01-24 13:58 ` Greg Kroah-Hartman
2018-01-24 14:03 ` Jiri Kosina
2018-01-24 14:22 ` Greg Kroah-Hartman
2018-01-11 21:46 ` [PATCH v8 05/12] x86/retpoline/crypto: Convert crypto assembler indirect jumps David Woodhouse
2018-01-11 23:24 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 06/12] x86/retpoline/entry: Convert entry " David Woodhouse
2018-01-11 23:24 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 07/12] x86/retpoline/ftrace: Convert ftrace " David Woodhouse
2018-01-11 23:25 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 08/12] x86/retpoline/hyperv: Convert " David Woodhouse
2018-01-11 23:25 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 09/12] x86/retpoline/xen: Convert Xen hypercall " David Woodhouse
2018-01-11 23:25 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 10/12] x86/retpoline/checksum32: Convert assembler " David Woodhouse
2018-01-11 23:26 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 11/12] x86/retpoline/irq32: " David Woodhouse
2018-01-11 23:26 ` [tip:x86/pti] " tip-bot for Andi Kleen
2018-01-11 21:46 ` [PATCH v8 12/12] x86/retpoline: Fill return stack buffer on vmexit David Woodhouse
2018-01-11 23:27 ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 23:51 ` [PATCH v8 12/12] " Andi Kleen
2018-01-12 11:11 ` [PATCH v8.1 " David Woodhouse
2018-01-12 11:15 ` Thomas Gleixner
2018-01-12 11:21 ` Woodhouse, David
2018-01-12 11:37 ` tip-bot for David Woodhouse [this message]
2018-01-14 14:50 ` [tip:x86/pti] " Borislav Petkov
2018-01-14 15:28 ` Thomas Gleixner
2018-01-14 15:35 ` Borislav Petkov
2018-01-25 12:07 ` Borislav Petkov
2018-01-25 12:20 ` David Woodhouse
2018-01-25 12:45 ` Borislav Petkov
2018-01-25 15:10 ` Josh Poimboeuf
2018-01-25 15:51 ` Borislav Petkov
2018-01-25 16:03 ` David Woodhouse
2018-01-25 16:56 ` Josh Poimboeuf
2018-01-25 17:00 ` David Woodhouse
2018-01-25 17:05 ` Andy Lutomirski
2018-01-25 17:44 ` Josh Poimboeuf
2018-01-25 18:41 ` Jiri Kosina
2018-01-25 17:10 ` Thomas Gleixner
2018-01-25 17:32 ` Josh Poimboeuf
2018-01-25 17:53 ` Borislav Petkov
2018-01-25 18:04 ` David Woodhouse
2018-01-25 18:32 ` Josh Poimboeuf
2018-01-25 19:07 ` Borislav Petkov
2018-01-25 19:10 ` Borislav Petkov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=tip-117cc7a908c83697b0b737d15ae1eb5943afe35b@git.kernel.org \
--to=tipbot@zytor.com \
--cc=ak@linux.intel.com \
--cc=dave.hansen@intel.com \
--cc=dwmw@amazon.co.uk \
--cc=gregkh@linux-foundation.org \
--cc=hpa@zytor.com \
--cc=jikos@kernel.org \
--cc=jpoimboe@redhat.com \
--cc=keescook@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=luto@amacapital.net \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=pjt@google.com \
--cc=riel@redhat.com \
--cc=tglx@linutronix.de \
--cc=tim.c.chen@linux.intel.com \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).