LKML Archive on lore.kernel.org
 help / color / Atom feed
From: David Woodhouse <dwmw@amazon.co.uk>
To: Andi Kleen <ak@linux.intel.com>
Cc: Paul Turner <pjt@google.com>, LKML <linux-kernel@vger.kernel.org>,
	Linus Torvalds <torvalds@linux-foundation.org>,
	Greg Kroah-Hartman <gregkh@linux-foundation.org>,
	Tim Chen <tim.c.chen@linux.intel.com>,
	Dave Hansen <dave.hansen@intel.com>,
	tglx@linutronix.de, Kees Cook <keescook@google.com>,
	Rik van Riel <riel@redhat.com>,
	Peter Zijlstra <peterz@infradead.org>,
	Andy Lutomirski <luto@amacapital.net>,
	Jiri Kosina <jikos@kernel.org>,
	gnomes@lxorguk.ukuu.org.uk, x86@kernel.org,
	thomas.lendacky@amd.com, Josh Poimboeuf <jpoimboe@redhat.com>
Subject: [PATCH v8.1 12/12] x86/retpoline: Fill return stack buffer on vmexit
Date: Fri, 12 Jan 2018 11:11:27 +0000
Message-ID: <1515755487-8524-1-git-send-email-dwmw@amazon.co.uk> (raw)
In-Reply-To: <20180111235117.GO6718@tassilo.jf.intel.com>

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
I love the smell of bikeshed paint in the morning. But to be fair, this
one was actually an issue which might possibly have bitten in the future.

Can we please stop arguing about asm labels now though? Let's get this
stuff done, and we can set about the oh-so-important task of persuading
Linus to eliminate all numeric labels and rely on human-readable labels
with %= and \@ to make them unique, some time after the dust settles.

 arch/x86/include/asm/nospec-branch.h | 78 +++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm.c                   |  4 ++
 arch/x86/kvm/vmx.c                   |  4 ++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS		16	/* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
+	mov	$(nr/2), reg;			\
+771:						\
+	call	772f;				\
+773:	/* speculation trap */			\
+	pause;					\
+	jmp	773b;				\
+772:						\
+	call	774f;				\
+775:	/* speculation trap */			\
+	pause;					\
+	jmp	775b;				\
+774:						\
+	dec	reg;				\
+	jnz	771b;				\
+	add	$(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +118,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
+		\ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
@@ -119,7 +175,7 @@
 	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+	unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+		      ALTERNATIVE("jmp 910f",
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+				  X86_FEATURE_RETPOLINE)
+		      "910:"
+		      : "=&r" (loops), ASM_CALL_CONSTRAINT
+		      : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 #ifdef CONFIG_X86_64
 	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 
 #include "trace.h"
 #include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
 	if (debugctlmsr)
 		update_debugctlmsr(debugctlmsr);
-- 
2.7.4

  reply index

Thread overview: 85+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-01-11 21:46 [PATCH v8 00/12] Retpoline: Avoid speculative indirect calls in kernel David Woodhouse
2018-01-11 21:46 ` [PATCH v8 01/12] objtool: Detect jumps to retpoline thunks David Woodhouse
2018-01-11 23:22   ` [tip:x86/pti] " tip-bot for Josh Poimboeuf
2018-01-11 21:46 ` [PATCH v8 02/12] objtool: Allow alternatives to be ignored David Woodhouse
2018-01-11 23:22   ` [tip:x86/pti] " tip-bot for Josh Poimboeuf
2018-01-18 19:09   ` [v8,02/12] " Guenter Roeck
2018-01-18 19:33     ` Josh Poimboeuf
2018-01-18 19:41       ` Guenter Roeck
2018-01-22 19:34         ` David Woodhouse
2018-01-22 20:25           ` Guenter Roeck
2018-01-22 20:27             ` David Woodhouse
2018-01-28 21:06             ` Josh Poimboeuf
2018-01-29  1:26               ` Guenter Roeck
2018-01-29 17:15               ` Guenter Roeck
2018-01-29 17:30                 ` Josh Poimboeuf
2018-01-22 19:27       ` Guenter Roeck
2018-01-11 21:46 ` [PATCH v8 03/12] x86/retpoline: Add initial retpoline support David Woodhouse
2018-01-11 23:23   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 23:58   ` [PATCH v8 03/12] " Tom Lendacky
2018-01-12 10:28     ` David Woodhouse
2018-01-12 14:02       ` Tom Lendacky
2018-01-14 15:02   ` Borislav Petkov
2018-01-14 15:53     ` Josh Poimboeuf
2018-01-14 15:59       ` Borislav Petkov
2018-01-11 21:46 ` [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation David Woodhouse
2018-01-11 23:23   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-23 22:40   ` [PATCH v8 04/12] " Borislav Petkov
2018-01-23 22:53     ` David Woodhouse
2018-01-23 23:05       ` Andi Kleen
2018-01-23 22:55     ` Jiri Kosina
2018-01-23 23:05       ` Borislav Petkov
2018-01-24  0:32         ` Kees Cook
2018-01-24  9:58           ` Borislav Petkov
2018-01-23 23:06       ` Jiri Kosina
2018-01-23 23:21       ` Andi Kleen
2018-01-23 23:24         ` Jiri Kosina
2018-01-23 23:45           ` Andi Kleen
2018-01-23 23:49             ` Jiri Kosina
2018-01-24  4:26               ` Greg Kroah-Hartman
2018-01-24  9:56                 ` Jiri Kosina
2018-01-24 13:58                   ` Greg Kroah-Hartman
2018-01-24 14:03                     ` Jiri Kosina
2018-01-24 14:22                       ` Greg Kroah-Hartman
2018-01-11 21:46 ` [PATCH v8 05/12] x86/retpoline/crypto: Convert crypto assembler indirect jumps David Woodhouse
2018-01-11 23:24   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 06/12] x86/retpoline/entry: Convert entry " David Woodhouse
2018-01-11 23:24   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 07/12] x86/retpoline/ftrace: Convert ftrace " David Woodhouse
2018-01-11 23:25   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 08/12] x86/retpoline/hyperv: Convert " David Woodhouse
2018-01-11 23:25   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 09/12] x86/retpoline/xen: Convert Xen hypercall " David Woodhouse
2018-01-11 23:25   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 10/12] x86/retpoline/checksum32: Convert assembler " David Woodhouse
2018-01-11 23:26   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 21:46 ` [PATCH v8 11/12] x86/retpoline/irq32: " David Woodhouse
2018-01-11 23:26   ` [tip:x86/pti] " tip-bot for Andi Kleen
2018-01-11 21:46 ` [PATCH v8 12/12] x86/retpoline: Fill return stack buffer on vmexit David Woodhouse
2018-01-11 23:27   ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-11 23:51   ` [PATCH v8 12/12] " Andi Kleen
2018-01-12 11:11     ` David Woodhouse [this message]
2018-01-12 11:15       ` [PATCH v8.1 " Thomas Gleixner
2018-01-12 11:21         ` Woodhouse, David
2018-01-12 11:37       ` [tip:x86/pti] " tip-bot for David Woodhouse
2018-01-14 14:50         ` Borislav Petkov
2018-01-14 15:28           ` Thomas Gleixner
2018-01-14 15:35         ` Borislav Petkov
2018-01-25 12:07         ` Borislav Petkov
2018-01-25 12:20           ` David Woodhouse
2018-01-25 12:45             ` Borislav Petkov
2018-01-25 15:10               ` Josh Poimboeuf
2018-01-25 15:51                 ` Borislav Petkov
2018-01-25 16:03                   ` David Woodhouse
2018-01-25 16:56                     ` Josh Poimboeuf
2018-01-25 17:00                       ` David Woodhouse
2018-01-25 17:05                         ` Andy Lutomirski
2018-01-25 17:44                           ` Josh Poimboeuf
2018-01-25 18:41                           ` Jiri Kosina
2018-01-25 17:10                         ` Thomas Gleixner
2018-01-25 17:32                         ` Josh Poimboeuf
2018-01-25 17:53                         ` Borislav Petkov
2018-01-25 18:04                           ` David Woodhouse
2018-01-25 18:32                             ` Josh Poimboeuf
2018-01-25 19:07                             ` Borislav Petkov
2018-01-25 19:10                               ` Borislav Petkov

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1515755487-8524-1-git-send-email-dwmw@amazon.co.uk \
    --to=dwmw@amazon.co.uk \
    --cc=ak@linux.intel.com \
    --cc=dave.hansen@intel.com \
    --cc=gnomes@lxorguk.ukuu.org.uk \
    --cc=gregkh@linux-foundation.org \
    --cc=jikos@kernel.org \
    --cc=jpoimboe@redhat.com \
    --cc=keescook@google.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=luto@amacapital.net \
    --cc=peterz@infradead.org \
    --cc=pjt@google.com \
    --cc=riel@redhat.com \
    --cc=tglx@linutronix.de \
    --cc=thomas.lendacky@amd.com \
    --cc=tim.c.chen@linux.intel.com \
    --cc=torvalds@linux-foundation.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

LKML Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/lkml/0 lkml/git/0.git
	git clone --mirror https://lore.kernel.org/lkml/1 lkml/git/1.git
	git clone --mirror https://lore.kernel.org/lkml/2 lkml/git/2.git
	git clone --mirror https://lore.kernel.org/lkml/3 lkml/git/3.git
	git clone --mirror https://lore.kernel.org/lkml/4 lkml/git/4.git
	git clone --mirror https://lore.kernel.org/lkml/5 lkml/git/5.git
	git clone --mirror https://lore.kernel.org/lkml/6 lkml/git/6.git
	git clone --mirror https://lore.kernel.org/lkml/7 lkml/git/7.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 lkml lkml/ https://lore.kernel.org/lkml \
		linux-kernel@vger.kernel.org
	public-inbox-index lkml

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git