[MODERATED] [PATCH v2 2/8] MDSv2 1

[Andi Kleen <andi@firstfloor.org>]

For MDS the CPU might leak previously touched data
in CPU internal structures. Make sure to clear these structures
every time we exit the kernel. This prevents any leakage
between user processes or between kernel and user.

The flushing is provided by new microcode as a new side
effect of the otherwise unused VERW instruction. We
add VERW to all the kernel exit paths.

We don't need to do this for guests because the L1TF cache
flush will implicitely do the same flushing, and is
automatically selected on MDS affected systems.

This mitigation doesn't address Hyper Threading.

So far this is for 64bit only, 32bit is not covered yet.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/entry/calling.h             | 2 ++
 arch/x86/entry/entry_64.S            | 8 ++++++++
 arch/x86/entry/entry_64_compat.S     | 1 +
 arch/x86/include/asm/nospec-branch.h | 8 ++++++++
 4 files changed, 19 insertions(+)

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 25e5a6bda8c3..4b07f97e3874 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -1,4 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/stringify.h>
+#include <asm/segment.h>
 #include <linux/jump_label.h>
 #include <asm/unwind_hints.h>
 #include <asm/cpufeatures.h>
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ce25d84023c0..0bb7bb3dc728 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -248,6 +248,9 @@ GLOBAL(entry_SYSCALL_64_after_hwframe)
 	 * perf profiles. Nothing jumps here.
 	 */
 syscall_return_via_sysret:
+
+	EXIT_MDS
+
 	/* rcx and r11 are already restored (see code above) */
 	UNWIND_HINT_EMPTY
 	POP_REGS pop_rdi=0 skip_r11rcx=1
@@ -604,6 +607,8 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 	ud2
 1:
 #endif
+	EXIT_MDS
+
 	POP_REGS pop_rdi=0
 
 	/*
@@ -623,6 +628,7 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 	/* Push user RDI on the trampoline stack. */
 	pushq	(%rdi)
 
+
 	/*
 	 * We are on the trampoline stack.  All regs except RDI are live.
 	 * We can do future final exit work right here.
@@ -1616,6 +1622,8 @@ end_repeat_nmi:
 	movq	$-1, %rsi
 	call	do_nmi
 
+	EXIT_MDS
+
 	/* Always restore stashed CR3 value (see paranoid_entry) */
 	RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
 
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 8eaf8952c408..8b5a11c6a32e 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -276,6 +276,7 @@ sysret32_from_system_call:
 	popq	%rdx			/* Skip pt_regs->cx */
 	popq	%rdx			/* pt_regs->dx */
 	popq	%rsi			/* pt_regs->si */
+	EXIT_MDS
 	popq	%rdi			/* pt_regs->di */
 
         /*
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 032b6009baab..f780f29e351f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -150,6 +150,14 @@
 #endif
 .endm
 
+.macro EXIT_MDS
+	/* Clear CPU buffers that could leak. Instruction must be in memory form. */
+	ALTERNATIVE_2 "", __stringify(push $__USER_DS ; verw (% _ASM_SP ) ; add $8, % _ASM_SP ),\
+		X86_FEATURE_MB_CLEAR, \
+		"call do_clear_cpu", \
+		X86_BUG_MDS_CLEAR_CPU
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
-- 
2.17.2