[MODERATED] [PATCH 2/4] RFC-MDSv1 0

[Andi Kleen <andi@firstfloor.org>]

For MDS the CPU might leak previously touched data
in CPU internal structures. Make sure to clear these structures
every time we exit the kernel. This prevents any leakage
between user processes or between kernel and user.

The flushing is provided by new microcode as a new side
effect of the otherwise unused VERW instruction. We
add VERW to all the kernel exit paths.

We don't need to do this for guests because the L1TF cache
flush will implicitely do the same flushing, and is
automatically selected on MDS affected systems.

This mitigation doesn't address Hyper Threading.

So far this is for 64bit only, 32bit is not covered yet.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/entry/calling.h         | 8 ++++++++
 arch/x86/entry/entry_64.S        | 6 ++++++
 arch/x86/entry/entry_64_compat.S | 1 +
 3 files changed, 15 insertions(+)

diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 352e70cd33e8..e16f0f9f73cf 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -1,4 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/stringify.h>
+#include <asm/segment.h>
 #include <linux/jump_label.h>
 #include <asm/unwind_hints.h>
 #include <asm/cpufeatures.h>
@@ -329,6 +331,12 @@ For 32-bit we have the following conventions - kernel is built with
 
 #endif
 
+.macro EXIT_MDS
+	/* Clear CPU buffers that could leak. Instruction must be in memory form. */
+	ALTERNATIVE "", __stringify(pushq $__USER_DS ; verw (%rsp) ; addq $8, %rsp),\
+		 X86_FEATURE_MB_CLEAR
+.endm
+
 #endif /* CONFIG_X86_64 */
 
 /*
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index f95dcb209fdf..b6c768820c6c 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -325,6 +325,8 @@ syscall_return_via_sysret:
 	pushq	RSP-RDI(%rdi)	/* RSP */
 	pushq	(%rdi)		/* RDI */
 
+	EXIT_MDS
+
 	/*
 	 * We are on the trampoline stack.  All regs except RDI are live.
 	 * We can do future final exit work right here.
@@ -684,6 +686,8 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 	/* Push user RDI on the trampoline stack. */
 	pushq	(%rdi)
 
+	EXIT_MDS
+
 	/*
 	 * We are on the trampoline stack.  All regs except RDI are live.
 	 * We can do future final exit work right here.
@@ -806,6 +810,7 @@ native_irq_return_ldt:
 	orq	PER_CPU_VAR(espfix_stack), %rax
 
 	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
+	EXIT_MDS
 	SWAPGS					/* to user GS */
 	popq	%rdi				/* Restore user RDI */
 
@@ -1645,6 +1650,7 @@ end_repeat_nmi:
 	jnz	nmi_restore
 nmi_swapgs:
 	SWAPGS_UNSAFE_STACK
+	EXIT_MDS
 nmi_restore:
 	POP_REGS
 
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 7d0df78db727..d1abe8b7c605 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -271,6 +271,7 @@ sysret32_from_system_call:
 	popq	%rdx			/* Skip pt_regs->cx */
 	popq	%rdx			/* pt_regs->dx */
 	popq	%rsi			/* pt_regs->si */
+	EXIT_MDS
 	popq	%rdi			/* pt_regs->di */
 
         /*
-- 
2.17.2