Re: [PATCH v3 01/13] x86/retpoline: Add initial retpoline support

["Woodhouse, David" <dwmw@amazon.co.uk>]

[-- Attachment #1: Type: text/plain, Size: 8768 bytes --]

On Fri, 2018-01-05 at 09:28 -0800, Linus Torvalds wrote:
> That said, I honestly like the inline version (the one that is in the
> google paper first) of the retpoline more than the out-of-line one.
> And that one shouldn't have any relocagtion issues, because all the
> offsets are relative.
> 
> We want to use that one for the entry stub anyway, can't we just
> standardize on that one for all our assembly?
> 
> If the *compiler* uses the out-of-line version, that's a separate
> thing. But for our asm cases, let's just make it all be the inline
> case, ok?

OK, this one looks saner, and I think I've tested all the 32/64 bit
retpoline/amd/none permutations. Pushed to
http://git.infradead.org/users/dwmw2/linux-retpoline.git/

If this matches what you were thinking, I'll refactor the series in the
morning to do it this way.

From cfddb3bcae1524da52e782398da2809ec8faa200 Mon Sep 17 00:00:00 2001
From: David Woodhouse <dwmw@amazon.co.uk>
Date: Fri, 5 Jan 2018 21:50:41 +0000
Subject: [PATCH] x86/retpoline: Clean up inverted X86_FEATURE_NO_RETPOLINE
 logic

If we make the thunks inline so they don't need relocations to branch to
__x86.indirect_thunk.xxx then we don't fall foul of the alternatives
handling, and we can have the retpoline variant in 'altinstr'. This
means that we can use X86_FEATURE_RETPOLINE which is more natural.

Unfortunately, it does mean that the X86_FEATURE_K8 trick doesn't work
any more, so we need an additional X86_FEATURE_RETPOLINE_AMD for that.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 arch/x86/entry/entry_64.S            | 12 +--------
 arch/x86/include/asm/cpufeatures.h   |  3 ++-
 arch/x86/include/asm/nospec-branch.h | 50 +++++++++++++++++++++++++++---------
 arch/x86/kernel/cpu/common.c         |  5 ++++
 arch/x86/kernel/cpu/intel.c          |  3 ++-
 arch/x86/lib/retpoline.S             | 17 +++++-------
 6 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 76f94bbacaec..8f7e1129f493 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -188,17 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 	 */
 	pushq	%rdi
 	movq	$entry_SYSCALL_64_stage2, %rdi
-	/*
-	 * Open-code the retpoline from retpoline.S, because we can't
-	 * just jump to it directly.
-	 */
-	ALTERNATIVE "call 2f", "jmp *%rdi", X86_FEATURE_NO_RETPOLINE
-1:
-	lfence
-	jmp	1b
-2:
-	mov	%rdi, (%rsp)
-	ret
+	NOSPEC_JMP rdi
 END(entry_SYSCALL_64_trampoline)
 
 	.popsection
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 2d916fd13bf9..6f10edabbf82 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,7 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
-#define X86_FEATURE_NO_RETPOLINE	( 7*32+12) /* Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Intel Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index eced0dfaddc9..6e92edf64e53 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -8,23 +8,44 @@
 #include <asm/cpufeatures.h>
 
 #ifdef __ASSEMBLY__
+
 /*
- * The asm code uses CONFIG_RETPOLINE; this part will happen even if
- * the toolchain isn't retpoline-capable.
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly.
  */
+.macro RETPOLINE_JMP reg:req
+	call	1112f
+1111:	lfence
+	jmp	1111b
+1112:	mov	%\reg, (%_ASM_SP)
+	ret
+.endm
+
+.macro RETPOLINE_CALL reg:req
+	jmp	1113f
+1110:	RETPOLINE_JMP \reg
+1113:	call	1110b
+.endm
+
+
+
 .macro NOSPEC_JMP reg:req
 #ifdef CONFIG_RETPOLINE
-	ALTERNATIVE __stringify(jmp __x86.indirect_thunk.\reg), __stringify(jmp *%\reg), X86_FEATURE_NO_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *%\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD
 #else
-	jmp *%\reg
+	jmp	*%\reg
 #endif
 .endm
 
 .macro NOSPEC_CALL reg:req
 #ifdef CONFIG_RETPOLINE
-	ALTERNATIVE __stringify(call __x86.indirect_thunk.\reg), __stringify(call *%\reg), X86_FEATURE_NO_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *%\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *%\reg), X86_FEATURE_RETPOLINE_AMD
 #else
-	call *%\reg
+	call	*%\reg
 #endif
 .endm
 
@@ -36,16 +57,21 @@
  */
 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
 #  define NOSPEC_CALL ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
 	"call __x86.indirect_thunk.%V[thunk_target]\n",		\
-	"call *%[thunk_target]\n", X86_FEATURE_NO_RETPOLINE)
+	X86_FEATURE_RETPOLINE)
 #  define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 #elif defined(CONFIG_X86_64) && defined(CONFIG_RETPOLINE)
 # define NOSPEC_CALL ALTERNATIVE(				\
-	"       jmp 1221f; "					\
-	"1222:  push %[thunk_target];"				\
-	"       jmp __x86.indirect_thunk;"			\
-	"1221:  call 1222b;\n",					\
-	"call *%[thunk_target]\n", X86_FEATURE_NO_RETPOLINE)
+	"call	*%[thunk_target]\n",				\
+	"       jmp    1113f; "					\
+	"1110:  call   1112f; "					\
+	"1111:	lfence; "					\
+	"       jmp    1111b; "					\
+	"1112:	movl   %[thunk_target], (%esp); "		\
+	"       ret; "						\
+	"1113:  call   1110b;\n",				\
+	X86_FEATURE_RETPOLINE)
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #else
 # define NOSPEC_CALL "call *%[thunk_target]\n"
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3fb400f..40e6e54d8501 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -904,6 +904,11 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+	if (c->x86_vendor == X86_VENDOR_AMD)
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
+#endif
 
 	fpu__init_system(c);
 
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index e1812d07b53e..35e123e5f413 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -35,7 +35,8 @@
 static int __init noretpoline_setup(char *__unused)
 {
 	pr_info("Retpoline runtime disabled\n");
-	setup_force_cpu_cap(X86_FEATURE_NO_RETPOLINE);
+	setup_clear_cpu_cap(X86_FEATURE_RETPOLINE);
+	setup_clear_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 	return 1;
 }
 __setup("noretpoline", noretpoline_setup);
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index 2a4b1f09eb84..90d9a1589a54 100644
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -6,19 +6,14 @@
 #include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 #include <asm/export.h>
+#include <asm/nospec-branch.h>
 
-.macro THUNK sp reg
+.macro THUNK reg
 	.section .text.__x86.indirect_thunk.\reg
 
 ENTRY(__x86.indirect_thunk.\reg)
 	CFI_STARTPROC
-	ALTERNATIVE_2 "call 2f", __stringify(lfence;jmp *%\reg), X86_FEATURE_K8, __stringify(jmp *%\reg), X86_FEATURE_NO_RETPOLINE
-1:
-	lfence
-	jmp	1b
-2:
-	mov	%\reg, (%\sp)
-	ret
+	NOSPEC_JMP \reg
 	CFI_ENDPROC
 ENDPROC(__x86.indirect_thunk.\reg)
 EXPORT_SYMBOL(__x86.indirect_thunk.\reg)
@@ -26,11 +21,11 @@ EXPORT_SYMBOL(__x86.indirect_thunk.\reg)
 
 #ifdef CONFIG_64BIT
 .irp reg rax rbx rcx rdx rsi rdi rbp r8 r9 r10 r11 r12 r13 r14 r15
-	THUNK rsp \reg
+	THUNK \reg
 .endr
 #else
 .irp reg eax ebx ecx edx esi edi ebp
-	THUNK esp \reg
+	THUNK \reg
 .endr
 
 /*
@@ -39,7 +34,7 @@ EXPORT_SYMBOL(__x86.indirect_thunk.\reg)
  */
 ENTRY(__x86.indirect_thunk)
 	CFI_STARTPROC
-	ALTERNATIVE "call 2f", "ret", X86_FEATURE_NO_RETPOLINE
+	ALTERNATIVE "ret", "call 2f", X86_FEATURE_RETPOLINE
 1:
 	lfence
 	jmp	1b
-- 
2.14.3


[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5210 bytes --]