[v2,01/12] x86/retpoline: Define retpoline indirect thunk and macros
diff mbox series

Message ID 20180104020019.1173-2-andi@firstfloor.org
State New, archived
Headers show
Series
  • [v2,01/12] x86/retpoline: Define retpoline indirect thunk and macros
Related show

Commit Message

Andi Kleen Jan. 4, 2018, 2 a.m. UTC
From: Dave Hansen <dave.hansen@linux.intel.com>

From: David Woodhouse <dwmw@amazon.co.uk>

retpoline is a special sequence on Intel CPUs to stop speculation for
indirect branches.

Provide assembler infrastructure to use retpoline by the compiler
and for assembler. We add the out of line trampoline used by the
compiler, and NOSPEC_JUMP / NOSPEC_CALL macros for assembler

[Originally from David and Tim, heavily hacked by AK]

v2: Add CONFIG_RETPOLINE option
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/Kconfig                |  8 +++++
 arch/x86/include/asm/jump-asm.h | 70 +++++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/vmlinux.lds.S   |  1 +
 arch/x86/lib/Makefile           |  1 +
 arch/x86/lib/retpoline.S        | 35 +++++++++++++++++++++
 5 files changed, 115 insertions(+)
 create mode 100644 arch/x86/include/asm/jump-asm.h
 create mode 100644 arch/x86/lib/retpoline.S

Comments

Brian Gerst Jan. 4, 2018, 2:15 a.m. UTC | #1
On Wed, Jan 3, 2018 at 9:00 PM, Andi Kleen <andi@firstfloor.org> wrote:
> From: Dave Hansen <dave.hansen@linux.intel.com>
>
> From: David Woodhouse <dwmw@amazon.co.uk>
>
> retpoline is a special sequence on Intel CPUs to stop speculation for
> indirect branches.
>
> Provide assembler infrastructure to use retpoline by the compiler
> and for assembler. We add the out of line trampoline used by the
> compiler, and NOSPEC_JUMP / NOSPEC_CALL macros for assembler
>
> [Originally from David and Tim, heavily hacked by AK]
>
> v2: Add CONFIG_RETPOLINE option
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
> Signed-off-by: Andi Kleen <ak@linux.intel.com>
> ---
>  arch/x86/Kconfig                |  8 +++++
>  arch/x86/include/asm/jump-asm.h | 70 +++++++++++++++++++++++++++++++++++++++++
>  arch/x86/kernel/vmlinux.lds.S   |  1 +
>  arch/x86/lib/Makefile           |  1 +
>  arch/x86/lib/retpoline.S        | 35 +++++++++++++++++++++
>  5 files changed, 115 insertions(+)
>  create mode 100644 arch/x86/include/asm/jump-asm.h
>  create mode 100644 arch/x86/lib/retpoline.S
>
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index d4fc98c50378..8b0facfa35be 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -429,6 +429,14 @@ config GOLDFISH
>         def_bool y
>         depends on X86_GOLDFISH
>
> +config RETPOLINE
> +       bool "Avoid speculative indirect branches in kernel"
> +       default y
> +       help
> +         Compile kernel with the retpoline compiler options to guard against
> +         kernel to user data leaks by avoiding speculative indirect
> +         branches. Requires a new enough compiler. The kernel may run slower.
> +
>  config INTEL_RDT
>         bool "Intel Resource Director Technology support"
>         default n
> diff --git a/arch/x86/include/asm/jump-asm.h b/arch/x86/include/asm/jump-asm.h
> new file mode 100644
> index 000000000000..936fa620f346
> --- /dev/null
> +++ b/arch/x86/include/asm/jump-asm.h
> @@ -0,0 +1,70 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef JUMP_ASM_H
> +#define JUMP_ASM_H 1
> +
> +#ifdef __ASSEMBLY__
> +
> +#ifdef CONFIG_RETPOLINE
> +
> +/*
> + * Jump to an indirect pointer without speculation.
> + *
> + * The out of line __x86.indirect_thunk has special code sequences
> + * to stop speculation.
> + */
> +
> +.macro NOSPEC_JMP target
> +       push    \target
> +       jmp     __x86.indirect_thunk
> +.endm
> +
> +
> +/*
> + * Call an indirect pointer without speculation.
> + */
> +
> +.macro NOSPEC_CALL target
> +       jmp     1221f
> +1222:
> +       push    \target
> +       jmp     __x86.indirect_thunk
> +1221:
> +       call    1222b
> +.endm
> +
> +#else /* CONFIG_RETPOLINE */
> +
> +.macro NOSPEC_JMP target
> +       jmp *\target
> +.endm
> +
> +.macro NOSPEC_CALL target
> +       call *\target
> +.endm
> +
> +#endif /* !CONFIG_RETPOLINE */
> +
> +#else /* __ASSEMBLY__ */
> +
> +#ifdef CONFIG_RETPOLINE
> +
> +#define NOSPEC_JMP(t) \
> +       "push " t "; "                          \
> +       "jmp __x86.indirect_thunk; "
> +
> +#define NOSPEC_CALL(t) \
> +       "       jmp 1221f; "                    \
> +       "1222:  push " t ";"                    \
> +       "       jmp __x86.indirect_thunk;"      \
> +       "1221:  call 1222b;"
> +
> +#else /* CONFIG_RETPOLINE */
> +
> +#define NOSPEC_JMP(t) "jmp *" t "; "
> +#define NOSPEC_CALL(t) "call *" t "; "
> +
> +#endif /* !CONFIG_RETPOLINE */
> +
> +#endif /* !__ASSEMBLY */
> +
> +#endif
> diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
> index 1e413a9326aa..2e64241a6664 100644
> --- a/arch/x86/kernel/vmlinux.lds.S
> +++ b/arch/x86/kernel/vmlinux.lds.S
> @@ -103,6 +103,7 @@ SECTIONS
>                 /* bootstrapping code */
>                 HEAD_TEXT
>                 . = ALIGN(8);
> +               *(.text.__x86.indirect_thunk)
>                 TEXT_TEXT
>                 SCHED_TEXT
>                 CPUIDLE_TEXT
> diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
> index 7b181b61170e..f23934bbaf4e 100644
> --- a/arch/x86/lib/Makefile
> +++ b/arch/x86/lib/Makefile
> @@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
>  lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
>  lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
>  lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
> +lib-$(CONFIG_RETPOLINE) += retpoline.o
>
>  obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
>
> diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
> new file mode 100644
> index 000000000000..cb40781adbfe
> --- /dev/null
> +++ b/arch/x86/lib/retpoline.S
> @@ -0,0 +1,35 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +/*
> + * Out of line jump trampoline for calls that disable speculation.
> + *
> + * This is a special sequence that prevents the CPU speculating
> + * for indirect calls.
> + *
> + * This can be called by gcc generated code, or with the asm macros
> + * in asm/jump-asm.h
> + */
> +
> +#include <linux/linkage.h>
> +#include <asm/dwarf2.h>
> +#include <asm/export.h>
> +
> +       .section        .text.__x86.indirect_thunk,"ax"
> +
> +ENTRY(__x86.indirect_thunk)
> +       CFI_STARTPROC
> +       call    retpoline_call_target
> +2:
> +       lfence          /* stop speculation */
> +       jmp     2b
> +retpoline_call_target:
> +#ifdef CONFIG_64BIT
> +       lea     8(%rsp), %rsp
> +#else
> +       lea     4(%esp), %esp
> +#endif
> +       ret
> +       CFI_ENDPROC
> +ENDPROC(__x86.indirect_thunk)
> +
> +       EXPORT_SYMBOL(__x86.indirect_thunk)
> --
> 2.14.3
>

Can someone actually explain WTF this mess is trying to accomplish?

--
Brian Gerst
Alan Cox Jan. 4, 2018, 2:32 a.m. UTC | #2
> > +ENTRY(__x86.indirect_thunk)
> > +       CFI_STARTPROC
> > +       call    retpoline_call_target
> > +2:
> > +       lfence          /* stop speculation */
> > +       jmp     2b
> > +retpoline_call_target:
> > +#ifdef CONFIG_64BIT
> > +       lea     8(%rsp), %rsp
> > +#else
> > +       lea     4(%esp), %esp
> > +#endif
> > +       ret
> > +       CFI_ENDPROC
> > +ENDPROC(__x86.indirect_thunk)
> > +
> > +       EXPORT_SYMBOL(__x86.indirect_thunk)
> > --
> > 2.14.3
> >  
> 
> Can someone actually explain WTF this mess is trying to accomplish?

Think of it as an 'indirect call that doesn't speculate' instruction.
There isn't one in the processor but this specific sequence happens to
make the micro-architecture do just that as efficiently as possible.

What it's actually doing on the non-speculated path (ie the reachable
code) is to call, put the address we want to hit over the existing return
address and then return, to the address we want to indirectly go to.

It's faster than doing a far branch or flushing branch predictors and the
like.

Alan

Patch
diff mbox series

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d4fc98c50378..8b0facfa35be 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,14 @@  config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel to user data leaks by avoiding speculative indirect
+	  branches. Requires a new enough compiler. The kernel may run slower.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/include/asm/jump-asm.h b/arch/x86/include/asm/jump-asm.h
new file mode 100644
index 000000000000..936fa620f346
--- /dev/null
+++ b/arch/x86/include/asm/jump-asm.h
@@ -0,0 +1,70 @@ 
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef JUMP_ASM_H
+#define JUMP_ASM_H 1
+
+#ifdef __ASSEMBLY__
+
+#ifdef CONFIG_RETPOLINE
+
+/*
+ * Jump to an indirect pointer without speculation.
+ *
+ * The out of line __x86.indirect_thunk has special code sequences
+ * to stop speculation.
+ */
+
+.macro NOSPEC_JMP target
+	push	\target
+	jmp	__x86.indirect_thunk
+.endm
+
+
+/*
+ * Call an indirect pointer without speculation.
+ */
+
+.macro NOSPEC_CALL target
+	jmp     1221f
+1222:
+	push	\target
+	jmp	__x86.indirect_thunk
+1221:
+	call	1222b
+.endm
+
+#else /* CONFIG_RETPOLINE */
+
+.macro NOSPEC_JMP target
+	jmp *\target
+.endm
+
+.macro NOSPEC_CALL target
+	call *\target
+.endm
+
+#endif /* !CONFIG_RETPOLINE */
+
+#else /* __ASSEMBLY__ */
+
+#ifdef CONFIG_RETPOLINE
+
+#define NOSPEC_JMP(t) \
+	"push " t "; "				\
+	"jmp __x86.indirect_thunk; "
+
+#define NOSPEC_CALL(t) \
+	"	jmp 1221f; "			\
+	"1222:	push " t ";"			\
+	"	jmp __x86.indirect_thunk;"	\
+	"1221:	call 1222b;"
+
+#else /* CONFIG_RETPOLINE */
+
+#define NOSPEC_JMP(t) "jmp *" t "; "
+#define NOSPEC_CALL(t) "call *" t "; "
+
+#endif /* !CONFIG_RETPOLINE */
+
+#endif /* !__ASSEMBLY */
+
+#endif
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 1e413a9326aa..2e64241a6664 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -103,6 +103,7 @@  SECTIONS
 		/* bootstrapping code */
 		HEAD_TEXT
 		. = ALIGN(8);
+		*(.text.__x86.indirect_thunk)
 		TEXT_TEXT
 		SCHED_TEXT
 		CPUIDLE_TEXT
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 7b181b61170e..f23934bbaf4e 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@  lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 000000000000..cb40781adbfe
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,35 @@ 
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Out of line jump trampoline for calls that disable speculation.
+ *
+ * This is a special sequence that prevents the CPU speculating
+ * for indirect calls.
+ *
+ * This can be called by gcc generated code, or with the asm macros
+ * in asm/jump-asm.h
+ */
+
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/export.h>
+
+	.section	.text.__x86.indirect_thunk,"ax"
+
+ENTRY(__x86.indirect_thunk)
+	CFI_STARTPROC
+	call	retpoline_call_target
+2:
+	lfence		/* stop speculation */
+	jmp	2b
+retpoline_call_target:
+#ifdef CONFIG_64BIT
+	lea	8(%rsp), %rsp
+#else
+	lea	4(%esp), %esp
+#endif
+	ret
+	CFI_ENDPROC
+ENDPROC(__x86.indirect_thunk)
+
+	EXPORT_SYMBOL(__x86.indirect_thunk)