[PATCH v8 00/12] Retpoline: Avoid speculative indirect calls in kernel

[PATCH v8 00/12] Retpoline: Avoid speculative indirect calls in kernel

[David Woodhouse]

This is a mitigation for the 'variant 2' attack described in
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html

Using GCC patches available from the hjl/indirect/gcc-7-branch/master
branch of https://github.com/hjl-tools/gcc/commits/hjl and by manually
patching assembler code, all vulnerable indirect branches (that occur
after userspace first runs) are eliminated from the kernel.

They are replaced with a 'retpoline' call sequence which deliberately
prevents speculation.

Fedora 27 packages of the updated compiler are available at
https://koji.fedoraproject.org/koji/taskinfo?taskID=24065739


v1: Initial post.
v2: Add CONFIG_RETPOLINE to build kernel without it.
    Change warning messages.
    Hide modpost warning message
v3: Update to the latest CET-capable retpoline version
    Reinstate ALTERNATIVE support
v4: Finish reconciling Andi's and my patch sets, bug fixes.
    Exclude objtool support for now
    Add 'noretpoline' boot option
    Add AMD retpoline alternative
v5: Silence MODVERSIONS warnings
    Use pause;jmp loop instead of lfence;jmp
    Switch to X86_FEATURE_RETPOLINE positive feature logic
    Emit thunks inline from assembler macros
    Merge AMD support into initial patch
v6: Update to latest GCC patches with no dots in symbols
    Fix MODVERSIONS properly(ish)
    Fix typo breaking 32-bit, introduced in V5
    Never set X86_FEATURE_RETPOLINE_AMD yet, pending confirmation
v7: Further bikeshedding on macro names
    Stuff RSB on kernel entry
    Implement 'spectre_v2=' command line option for IBRS/IBPB too
    Revert to precisely the asm sequences from the Google paper
v8: Re-enable (I won't say "fix") objtool support
    Use numeric labels for GCC compatibility
    Add support for RSB-stuffing on vmexit
    I don't know... other bloody bikeshedding. Can I sleep now?

Andi Kleen (1):
  x86/retpoline/irq32: Convert assembler indirect jumps

David Woodhouse (10):
  objtool: Allow alternatives to be ignored
  x86/retpoline: Add initial retpoline support
  x86/spectre: Add boot time option to select Spectre v2 mitigation
  x86/retpoline/crypto: Convert crypto assembler indirect jumps
  x86/retpoline/entry: Convert entry assembler indirect jumps
  x86/retpoline/ftrace: Convert ftrace assembler indirect jumps
  x86/retpoline/hyperv: Convert assembler indirect jumps
  x86/retpoline/xen: Convert Xen hypercall indirect jumps
  x86/retpoline/checksum32: Convert assembler indirect jumps
  x86/retpoline: Fill return stack buffer on vmexit

Josh Poimboeuf (1):
  objtool: Detect jumps to retpoline thunks

 Documentation/admin-guide/kernel-parameters.txt |  28 ++++
 arch/x86/Kconfig                                |  13 ++
 arch/x86/Makefile                               |  10 ++
 arch/x86/crypto/aesni-intel_asm.S               |   5 +-
 arch/x86/crypto/camellia-aesni-avx-asm_64.S     |   3 +-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S    |   3 +-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S       |   3 +-
 arch/x86/entry/entry_32.S                       |   5 +-
 arch/x86/entry/entry_64.S                       |  12 +-
 arch/x86/include/asm/asm-prototypes.h           |  25 +++
 arch/x86/include/asm/cpufeatures.h              |   2 +
 arch/x86/include/asm/mshyperv.h                 |  18 +-
 arch/x86/include/asm/nospec-branch.h            | 209 ++++++++++++++++++++++++
 arch/x86/include/asm/xen/hypercall.h            |   5 +-
 arch/x86/kernel/cpu/bugs.c                      | 158 +++++++++++++++++-
 arch/x86/kernel/ftrace_32.S                     |   6 +-
 arch/x86/kernel/ftrace_64.S                     |   8 +-
 arch/x86/kernel/irq_32.c                        |   9 +-
 arch/x86/kvm/svm.c                              |   4 +
 arch/x86/kvm/vmx.c                              |   4 +
 arch/x86/lib/Makefile                           |   1 +
 arch/x86/lib/checksum_32.S                      |   7 +-
 arch/x86/lib/retpoline.S                        |  48 ++++++
 tools/objtool/check.c                           |  69 +++++++-
 tools/objtool/check.h                           |   2 +-
 25 files changed, 616 insertions(+), 41 deletions(-)
 create mode 100644 arch/x86/include/asm/nospec-branch.h
 create mode 100644 arch/x86/lib/retpoline.S

-- 
2.7.4

[PATCH v8 01/12] objtool: Detect jumps to retpoline thunks

[David Woodhouse]

From: Josh Poimboeuf <jpoimboe@redhat.com>

A direct jump to a retpoline thunk is really an indirect jump in
disguise.  Change the objtool instruction type accordingly.

Objtool needs to know where indirect branches are so it can detect
switch statement jump tables.

This fixes a bunch of warnings with CONFIG_RETPOLINE like:

  arch/x86/events/intel/uncore_nhmex.o: warning: objtool: nhmex_rbox_msr_enable_event()+0x44: sibling call from callable instruction with modified stack frame
  kernel/signal.o: warning: objtool: copy_siginfo_to_user()+0x91: sibling call from callable instruction with modified stack frame
  ...

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 tools/objtool/check.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 9b341584..de053fb 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -456,6 +456,13 @@ static int add_jump_destinations(struct objtool_file *file)
 		} else if (rela->sym->sec->idx) {
 			dest_sec = rela->sym->sec;
 			dest_off = rela->sym->sym.st_value + rela->addend + 4;
+		} else if (strstr(rela->sym->name, "_indirect_thunk_")) {
+			/*
+			 * Retpoline jumps are really dynamic jumps in
+			 * disguise, so convert them accordingly.
+			 */
+			insn->type = INSN_JUMP_DYNAMIC;
+			continue;
 		} else {
 			/* sibling call */
 			insn->jump_dest = 0;
-- 
2.7.4

[tip:x86/pti] objtool: Detect jumps to retpoline thunks

[tip-bot for Josh Poimboeuf]

Commit-ID:  39b735332cb8b33a27c28592d969e4016c86c3ea
Gitweb:     https://git.kernel.org/tip/39b735332cb8b33a27c28592d969e4016c86c3ea
Author:     Josh Poimboeuf <jpoimboe@redhat.com>
AuthorDate: Thu, 11 Jan 2018 21:46:23 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:28 +0100

objtool: Detect jumps to retpoline thunks

A direct jump to a retpoline thunk is really an indirect jump in
disguise.  Change the objtool instruction type accordingly.

Objtool needs to know where indirect branches are so it can detect
switch statement jump tables.

This fixes a bunch of warnings with CONFIG_RETPOLINE like:

  arch/x86/events/intel/uncore_nhmex.o: warning: objtool: nhmex_rbox_msr_enable_event()+0x44: sibling call from callable instruction with modified stack frame
  kernel/signal.o: warning: objtool: copy_siginfo_to_user()+0x91: sibling call from callable instruction with modified stack frame
  ...

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-2-git-send-email-dwmw@amazon.co.uk

---
 tools/objtool/check.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 9b341584..de053fb 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -456,6 +456,13 @@ static int add_jump_destinations(struct objtool_file *file)
 		} else if (rela->sym->sec->idx) {
 			dest_sec = rela->sym->sec;
 			dest_off = rela->sym->sym.st_value + rela->addend + 4;
+		} else if (strstr(rela->sym->name, "_indirect_thunk_")) {
+			/*
+			 * Retpoline jumps are really dynamic jumps in
+			 * disguise, so convert them accordingly.
+			 */
+			insn->type = INSN_JUMP_DYNAMIC;
+			continue;
 		} else {
 			/* sibling call */
 			insn->jump_dest = 0;

[PATCH v8 02/12] objtool: Allow alternatives to be ignored

[David Woodhouse]

Getting objtool to understand retpolines is going to be a bit of a
challenge.  For now, take advantage of the fact that retpolines are
patched in with alternatives.  Just read the original (sane)
non-alternative instruction, and ignore the patched-in retpoline.

This allows objtool to understand the control flow *around* the
retpoline, even if it can't yet follow what's inside.  This means the
ORC unwinder will fail to unwind from inside a retpoline, but will work
fine otherwise.

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
 tools/objtool/check.h |  2 +-
 2 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index de053fb..f40d46e 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
 }
 
 /*
+ * FIXME: For now, just ignore any alternatives which add retpolines.  This is
+ * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
+ * But it at least allows objtool to understand the control flow *around* the
+ * retpoline.
+ */
+static int add_nospec_ignores(struct objtool_file *file)
+{
+	struct section *sec;
+	struct rela *rela;
+	struct instruction *insn;
+
+	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
+	if (!sec)
+		return 0;
+
+	list_for_each_entry(rela, &sec->rela_list, list) {
+		if (rela->sym->type != STT_SECTION) {
+			WARN("unexpected relocation symbol type in %s", sec->name);
+			return -1;
+		}
+
+		insn = find_insn(file, rela->sym->sec, rela->addend);
+		if (!insn) {
+			WARN("bad .discard.nospec entry");
+			return -1;
+		}
+
+		insn->ignore_alts = true;
+	}
+
+	return 0;
+}
+
+/*
  * Find the destination instructions for all jumps.
  */
 static int add_jump_destinations(struct objtool_file *file)
@@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
 			dest_off = insn->offset + insn->len + insn->immediate;
 			insn->call_dest = find_symbol_by_offset(insn->sec,
 								dest_off);
+			/*
+			 * FIXME: Thanks to retpolines, it's now considered
+			 * normal for a function to call within itself.  So
+			 * disable this warning for now.
+			 */
+#if 0
 			if (!insn->call_dest) {
 				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
 					  insn->sec, insn->offset, dest_off);
 				return -1;
 			}
+#endif
 		} else if (rela->sym->type == STT_SECTION) {
 			insn->call_dest = find_symbol_by_offset(rela->sym->sec,
 								rela->addend+4);
@@ -678,12 +719,6 @@ static int add_special_section_alts(struct objtool_file *file)
 		return ret;
 
 	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
-		alt = malloc(sizeof(*alt));
-		if (!alt) {
-			WARN("malloc failed");
-			ret = -1;
-			goto out;
-		}
 
 		orig_insn = find_insn(file, special_alt->orig_sec,
 				      special_alt->orig_off);
@@ -694,6 +729,10 @@ static int add_special_section_alts(struct objtool_file *file)
 			goto out;
 		}
 
+		/* Ignore retpoline alternatives. */
+		if (orig_insn->ignore_alts)
+			continue;
+
 		new_insn = NULL;
 		if (!special_alt->group || special_alt->new_len) {
 			new_insn = find_insn(file, special_alt->new_sec,
@@ -719,6 +758,13 @@ static int add_special_section_alts(struct objtool_file *file)
 				goto out;
 		}
 
+		alt = malloc(sizeof(*alt));
+		if (!alt) {
+			WARN("malloc failed");
+			ret = -1;
+			goto out;
+		}
+
 		alt->insn = new_insn;
 		list_add_tail(&alt->list, &orig_insn->alts);
 
@@ -1035,6 +1081,10 @@ static int decode_sections(struct objtool_file *file)
 
 	add_ignores(file);
 
+	ret = add_nospec_ignores(file);
+	if (ret)
+		return ret;
+
 	ret = add_jump_destinations(file);
 	if (ret)
 		return ret;
diff --git a/tools/objtool/check.h b/tools/objtool/check.h
index 47d9ea7..dbadb30 100644
--- a/tools/objtool/check.h
+++ b/tools/objtool/check.h
@@ -44,7 +44,7 @@ struct instruction {
 	unsigned int len;
 	unsigned char type;
 	unsigned long immediate;
-	bool alt_group, visited, dead_end, ignore, hint, save, restore;
+	bool alt_group, visited, dead_end, ignore, hint, save, restore, ignore_alts;
 	struct symbol *call_dest;
 	struct instruction *jump_dest;
 	struct list_head alts;
-- 
2.7.4

[tip:x86/pti] objtool: Allow alternatives to be ignored

[tip-bot for Josh Poimboeuf]

Commit-ID:  258c76059cece01bebae098e81bacb1af2edad17
Gitweb:     https://git.kernel.org/tip/258c76059cece01bebae098e81bacb1af2edad17
Author:     Josh Poimboeuf <jpoimboe@redhat.com>
AuthorDate: Thu, 11 Jan 2018 21:46:24 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:28 +0100

objtool: Allow alternatives to be ignored

Getting objtool to understand retpolines is going to be a bit of a
challenge.  For now, take advantage of the fact that retpolines are
patched in with alternatives.  Just read the original (sane)
non-alternative instruction, and ignore the patched-in retpoline.

This allows objtool to understand the control flow *around* the
retpoline, even if it can't yet follow what's inside.  This means the
ORC unwinder will fail to unwind from inside a retpoline, but will work
fine otherwise.

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-3-git-send-email-dwmw@amazon.co.uk

---
 tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
 tools/objtool/check.h |  2 +-
 2 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index de053fb..f40d46e 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
 }
 
 /*
+ * FIXME: For now, just ignore any alternatives which add retpolines.  This is
+ * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
+ * But it at least allows objtool to understand the control flow *around* the
+ * retpoline.
+ */
+static int add_nospec_ignores(struct objtool_file *file)
+{
+	struct section *sec;
+	struct rela *rela;
+	struct instruction *insn;
+
+	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
+	if (!sec)
+		return 0;
+
+	list_for_each_entry(rela, &sec->rela_list, list) {
+		if (rela->sym->type != STT_SECTION) {
+			WARN("unexpected relocation symbol type in %s", sec->name);
+			return -1;
+		}
+
+		insn = find_insn(file, rela->sym->sec, rela->addend);
+		if (!insn) {
+			WARN("bad .discard.nospec entry");
+			return -1;
+		}
+
+		insn->ignore_alts = true;
+	}
+
+	return 0;
+}
+
+/*
  * Find the destination instructions for all jumps.
  */
 static int add_jump_destinations(struct objtool_file *file)
@@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
 			dest_off = insn->offset + insn->len + insn->immediate;
 			insn->call_dest = find_symbol_by_offset(insn->sec,
 								dest_off);
+			/*
+			 * FIXME: Thanks to retpolines, it's now considered
+			 * normal for a function to call within itself.  So
+			 * disable this warning for now.
+			 */
+#if 0
 			if (!insn->call_dest) {
 				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
 					  insn->sec, insn->offset, dest_off);
 				return -1;
 			}
+#endif
 		} else if (rela->sym->type == STT_SECTION) {
 			insn->call_dest = find_symbol_by_offset(rela->sym->sec,
 								rela->addend+4);
@@ -678,12 +719,6 @@ static int add_special_section_alts(struct objtool_file *file)
 		return ret;
 
 	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
-		alt = malloc(sizeof(*alt));
-		if (!alt) {
-			WARN("malloc failed");
-			ret = -1;
-			goto out;
-		}
 
 		orig_insn = find_insn(file, special_alt->orig_sec,
 				      special_alt->orig_off);
@@ -694,6 +729,10 @@ static int add_special_section_alts(struct objtool_file *file)
 			goto out;
 		}
 
+		/* Ignore retpoline alternatives. */
+		if (orig_insn->ignore_alts)
+			continue;
+
 		new_insn = NULL;
 		if (!special_alt->group || special_alt->new_len) {
 			new_insn = find_insn(file, special_alt->new_sec,
@@ -719,6 +758,13 @@ static int add_special_section_alts(struct objtool_file *file)
 				goto out;
 		}
 
+		alt = malloc(sizeof(*alt));
+		if (!alt) {
+			WARN("malloc failed");
+			ret = -1;
+			goto out;
+		}
+
 		alt->insn = new_insn;
 		list_add_tail(&alt->list, &orig_insn->alts);
 
@@ -1035,6 +1081,10 @@ static int decode_sections(struct objtool_file *file)
 
 	add_ignores(file);
 
+	ret = add_nospec_ignores(file);
+	if (ret)
+		return ret;
+
 	ret = add_jump_destinations(file);
 	if (ret)
 		return ret;
diff --git a/tools/objtool/check.h b/tools/objtool/check.h
index 47d9ea7..dbadb30 100644
--- a/tools/objtool/check.h
+++ b/tools/objtool/check.h
@@ -44,7 +44,7 @@ struct instruction {
 	unsigned int len;
 	unsigned char type;
 	unsigned long immediate;
-	bool alt_group, visited, dead_end, ignore, hint, save, restore;
+	bool alt_group, visited, dead_end, ignore, hint, save, restore, ignore_alts;
 	struct symbol *call_dest;
 	struct instruction *jump_dest;
 	struct list_head alts;

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

Hi folks,

On Thu, Jan 11, 2018 at 09:46:24PM +0000, Woodhouse, David wrote:
> Getting objtool to understand retpolines is going to be a bit of a
> challenge.  For now, take advantage of the fact that retpolines are
> patched in with alternatives.  Just read the original (sane)
> non-alternative instruction, and ignore the patched-in retpoline.
> 
> This allows objtool to understand the control flow *around* the
> retpoline, even if it can't yet follow what's inside.  This means the
> ORC unwinder will fail to unwind from inside a retpoline, but will work
> fine otherwise.
> 
> Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> ---
>  tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
>  tools/objtool/check.h |  2 +-
>  2 files changed, 57 insertions(+), 7 deletions(-)
> 
> diff --git a/tools/objtool/check.c b/tools/objtool/check.c
> index de053fb..f40d46e 100644
> --- a/tools/objtool/check.c
> +++ b/tools/objtool/check.c
> @@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
>  }
>  
>  /*
> + * FIXME: For now, just ignore any alternatives which add retpolines.  This is
> + * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
> + * But it at least allows objtool to understand the control flow *around* the
> + * retpoline.
> + */
> +static int add_nospec_ignores(struct objtool_file *file)
> +{
> +	struct section *sec;
> +	struct rela *rela;
> +	struct instruction *insn;
> +
> +	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
> +	if (!sec)
> +		return 0;
> +
> +	list_for_each_entry(rela, &sec->rela_list, list) {
> +		if (rela->sym->type != STT_SECTION) {
> +			WARN("unexpected relocation symbol type in %s", sec->name);
> +			return -1;
> +		}
> +
> +		insn = find_insn(file, rela->sym->sec, rela->addend);
> +		if (!insn) {
> +			WARN("bad .discard.nospec entry");
> +			return -1;
> +		}
> +
> +		insn->ignore_alts = true;
> +	}
> +
> +	return 0;
> +}
> +
> +/*
>   * Find the destination instructions for all jumps.
>   */
>  static int add_jump_destinations(struct objtool_file *file)
> @@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
>  			dest_off = insn->offset + insn->len + insn->immediate;
>  			insn->call_dest = find_symbol_by_offset(insn->sec,
>  								dest_off);
> +			/*
> +			 * FIXME: Thanks to retpolines, it's now considered
> +			 * normal for a function to call within itself.  So
> +			 * disable this warning for now.
> +			 */
> +#if 0
>  			if (!insn->call_dest) {
>  				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
>  					  insn->sec, insn->offset, dest_off);
>  				return -1;
>  			}
> +#endif

This crashes for me in is_fentry_call().

Program received signal SIGSEGV, Segmentation fault.
is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
1113		if (insn->type == INSN_CALL &&
(gdb) info stack
#0  is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
#1  validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1747
#2  0x0000000000404bd3 in validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1770
#3  0x0000000000406783 in validate_functions (file=<optimized out>) at check.c:1933
#4  check (_objname=0x6bb9d0 "", _no_fp=40, no_unreachable=4, orc=false) at check.c:2006
#5  0x00000000004021c1 in handle_internal_command (argv=0x7fffffffe5c0, argc=4) at objtool.c:108
#6  main (argc=4, argv=0x7fffffffe5c0) at objtool.c:131

This is not entirely surprising, since insn->call_dest is NULL and
is_fentry_call() doesn't expect that.

How is this supposed to work ? What am I missing ?

Guenter

>  		} else if (rela->sym->type == STT_SECTION) {
>  			insn->call_dest = find_symbol_by_offset(rela->sym->sec,
>  								rela->addend+4);
> @@ -678,12 +719,6 @@ static int add_special_section_alts(struct objtool_file *file)
>  		return ret;
>  
>  	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
> -		alt = malloc(sizeof(*alt));
> -		if (!alt) {
> -			WARN("malloc failed");
> -			ret = -1;
> -			goto out;
> -		}
>  
>  		orig_insn = find_insn(file, special_alt->orig_sec,
>  				      special_alt->orig_off);
> @@ -694,6 +729,10 @@ static int add_special_section_alts(struct objtool_file *file)
>  			goto out;
>  		}
>  
> +		/* Ignore retpoline alternatives. */
> +		if (orig_insn->ignore_alts)
> +			continue;
> +
>  		new_insn = NULL;
>  		if (!special_alt->group || special_alt->new_len) {
>  			new_insn = find_insn(file, special_alt->new_sec,
> @@ -719,6 +758,13 @@ static int add_special_section_alts(struct objtool_file *file)
>  				goto out;
>  		}
>  
> +		alt = malloc(sizeof(*alt));
> +		if (!alt) {
> +			WARN("malloc failed");
> +			ret = -1;
> +			goto out;
> +		}
> +
>  		alt->insn = new_insn;
>  		list_add_tail(&alt->list, &orig_insn->alts);
>  
> @@ -1035,6 +1081,10 @@ static int decode_sections(struct objtool_file *file)
>  
>  	add_ignores(file);
>  
> +	ret = add_nospec_ignores(file);
> +	if (ret)
> +		return ret;
> +
>  	ret = add_jump_destinations(file);
>  	if (ret)
>  		return ret;
> diff --git a/tools/objtool/check.h b/tools/objtool/check.h
> index 47d9ea7..dbadb30 100644
> --- a/tools/objtool/check.h
> +++ b/tools/objtool/check.h
> @@ -44,7 +44,7 @@ struct instruction {
>  	unsigned int len;
>  	unsigned char type;
>  	unsigned long immediate;
> -	bool alt_group, visited, dead_end, ignore, hint, save, restore;
> +	bool alt_group, visited, dead_end, ignore, hint, save, restore, ignore_alts;
>  	struct symbol *call_dest;
>  	struct instruction *jump_dest;
>  	struct list_head alts;

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Josh Poimboeuf]

On Thu, Jan 18, 2018 at 11:09:31AM -0800, Guenter Roeck wrote:
> Hi folks,
> 
> On Thu, Jan 11, 2018 at 09:46:24PM +0000, Woodhouse, David wrote:
> > Getting objtool to understand retpolines is going to be a bit of a
> > challenge.  For now, take advantage of the fact that retpolines are
> > patched in with alternatives.  Just read the original (sane)
> > non-alternative instruction, and ignore the patched-in retpoline.
> > 
> > This allows objtool to understand the control flow *around* the
> > retpoline, even if it can't yet follow what's inside.  This means the
> > ORC unwinder will fail to unwind from inside a retpoline, but will work
> > fine otherwise.
> > 
> > Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
> > Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> > ---
> >  tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
> >  tools/objtool/check.h |  2 +-
> >  2 files changed, 57 insertions(+), 7 deletions(-)
> > 
> > diff --git a/tools/objtool/check.c b/tools/objtool/check.c
> > index de053fb..f40d46e 100644
> > --- a/tools/objtool/check.c
> > +++ b/tools/objtool/check.c
> > @@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
> >  }
> >  
> >  /*
> > + * FIXME: For now, just ignore any alternatives which add retpolines.  This is
> > + * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
> > + * But it at least allows objtool to understand the control flow *around* the
> > + * retpoline.
> > + */
> > +static int add_nospec_ignores(struct objtool_file *file)
> > +{
> > +	struct section *sec;
> > +	struct rela *rela;
> > +	struct instruction *insn;
> > +
> > +	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
> > +	if (!sec)
> > +		return 0;
> > +
> > +	list_for_each_entry(rela, &sec->rela_list, list) {
> > +		if (rela->sym->type != STT_SECTION) {
> > +			WARN("unexpected relocation symbol type in %s", sec->name);
> > +			return -1;
> > +		}
> > +
> > +		insn = find_insn(file, rela->sym->sec, rela->addend);
> > +		if (!insn) {
> > +			WARN("bad .discard.nospec entry");
> > +			return -1;
> > +		}
> > +
> > +		insn->ignore_alts = true;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/*
> >   * Find the destination instructions for all jumps.
> >   */
> >  static int add_jump_destinations(struct objtool_file *file)
> > @@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
> >  			dest_off = insn->offset + insn->len + insn->immediate;
> >  			insn->call_dest = find_symbol_by_offset(insn->sec,
> >  								dest_off);
> > +			/*
> > +			 * FIXME: Thanks to retpolines, it's now considered
> > +			 * normal for a function to call within itself.  So
> > +			 * disable this warning for now.
> > +			 */
> > +#if 0
> >  			if (!insn->call_dest) {
> >  				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
> >  					  insn->sec, insn->offset, dest_off);
> >  				return -1;
> >  			}
> > +#endif
> 
> This crashes for me in is_fentry_call().
> 
> Program received signal SIGSEGV, Segmentation fault.
> is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> 1113		if (insn->type == INSN_CALL &&
> (gdb) info stack
> #0  is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> #1  validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1747
> #2  0x0000000000404bd3 in validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1770
> #3  0x0000000000406783 in validate_functions (file=<optimized out>) at check.c:1933
> #4  check (_objname=0x6bb9d0 "", _no_fp=40, no_unreachable=4, orc=false) at check.c:2006
> #5  0x00000000004021c1 in handle_internal_command (argv=0x7fffffffe5c0, argc=4) at objtool.c:108
> #6  main (argc=4, argv=0x7fffffffe5c0) at objtool.c:131
> 
> This is not entirely surprising, since insn->call_dest is NULL and
> is_fentry_call() doesn't expect that.
> 
> How is this supposed to work ? What am I missing ?

Not sure, does your gcc have retpolines?  Give me your .o file and I can
diagnose it.

I intended to have an error msg instead of a seg fault for this
situation, just haven't had a chance to improve that yet in the midst of
all the hoopla.

-- 
Josh

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

[-- Attachment #1: Type: text/plain, Size: 4598 bytes --]

On Thu, Jan 18, 2018 at 01:33:15PM -0600, Josh Poimboeuf wrote:
> On Thu, Jan 18, 2018 at 11:09:31AM -0800, Guenter Roeck wrote:
> > Hi folks,
> > 
> > On Thu, Jan 11, 2018 at 09:46:24PM +0000, Woodhouse, David wrote:
> > > Getting objtool to understand retpolines is going to be a bit of a
> > > challenge.  For now, take advantage of the fact that retpolines are
> > > patched in with alternatives.  Just read the original (sane)
> > > non-alternative instruction, and ignore the patched-in retpoline.
> > > 
> > > This allows objtool to understand the control flow *around* the
> > > retpoline, even if it can't yet follow what's inside.  This means the
> > > ORC unwinder will fail to unwind from inside a retpoline, but will work
> > > fine otherwise.
> > > 
> > > Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
> > > Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> > > ---
> > >  tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
> > >  tools/objtool/check.h |  2 +-
> > >  2 files changed, 57 insertions(+), 7 deletions(-)
> > > 
> > > diff --git a/tools/objtool/check.c b/tools/objtool/check.c
> > > index de053fb..f40d46e 100644
> > > --- a/tools/objtool/check.c
> > > +++ b/tools/objtool/check.c
> > > @@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
> > >  }
> > >  
> > >  /*
> > > + * FIXME: For now, just ignore any alternatives which add retpolines.  This is
> > > + * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
> > > + * But it at least allows objtool to understand the control flow *around* the
> > > + * retpoline.
> > > + */
> > > +static int add_nospec_ignores(struct objtool_file *file)
> > > +{
> > > +	struct section *sec;
> > > +	struct rela *rela;
> > > +	struct instruction *insn;
> > > +
> > > +	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
> > > +	if (!sec)
> > > +		return 0;
> > > +
> > > +	list_for_each_entry(rela, &sec->rela_list, list) {
> > > +		if (rela->sym->type != STT_SECTION) {
> > > +			WARN("unexpected relocation symbol type in %s", sec->name);
> > > +			return -1;
> > > +		}
> > > +
> > > +		insn = find_insn(file, rela->sym->sec, rela->addend);
> > > +		if (!insn) {
> > > +			WARN("bad .discard.nospec entry");
> > > +			return -1;
> > > +		}
> > > +
> > > +		insn->ignore_alts = true;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/*
> > >   * Find the destination instructions for all jumps.
> > >   */
> > >  static int add_jump_destinations(struct objtool_file *file)
> > > @@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
> > >  			dest_off = insn->offset + insn->len + insn->immediate;
> > >  			insn->call_dest = find_symbol_by_offset(insn->sec,
> > >  								dest_off);
> > > +			/*
> > > +			 * FIXME: Thanks to retpolines, it's now considered
> > > +			 * normal for a function to call within itself.  So
> > > +			 * disable this warning for now.
> > > +			 */
> > > +#if 0
> > >  			if (!insn->call_dest) {
> > >  				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
> > >  					  insn->sec, insn->offset, dest_off);
> > >  				return -1;
> > >  			}
> > > +#endif
> > 
> > This crashes for me in is_fentry_call().
> > 
> > Program received signal SIGSEGV, Segmentation fault.
> > is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> > 1113		if (insn->type == INSN_CALL &&
> > (gdb) info stack
> > #0  is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> > #1  validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1747
> > #2  0x0000000000404bd3 in validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1770
> > #3  0x0000000000406783 in validate_functions (file=<optimized out>) at check.c:1933
> > #4  check (_objname=0x6bb9d0 "", _no_fp=40, no_unreachable=4, orc=false) at check.c:2006
> > #5  0x00000000004021c1 in handle_internal_command (argv=0x7fffffffe5c0, argc=4) at objtool.c:108
> > #6  main (argc=4, argv=0x7fffffffe5c0) at objtool.c:131
> > 
> > This is not entirely surprising, since insn->call_dest is NULL and
> > is_fentry_call() doesn't expect that.
> > 
> > How is this supposed to work ? What am I missing ?
> 
> Not sure, does your gcc have retpolines?  Give me your .o file and I can
> diagnose it.
> 
Yes, it does, only it is the gcc from the Google toolchain which may
generate different code than the upstream version.

I attached an affected object file. Please let me know if there is anything else
I can do to help.

Thanks,
Guenter

[-- Attachment #2: quirks.o --]
[-- Type: application/x-object, Size: 222176 bytes --]

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 1134 bytes --]

On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
> 
> > Not sure, does your gcc have retpolines?  Give me your .o file and I can
> > diagnose it.
> > 
> Yes, it does, only it is the gcc from the Google toolchain which may
> generate different code than the upstream version.
> 
> I attached an affected object file. Please let me know if there is anything else
> I can do to help.
Disassembly of section .text.__x86.indirect_thunk:

0000000000000000 <__x86.indirect_thunk>:
   0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
   5:	f3 90                	pause  
   7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
   9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
   e:	c3                   	retq   

That has the old-style CET-incompatible retpoline in a COMDAT section
in the .o file. What compiler options are being used for that? The
kernel should only use retpoline if GCC supports both of
-mindirect-branch=thunk-extern and -mindirect-branch-register, and this
compiler is doing *neither* of those. 

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

Hi David,

On Mon, Jan 22, 2018 at 07:34:04PM +0000, David Woodhouse wrote:
> On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
> > 
> > > Not sure, does your gcc have retpolines?  Give me your .o file and I can
> > > diagnose it.
> > > 
> > Yes, it does, only it is the gcc from the Google toolchain which may
> > generate different code than the upstream version.
> > 
> > I attached an affected object file. Please let me know if there is anything else
> > I can do to help.
> Disassembly of section .text.__x86.indirect_thunk:
> 
> 0000000000000000 <__x86.indirect_thunk>:
>    0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
>    5:	f3 90                	pause  
>    7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
>    9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
>    e:	c3                   	retq   
> 
> That has the old-style CET-incompatible retpoline in a COMDAT section
> in the .o file. What compiler options are being used for that? The
> kernel should only use retpoline if GCC supports both of
> -mindirect-branch=thunk-extern and -mindirect-branch-register, and this
> compiler is doing *neither* of those. 

It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
-fno-jump-tables", though I don't know if that even exists in
upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
your feedback to our compiler team.

Either case, I think it is less than optimal that objtool crashes
with _any_ object code.

Thanks,
Guenter

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 524 bytes --]

On Mon, 2018-01-22 at 12:25 -0800, Guenter Roeck wrote:
> 
> It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
> -fno-jump-tables", though I don't know if that even exists in
> upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
> your feedback to our compiler team.

We moved on from that and the upstream retpoline support in the kernel
won't recognise it.

> Either case, I think it is less than optimal that objtool crashes
> with _any_ object code.

Yeah, I'll give you that one :)

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Josh Poimboeuf]

On Mon, Jan 22, 2018 at 12:25:22PM -0800, Guenter Roeck wrote:
> Hi David,
> 
> On Mon, Jan 22, 2018 at 07:34:04PM +0000, David Woodhouse wrote:
> > On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
> > > 
> > > > Not sure, does your gcc have retpolines?  Give me your .o file and I can
> > > > diagnose it.
> > > > 
> > > Yes, it does, only it is the gcc from the Google toolchain which may
> > > generate different code than the upstream version.
> > > 
> > > I attached an affected object file. Please let me know if there is anything else
> > > I can do to help.
> > Disassembly of section .text.__x86.indirect_thunk:
> > 
> > 0000000000000000 <__x86.indirect_thunk>:
> >    0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
> >    5:	f3 90                	pause  
> >    7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
> >    9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
> >    e:	c3                   	retq   
> > 
> > That has the old-style CET-incompatible retpoline in a COMDAT section
> > in the .o file. What compiler options are being used for that? The
> > kernel should only use retpoline if GCC supports both of
> > -mindirect-branch=thunk-extern and -mindirect-branch-register, and this
> > compiler is doing *neither* of those. 
> 
> It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
> -fno-jump-tables", though I don't know if that even exists in
> upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
> your feedback to our compiler team.
> 
> Either case, I think it is less than optimal that objtool crashes
> with _any_ object code.

I've got a pending fix for this, so that objtool doesn't seg fault, and
instead prints out a warning:

  quirks.o: warning: objtool: efi_delete_dummy_variable()+0x99: unsupported intra-function call
  quirks.o: warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate it with ANNOTATE_NOSPEC_ALTERNATIVE.

The code is here, along with a few more fixes:

  https://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git/log/?h=TODO-objtool-seg-fault

Will post it soon.

-- 
Josh

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

On 01/28/2018 01:06 PM, Josh Poimboeuf wrote:
> On Mon, Jan 22, 2018 at 12:25:22PM -0800, Guenter Roeck wrote:
>> Hi David,
>>
>> On Mon, Jan 22, 2018 at 07:34:04PM +0000, David Woodhouse wrote:
>>> On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
>>>>
>>>>> Not sure, does your gcc have retpolines?  Give me your .o file and I can
>>>>> diagnose it.
>>>>>   
>>>> Yes, it does, only it is the gcc from the Google toolchain which may
>>>> generate different code than the upstream version.
>>>>
>>>> I attached an affected object file. Please let me know if there is anything else
>>>> I can do to help.
>>> Disassembly of section .text.__x86.indirect_thunk:
>>>
>>> 0000000000000000 <__x86.indirect_thunk>:
>>>     0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
>>>     5:	f3 90                	pause
>>>     7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
>>>     9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
>>>     e:	c3                   	retq
>>>
>>> That has the old-style CET-incompatible retpoline in a COMDAT section
>>> in the .o file. What compiler options are being used for that? The
>>> kernel should only use retpoline if GCC supports both of
>>> -mindirect-branch=thunk-extern and -mindirect-branch-register, and this
>>> compiler is doing *neither* of those.
>>
>> It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
>> -fno-jump-tables", though I don't know if that even exists in
>> upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
>> your feedback to our compiler team.
>>
>> Either case, I think it is less than optimal that objtool crashes
>> with _any_ object code.
> 
> I've got a pending fix for this, so that objtool doesn't seg fault, and
> instead prints out a warning:
> 
>    quirks.o: warning: objtool: efi_delete_dummy_variable()+0x99: unsupported intra-function call
>    quirks.o: warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate it with ANNOTATE_NOSPEC_ALTERNATIVE.
> 
> The code is here, along with a few more fixes:
> 
>    https://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git/log/?h=TODO-objtool-seg-fault
> 

Excellent. I'll give it a try tomorrow.

Thanks for looking into this!

Guenter

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

On Sun, Jan 28, 2018 at 03:06:42PM -0600, Josh Poimboeuf wrote:
> On Mon, Jan 22, 2018 at 12:25:22PM -0800, Guenter Roeck wrote:
> > Hi David,
> > 
> > On Mon, Jan 22, 2018 at 07:34:04PM +0000, David Woodhouse wrote:
> > > On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
> > > > 
> > > > > Not sure, does your gcc have retpolines?  Give me your .o file and I can
> > > > > diagnose it.
> > > > > 
> > > > Yes, it does, only it is the gcc from the Google toolchain which may
> > > > generate different code than the upstream version.
> > > > 
> > > > I attached an affected object file. Please let me know if there is anything else
> > > > I can do to help.
> > > Disassembly of section .text.__x86.indirect_thunk:
> > > 
> > > 0000000000000000 <__x86.indirect_thunk>:
> > >    0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
> > >    5:	f3 90                	pause  
> > >    7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
> > >    9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
> > >    e:	c3                   	retq   
> > > 
> > > That has the old-style CET-incompatible retpoline in a COMDAT section
> > > in the .o file. What compiler options are being used for that? The
> > > kernel should only use retpoline if GCC supports both of
> > > -mindirect-branch=thunk-extern and -mindirect-branch-register, and this
> > > compiler is doing *neither* of those. 
> > 
> > It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
> > -fno-jump-tables", though I don't know if that even exists in
> > upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
> > your feedback to our compiler team.
> > 
> > Either case, I think it is less than optimal that objtool crashes
> > with _any_ object code.
> 
> I've got a pending fix for this, so that objtool doesn't seg fault, and
> instead prints out a warning:
> 
>   quirks.o: warning: objtool: efi_delete_dummy_variable()+0x99: unsupported intra-function call
>   quirks.o: warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate it with ANNOTATE_NOSPEC_ALTERNATIVE.
> 
> The code is here, along with a few more fixes:
> 
>   https://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git/log/?h=TODO-objtool-seg-fault
> 

'objtool: Improve retpoline alternative handling' works for me.

Thanks!
Guenter

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Josh Poimboeuf]

On Mon, Jan 29, 2018 at 09:15:26AM -0800, Guenter Roeck wrote:
> On Sun, Jan 28, 2018 at 03:06:42PM -0600, Josh Poimboeuf wrote:
> > On Mon, Jan 22, 2018 at 12:25:22PM -0800, Guenter Roeck wrote:
> > > Hi David,
> > > 
> > > On Mon, Jan 22, 2018 at 07:34:04PM +0000, David Woodhouse wrote:
> > > > On Thu, 2018-01-18 at 11:41 -0800, Guenter Roeck wrote:
> > > > > 
> > > > > > Not sure, does your gcc have retpolines?  Give me your .o file and I can
> > > > > > diagnose it.
> > > > > > 
> > > > > Yes, it does, only it is the gcc from the Google toolchain which may
> > > > > generate different code than the upstream version.
> > > > > 
> > > > > I attached an affected object file. Please let me know if there is anything else
> > > > > I can do to help.
> > > > Disassembly of section .text.__x86.indirect_thunk:
> > > > 
> > > > 0000000000000000 <__x86.indirect_thunk>:
> > > >    0:	e8 04 00 00 00       	callq  9 <__x86.indirect_thunk+0x9>
> > > >    5:	f3 90                	pause  
> > > >    7:	eb fc                	jmp    5 <__x86.indirect_thunk+0x5>
> > > >    9:	48 8d 64 24 08       	lea    0x8(%rsp),%rsp
> > > >    e:	c3                   	retq   
> > > > 
> > > > That has the old-style CET-incompatible retpoline in a COMDAT section
> > > > in the .o file. What compiler options are being used for that? The
> > > > kernel should only use retpoline if GCC supports both of
> > > > -mindirect-branch=thunk-extern and -mindirect-branch-register, and this
> > > > compiler is doing *neither* of those. 
> > > 
> > > It uses "-mindirect-branch=thunk -mindirect-branch-loop=pause
> > > -fno-jump-tables", though I don't know if that even exists in
> > > upstream gcc (it is the gcc use for Chrome OS builds). I'll pass
> > > your feedback to our compiler team.
> > > 
> > > Either case, I think it is less than optimal that objtool crashes
> > > with _any_ object code.
> > 
> > I've got a pending fix for this, so that objtool doesn't seg fault, and
> > instead prints out a warning:
> > 
> >   quirks.o: warning: objtool: efi_delete_dummy_variable()+0x99: unsupported intra-function call
> >   quirks.o: warning: objtool: If this is a retpoline, please patch it in with alternatives and annotate it with ANNOTATE_NOSPEC_ALTERNATIVE.
> > 
> > The code is here, along with a few more fixes:
> > 
> >   https://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git/log/?h=TODO-objtool-seg-fault
> > 
> 
> 'objtool: Improve retpoline alternative handling' works for me.

Thanks! I'll give you a

  Reported-and-tested-by: Guenter Roeck <linux@roeck-us.net>

-- 
Josh

Re: [v8,02/12] objtool: Allow alternatives to be ignored

[Guenter Roeck]

On Thu, Jan 18, 2018 at 01:33:15PM -0600, Josh Poimboeuf wrote:
> On Thu, Jan 18, 2018 at 11:09:31AM -0800, Guenter Roeck wrote:
> > Hi folks,
> > 
> > On Thu, Jan 11, 2018 at 09:46:24PM +0000, Woodhouse, David wrote:
> > > Getting objtool to understand retpolines is going to be a bit of a
> > > challenge.  For now, take advantage of the fact that retpolines are
> > > patched in with alternatives.  Just read the original (sane)
> > > non-alternative instruction, and ignore the patched-in retpoline.
> > > 
> > > This allows objtool to understand the control flow *around* the
> > > retpoline, even if it can't yet follow what's inside.  This means the
> > > ORC unwinder will fail to unwind from inside a retpoline, but will work
> > > fine otherwise.
> > > 
> > > Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
> > > Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> > > ---
> > >  tools/objtool/check.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++-----
> > >  tools/objtool/check.h |  2 +-
> > >  2 files changed, 57 insertions(+), 7 deletions(-)
> > > 
> > > diff --git a/tools/objtool/check.c b/tools/objtool/check.c
> > > index de053fb..f40d46e 100644
> > > --- a/tools/objtool/check.c
> > > +++ b/tools/objtool/check.c
> > > @@ -428,6 +428,40 @@ static void add_ignores(struct objtool_file *file)
> > >  }
> > >  
> > >  /*
> > > + * FIXME: For now, just ignore any alternatives which add retpolines.  This is
> > > + * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
> > > + * But it at least allows objtool to understand the control flow *around* the
> > > + * retpoline.
> > > + */
> > > +static int add_nospec_ignores(struct objtool_file *file)
> > > +{
> > > +	struct section *sec;
> > > +	struct rela *rela;
> > > +	struct instruction *insn;
> > > +
> > > +	sec = find_section_by_name(file->elf, ".rela.discard.nospec");
> > > +	if (!sec)
> > > +		return 0;
> > > +
> > > +	list_for_each_entry(rela, &sec->rela_list, list) {
> > > +		if (rela->sym->type != STT_SECTION) {
> > > +			WARN("unexpected relocation symbol type in %s", sec->name);
> > > +			return -1;
> > > +		}
> > > +
> > > +		insn = find_insn(file, rela->sym->sec, rela->addend);
> > > +		if (!insn) {
> > > +			WARN("bad .discard.nospec entry");
> > > +			return -1;
> > > +		}
> > > +
> > > +		insn->ignore_alts = true;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/*
> > >   * Find the destination instructions for all jumps.
> > >   */
> > >  static int add_jump_destinations(struct objtool_file *file)
> > > @@ -509,11 +543,18 @@ static int add_call_destinations(struct objtool_file *file)
> > >  			dest_off = insn->offset + insn->len + insn->immediate;
> > >  			insn->call_dest = find_symbol_by_offset(insn->sec,
> > >  								dest_off);
> > > +			/*
> > > +			 * FIXME: Thanks to retpolines, it's now considered
> > > +			 * normal for a function to call within itself.  So
> > > +			 * disable this warning for now.
> > > +			 */
> > > +#if 0
> > >  			if (!insn->call_dest) {
> > >  				WARN_FUNC("can't find call dest symbol at offset 0x%lx",
> > >  					  insn->sec, insn->offset, dest_off);
> > >  				return -1;
> > >  			}
> > > +#endif
> > 
> > This crashes for me in is_fentry_call().
> > 
> > Program received signal SIGSEGV, Segmentation fault.
> > is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> > 1113		if (insn->type == INSN_CALL &&
> > (gdb) info stack
> > #0  is_fentry_call (insn=<optimized out>, insn=<optimized out>) at check.c:1113
> > #1  validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1747
> > #2  0x0000000000404bd3 in validate_branch (file=0x7ffffff7e440, first=0x7ffffff7e128, state=...) at check.c:1770
> > #3  0x0000000000406783 in validate_functions (file=<optimized out>) at check.c:1933
> > #4  check (_objname=0x6bb9d0 "", _no_fp=40, no_unreachable=4, orc=false) at check.c:2006
> > #5  0x00000000004021c1 in handle_internal_command (argv=0x7fffffffe5c0, argc=4) at objtool.c:108
> > #6  main (argc=4, argv=0x7fffffffe5c0) at objtool.c:131
> > 
> > This is not entirely surprising, since insn->call_dest is NULL and
> > is_fentry_call() doesn't expect that.
> > 
> > How is this supposed to work ? What am I missing ?
> 
> Not sure, does your gcc have retpolines?  Give me your .o file and I can
> diagnose it.
> 
> I intended to have an error msg instead of a seg fault for this
> situation, just haven't had a chance to improve that yet in the midst of
> all the hoopla.
> 
Any updates ? I reverted thee offending patch from our latest stable tree
merge, but that doesn't really solve the issue.

Thanks,
Guenter

[PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[David Woodhouse]

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 +++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 128 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++
 8 files changed, 231 insertions(+)
 create mode 100644 arch/x86/include/asm/nospec-branch.h
 create mode 100644 arch/x86/lib/retpoline.S

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..e20e92e
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * This should be used immediately before a retpoline alternative.  It tells
+ * objtool where the retpolines are so that it can make sense of the control
+ * flow by just reading the original instruction(s) and ignoring the
+ * alternatives.
+ */
+.macro ANNOTATE_NOSPEC_ALTERNATIVE
+	.Lannotate_\@:
+	.pushsection .discard.nospec
+	.long .Lannotate_\@ - .
+	.popsection
+.endm
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#define ANNOTATE_NOSPEC_ALTERNATIVE				\
+	"999:\n\t"						\
+	".pushsection .discard.nospec\n\t"			\
+	".long 999b - .\n\t"					\
+	".popsection\n\t"
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC						\
+	ANNOTATE_NOSPEC_ALTERNATIVE				\
+	ALTERNATIVE(						\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",	\
+	"       jmp    904f;\n"					\
+	"       .align 16\n"					\
+	"901:	call   903f;\n"					\
+	"902:	pause;\n"					\
+	"       jmp    902b;\n"					\
+	"       .align 16\n"					\
+	"903:	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"904:	call   901b;\n",				\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif
-- 
2.7.4

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  76b043848fd22dbf7f8bf3a1452f8c70d557b860
Gitweb:     https://git.kernel.org/tip/76b043848fd22dbf7f8bf3a1452f8c70d557b860
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:25 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:28 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 +++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 128 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++
 8 files changed, 231 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..e20e92e
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * This should be used immediately before a retpoline alternative.  It tells
+ * objtool where the retpolines are so that it can make sense of the control
+ * flow by just reading the original instruction(s) and ignoring the
+ * alternatives.
+ */
+.macro ANNOTATE_NOSPEC_ALTERNATIVE
+	.Lannotate_\@:
+	.pushsection .discard.nospec
+	.long .Lannotate_\@ - .
+	.popsection
+.endm
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#define ANNOTATE_NOSPEC_ALTERNATIVE				\
+	"999:\n\t"						\
+	".pushsection .discard.nospec\n\t"			\
+	".long 999b - .\n\t"					\
+	".popsection\n\t"
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC						\
+	ANNOTATE_NOSPEC_ALTERNATIVE				\
+	ALTERNATIVE(						\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",	\
+	"       jmp    904f;\n"					\
+	"       .align 16\n"					\
+	"901:	call   903f;\n"					\
+	"902:	pause;\n"					\
+	"       jmp    902b;\n"					\
+	"       .align 16\n"					\
+	"903:	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"904:	call   901b;\n",				\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[Tom Lendacky]

On 1/11/2018 3:46 PM, David Woodhouse wrote:
> Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
> the corresponding thunks. Provide assembler macros for invoking the thunks
> in the same way that GCC does, from native and inline assembler.
> 
> This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
> some circumstances, IBRS microcode features may be used instead, and the
> retpoline can be disabled.
> 
> On AMD CPUs if lfence is serialising, the retpoline can be dramatically
> simplified to a simple "lfence; jmp *\reg". A future patch, after it has
> been verified that lfence really is serialising in all circumstances, can
> enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
> to X86_FEATURE_RETPOLINE.
> 
> Do not align the retpoline in the altinstr section, because there is no
> guarantee that it stays aligned when it's copied over the oldinstr during
> alternative patching.
> 
> [ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
> [ tglx: Put actual function CALL/JMP in front of the macros, convert to
>   	symbolic labels ]
> [ dwmw2: Convert back to numeric labels, merge objtool fixes ]
> 
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Acked-by: Arjan van de Ven <arjan@linux.intel.com>
> Acked-by: Ingo Molnar <mingo@kernel.org>
> Cc: gnomes@lxorguk.ukuu.org.uk
> Cc: Rik van Riel <riel@redhat.com>
> Cc: Andi Kleen <ak@linux.intel.com>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Linus Torvalds <torvalds@linux-foundation.org>
> Cc: Jiri Kosina <jikos@kernel.org>
> Cc: Andy Lutomirski <luto@amacapital.net>
> Cc: Dave Hansen <dave.hansen@intel.com>
> Cc: Kees Cook <keescook@google.com>
> Cc: Tim Chen <tim.c.chen@linux.intel.com>
> Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
> Cc: Paul Turner <pjt@google.com>
> Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk
> ---
>  arch/x86/Kconfig                      |  13 ++++
>  arch/x86/Makefile                     |  10 +++
>  arch/x86/include/asm/asm-prototypes.h |  25 +++++++
>  arch/x86/include/asm/cpufeatures.h    |   2 +
>  arch/x86/include/asm/nospec-branch.h  | 128 ++++++++++++++++++++++++++++++++++
>  arch/x86/kernel/cpu/common.c          |   4 ++
>  arch/x86/lib/Makefile                 |   1 +
>  arch/x86/lib/retpoline.S              |  48 +++++++++++++
>  8 files changed, 231 insertions(+)
>  create mode 100644 arch/x86/include/asm/nospec-branch.h
>  create mode 100644 arch/x86/lib/retpoline.S
> 

...

> diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
> new file mode 100644
> index 0000000..e20e92e
> --- /dev/null
> +++ b/arch/x86/include/asm/nospec-branch.h
> @@ -0,0 +1,128 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#ifndef __NOSPEC_BRANCH_H__
> +#define __NOSPEC_BRANCH_H__
> +
> +#include <asm/alternative.h>
> +#include <asm/alternative-asm.h>
> +#include <asm/cpufeatures.h>
> +
> +#ifdef __ASSEMBLY__
> +
> +/*
> + * This should be used immediately before a retpoline alternative.  It tells
> + * objtool where the retpolines are so that it can make sense of the control
> + * flow by just reading the original instruction(s) and ignoring the
> + * alternatives.
> + */
> +.macro ANNOTATE_NOSPEC_ALTERNATIVE
> +	.Lannotate_\@:
> +	.pushsection .discard.nospec
> +	.long .Lannotate_\@ - .
> +	.popsection
> +.endm
> +
> +/*
> + * These are the bare retpoline primitives for indirect jmp and call.
> + * Do not use these directly; they only exist to make the ALTERNATIVE
> + * invocation below less ugly.
> + */
> +.macro RETPOLINE_JMP reg:req
> +	call	.Ldo_rop_\@
> +.Lspec_trap_\@:
> +	pause

Talked with our engineers some more on using pause vs. lfence.  Pause is
not serializing on AMD, so the pause/jmp loop will use power as it is
speculated over waiting for return to mispredict to the correct target.
Can this be changed back to lfence?  It looked like a very small
difference in cycles/time.

Thanks,
Tom

> +	jmp	.Lspec_trap_\@
> +.Ldo_rop_\@:
> +	mov	\reg, (%_ASM_SP)
> +	ret
> +.endm
> +
> +/*
> + * This is a wrapper around RETPOLINE_JMP so the called function in reg
> + * returns to the instruction after the macro.
> + */
> +.macro RETPOLINE_CALL reg:req
> +	jmp	.Ldo_call_\@
> +.Ldo_retpoline_jmp_\@:
> +	RETPOLINE_JMP \reg
> +.Ldo_call_\@:
> +	call	.Ldo_retpoline_jmp_\@
> +.endm
> +
> +/*
> + * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
> + * indirect jmp/call which may be susceptible to the Spectre variant 2
> + * attack.
> + */
> +.macro JMP_NOSPEC reg:req
> +#ifdef CONFIG_RETPOLINE
> +	ANNOTATE_NOSPEC_ALTERNATIVE
> +	ALTERNATIVE_2 __stringify(jmp *\reg),				\
> +		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
> +		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
> +#else
> +	jmp	*\reg
> +#endif
> +.endm
> +
> +.macro CALL_NOSPEC reg:req
> +#ifdef CONFIG_RETPOLINE
> +	ANNOTATE_NOSPEC_ALTERNATIVE
> +	ALTERNATIVE_2 __stringify(call *\reg),				\
> +		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
> +		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
> +#else
> +	call	*\reg
> +#endif
> +.endm
> +
> +#else /* __ASSEMBLY__ */
> +
> +#define ANNOTATE_NOSPEC_ALTERNATIVE				\
> +	"999:\n\t"						\
> +	".pushsection .discard.nospec\n\t"			\
> +	".long 999b - .\n\t"					\
> +	".popsection\n\t"
> +
> +#if defined(CONFIG_X86_64) && defined(RETPOLINE)
> +
> +/*
> + * Since the inline asm uses the %V modifier which is only in newer GCC,
> + * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
> + */
> +# define CALL_NOSPEC						\
> +	ANNOTATE_NOSPEC_ALTERNATIVE				\
> +	ALTERNATIVE(						\
> +	"call *%[thunk_target]\n",				\
> +	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
> +	X86_FEATURE_RETPOLINE)
> +# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
> +
> +#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
> +/*
> + * For i386 we use the original ret-equivalent retpoline, because
> + * otherwise we'll run out of registers. We don't care about CET
> + * here, anyway.
> + */
> +# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",	\
> +	"       jmp    904f;\n"					\
> +	"       .align 16\n"					\
> +	"901:	call   903f;\n"					\
> +	"902:	pause;\n"					\
> +	"       jmp    902b;\n"					\
> +	"       .align 16\n"					\
> +	"903:	addl   $4, %%esp;\n"				\
> +	"       pushl  %[thunk_target];\n"			\
> +	"       ret;\n"						\
> +	"       .align 16\n"					\
> +	"904:	call   901b;\n",				\
> +	X86_FEATURE_RETPOLINE)
> +
> +# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
> +#else /* No retpoline */
> +# define CALL_NOSPEC "call *%[thunk_target]\n"
> +# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
> +#endif
> +
> +#endif /* __ASSEMBLY__ */
> +#endif /* __NOSPEC_BRANCH_H__ */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index 372ba3f..7a671d1 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
>  	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
>  	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
>  
> +#ifdef CONFIG_RETPOLINE
> +	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
> +#endif
> +
>  	fpu__init_system(c);
>  
>  #ifdef CONFIG_X86_32
> diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
> index 457f681..d435c89 100644
> --- a/arch/x86/lib/Makefile
> +++ b/arch/x86/lib/Makefile
> @@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
>  lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
>  lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
>  lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
> +lib-$(CONFIG_RETPOLINE) += retpoline.o
>  
>  obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
>  
> diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
> new file mode 100644
> index 0000000..cb45c6c
> --- /dev/null
> +++ b/arch/x86/lib/retpoline.S
> @@ -0,0 +1,48 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#include <linux/stringify.h>
> +#include <linux/linkage.h>
> +#include <asm/dwarf2.h>
> +#include <asm/cpufeatures.h>
> +#include <asm/alternative-asm.h>
> +#include <asm/export.h>
> +#include <asm/nospec-branch.h>
> +
> +.macro THUNK reg
> +	.section .text.__x86.indirect_thunk.\reg
> +
> +ENTRY(__x86_indirect_thunk_\reg)
> +	CFI_STARTPROC
> +	JMP_NOSPEC %\reg
> +	CFI_ENDPROC
> +ENDPROC(__x86_indirect_thunk_\reg)
> +.endm
> +
> +/*
> + * Despite being an assembler file we can't just use .irp here
> + * because __KSYM_DEPS__ only uses the C preprocessor and would
> + * only see one instance of "__x86_indirect_thunk_\reg" rather
> + * than one per register with the correct names. So we do it
> + * the simple and nasty way...
> + */
> +#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
> +#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
> +
> +GENERATE_THUNK(_ASM_AX)
> +GENERATE_THUNK(_ASM_BX)
> +GENERATE_THUNK(_ASM_CX)
> +GENERATE_THUNK(_ASM_DX)
> +GENERATE_THUNK(_ASM_SI)
> +GENERATE_THUNK(_ASM_DI)
> +GENERATE_THUNK(_ASM_BP)
> +GENERATE_THUNK(_ASM_SP)
> +#ifdef CONFIG_64BIT
> +GENERATE_THUNK(r8)
> +GENERATE_THUNK(r9)
> +GENERATE_THUNK(r10)
> +GENERATE_THUNK(r11)
> +GENERATE_THUNK(r12)
> +GENERATE_THUNK(r13)
> +GENERATE_THUNK(r14)
> +GENERATE_THUNK(r15)
> +#endif
> 

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 2128 bytes --]

On Thu, 2018-01-11 at 17:58 -0600, Tom Lendacky wrote:
> 
> > + * These are the bare retpoline primitives for indirect jmp and call.
> > + * Do not use these directly; they only exist to make the ALTERNATIVE
> > + * invocation below less ugly.
> > + */
> > +.macro RETPOLINE_JMP reg:req
> > +     call    .Ldo_rop_\@
> > +.Lspec_trap_\@:
> > +     pause

Note that we never use that one on AMD. You just get 'lfence; jmp *reg'
instead because you promised us that would work.... while Intel said it
would work for a month or two and then said "er, oops, no it doesn't in
all cases." — so we're half-waiting for you lot to do the same thing :)

You *do* get the RSB-stuffing one though, which is the same. So...

> Talked with our engineers some more on using pause vs. lfence.  Pause is
> not serializing on AMD, so the pause/jmp loop will use power as it is
> speculated over waiting for return to mispredict to the correct target.
> Can this be changed back to lfence?  It looked like a very small
> difference in cycles/time.

That seems reasonable, although at this stage I'm also tempted to
suggest we can do that kind of fine-tuning in a followup patch. Like
the bikeshedding about numbers vs. readable labels. We really need the
IBRS and IBPB patches to be landing on top of this as soon as possible.

Paul, the lfence→pause change was only a tiny micro-optimisation on
Intel, wasn't it? Are you happy with changing the implementations of
the RSB stuffing code to use lfence again (or what about 'hlt')?

It currently looks like this... the capture loop is using 'jmp' to
match the retpoline instead of 'call' as in your examples:


#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)	\
	mov	$(nr/2), reg;			\
.Ldo_call1_ ## uniq:				\
	call	.Ldo_call2_ ## uniq;		\
.Ltrap1_ ## uniq:				\
	pause;					\
	jmp	.Ltrap1_ ## uniq;		\
.Ldo_call2_ ## uniq:				\
	call	.Ldo_loop_ ## uniq;		\
.Ltrap2_ ## uniq:				\
	pause;					\
	jmp	.Ltrap2_ ## uniq;		\
.Ldo_loop_ ## uniq:				\
	dec	reg;				\
	jnz	.Ldo_call1_ ## uniq;		\
	add	$(BITS_PER_LONG/8) * nr, sp;


[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[Tom Lendacky]

On 1/12/2018 4:28 AM, David Woodhouse wrote:
> On Thu, 2018-01-11 at 17:58 -0600, Tom Lendacky wrote:
>>
>>> + * These are the bare retpoline primitives for indirect jmp and call.
>>> + * Do not use these directly; they only exist to make the ALTERNATIVE
>>> + * invocation below less ugly.
>>> + */
>>> +.macro RETPOLINE_JMP reg:req
>>> +     call    .Ldo_rop_\@
>>> +.Lspec_trap_\@:
>>> +     pause
> 
> Note that we never use that one on AMD. You just get 'lfence; jmp *reg'
> instead because you promised us that would work.... while Intel said it
> would work for a month or two and then said "er, oops, no it doesn't in
> all cases." — so we're half-waiting for you lot to do the same thing :)

In theory we never get that one on AMD.  But because of the case where
we could be running under a hypervisor and might not be able to verify
that lfence was made serializing, we would fall back to the generic
retpoline.

> 
> You *do* get the RSB-stuffing one though, which is the same. So...

Right.

> 
>> Talked with our engineers some more on using pause vs. lfence.  Pause is
>> not serializing on AMD, so the pause/jmp loop will use power as it is
>> speculated over waiting for return to mispredict to the correct target.
>> Can this be changed back to lfence?  It looked like a very small
>> difference in cycles/time.
> 
> That seems reasonable, although at this stage I'm also tempted to
> suggest we can do that kind of fine-tuning in a followup patch. Like
> the bikeshedding about numbers vs. readable labels. We really need the
> IBRS and IBPB patches to be landing on top of this as soon as possible.

Yup, I understand.

Thanks,
Tom

> 
> Paul, the lfence→pause change was only a tiny micro-optimisation on
> Intel, wasn't it? Are you happy with changing the implementations of
> the RSB stuffing code to use lfence again (or what about 'hlt')?
> 
> It currently looks like this... the capture loop is using 'jmp' to
> match the retpoline instead of 'call' as in your examples:
> 
> 
> #define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)	\
> 	mov	$(nr/2), reg;			\
> .Ldo_call1_ ## uniq:				\
> 	call	.Ldo_call2_ ## uniq;		\
> .Ltrap1_ ## uniq:				\
> 	pause;					\
> 	jmp	.Ltrap1_ ## uniq;		\
> .Ldo_call2_ ## uniq:				\
> 	call	.Ldo_loop_ ## uniq;		\
> .Ltrap2_ ## uniq:				\
> 	pause;					\
> 	jmp	.Ltrap2_ ## uniq;		\
> .Ldo_loop_ ## uniq:				\
> 	dec	reg;				\
> 	jnz	.Ldo_call1_ ## uniq;		\
> 	add	$(BITS_PER_LONG/8) * nr, sp;
> 

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[Borislav Petkov]

On Thu, Jan 11, 2018 at 09:46:25PM +0000, David Woodhouse wrote:
> +/*
> + * This should be used immediately before a retpoline alternative.  It tells
> + * objtool where the retpolines are so that it can make sense of the control
> + * flow by just reading the original instruction(s) and ignoring the
> + * alternatives.
> + */
> +.macro ANNOTATE_NOSPEC_ALTERNATIVE
> +	.Lannotate_\@:
> +	.pushsection .discard.nospec
> +	.long .Lannotate_\@ - .
> +	.popsection
> +.endm

Hey Josh, what happened to parsing only the retpoline-related
.altinstructions sections with the X86_FEATURE bits? Or is this
annotation used for something more additionally?

Thx.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[Josh Poimboeuf]

On Sun, Jan 14, 2018 at 04:02:19PM +0100, Borislav Petkov wrote:
> On Thu, Jan 11, 2018 at 09:46:25PM +0000, David Woodhouse wrote:
> > +/*
> > + * This should be used immediately before a retpoline alternative.  It tells
> > + * objtool where the retpolines are so that it can make sense of the control
> > + * flow by just reading the original instruction(s) and ignoring the
> > + * alternatives.
> > + */
> > +.macro ANNOTATE_NOSPEC_ALTERNATIVE
> > +	.Lannotate_\@:
> > +	.pushsection .discard.nospec
> > +	.long .Lannotate_\@ - .
> > +	.popsection
> > +.endm
> 
> Hey Josh, what happened to parsing only the retpoline-related
> .altinstructions sections with the X86_FEATURE bits? Or is this
> annotation used for something more additionally?

Yeah, that idea came after this patch was already written.  I haven't
implemented it yet.

-- 
Josh

Re: [PATCH v8 03/12] x86/retpoline: Add initial retpoline support

[Borislav Petkov]

On Sun, Jan 14, 2018 at 09:53:45AM -0600, Josh Poimboeuf wrote:
> Yeah, that idea came after this patch was already written.  I haven't
> implemented it yet.

Oh ok. I guess that's fine for the current situation. When you do, you
can simply kill the macro so all good.

Thx.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

[PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[David Woodhouse]

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
  	integration becomes simple ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lkml.kernel.org/r/1515508997-6154-4-git-send-email-dwmw@amazon.co.uk
---
 Documentation/admin-guide/kernel-parameters.txt |  28 +++++
 arch/x86/include/asm/nospec-branch.h            |  10 ++
 arch/x86/kernel/cpu/bugs.c                      | 158 +++++++++++++++++++++++-
 arch/x86/kernel/cpu/common.c                    |   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
 	nosmt		[KNL,S390] Disable symmetric multithreading (SMT).
 			Equivalent to smt=1.
 
+	nospectre_v2	[X86] Disable all mitigations for the Spectre variant 2
+			(indirect branch prediction) vulnerability. System may
+			allow data leaks with this option, which is equivalent
+			to spectre_v2=off.
+
 	noxsave		[BUGS=X86] Disables x86 extended register state save
 			and restore using xsave. The kernel will fallback to
 			enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
 	sonypi.*=	[HW] Sony Programmable I/O Control Device driver
 			See Documentation/laptops/sonypi.txt
 
+	spectre_v2=	[X86] Control mitigation of Spectre variant 2
+			(indirect branch speculation) vulnerability.
+
+			on   - unconditionally enable
+			off  - unconditionally disable
+			auto - kernel detects whether your CPU model is
+			       vulnerable
+
+			Selecting 'on' will, and 'auto' may, choose a
+			mitigation method at run time according to the
+			CPU, the available microcode, the setting of the
+			CONFIG_RETPOLINE configuration option, and the
+			compiler with which the kernel was built.
+
+			Specific mitigations can also be selected manually:
+
+			retpoline	  - replace indirect branches
+			retpoline,generic - google's original retpoline
+			retpoline,amd     - AMD-specific minimal thunk
+
+			Not specifying this option is equivalent to
+			spectre_v2=auto.
+
 	spia_io_base=	[HW,MTD]
 	spia_fio_base=
 	spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index e20e92e..ea034fa 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -124,5 +124,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+	SPECTRE_V2_NONE,
+	SPECTRE_V2_RETPOLINE_MINIMAL,
+	SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+	SPECTRE_V2_RETPOLINE_GENERIC,
+	SPECTRE_V2_RETPOLINE_AMD,
+	SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,9 @@
 #include <linux/init.h>
 #include <linux/utsname.h>
 #include <linux/cpu.h>
+
+#include <asm/nospec-branch.h>
+#include <asm/cmdline.h>
 #include <asm/bugs.h>
 #include <asm/processor.h>
 #include <asm/processor-flags.h>
@@ -21,6 +24,8 @@
 #include <asm/pgtable.h>
 #include <asm/set_memory.h>
 
+static void __init spectre_v2_select_mitigation(void);
+
 void __init check_bugs(void)
 {
 	identify_boot_cpu();
@@ -30,6 +35,9 @@ void __init check_bugs(void)
 		print_cpu_info(&boot_cpu_data);
 	}
 
+	/* Select the proper spectre mitigation before patching alternatives */
+	spectre_v2_select_mitigation();
+
 #ifdef CONFIG_X86_32
 	/*
 	 * Check whether we are able to run this kernel safely on SMP.
@@ -62,6 +70,153 @@ void __init check_bugs(void)
 #endif
 }
 
+/* The kernel command line selection */
+enum spectre_v2_mitigation_cmd {
+	SPECTRE_V2_CMD_NONE,
+	SPECTRE_V2_CMD_AUTO,
+	SPECTRE_V2_CMD_FORCE,
+	SPECTRE_V2_CMD_RETPOLINE,
+	SPECTRE_V2_CMD_RETPOLINE_GENERIC,
+	SPECTRE_V2_CMD_RETPOLINE_AMD,
+};
+
+static const char *spectre_v2_strings[] = {
+	[SPECTRE_V2_NONE]			= "Vulnerable",
+	[SPECTRE_V2_RETPOLINE_MINIMAL]		= "Vulnerable: Minimal generic ASM retpoline",
+	[SPECTRE_V2_RETPOLINE_MINIMAL_AMD]	= "Vulnerable: Minimal AMD ASM retpoline",
+	[SPECTRE_V2_RETPOLINE_GENERIC]		= "Mitigation: Full generic retpoline",
+	[SPECTRE_V2_RETPOLINE_AMD]		= "Mitigation: Full AMD retpoline",
+};
+
+#undef pr_fmt
+#define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
+
+static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
+
+static void __init spec2_print_if_insecure(const char *reason)
+{
+	if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
+		pr_info("%s\n", reason);
+}
+
+static void __init spec2_print_if_secure(const char *reason)
+{
+	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
+		pr_info("%s\n", reason);
+}
+
+static inline bool retp_compiler(void)
+{
+	return __is_defined(RETPOLINE);
+}
+
+static inline bool match_option(const char *arg, int arglen, const char *opt)
+{
+	int len = strlen(opt);
+
+	return len == arglen && !strncmp(arg, opt, len);
+}
+
+static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
+{
+	char arg[20];
+	int ret;
+
+	ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
+				  sizeof(arg));
+	if (ret > 0)  {
+		if (match_option(arg, ret, "off")) {
+			goto disable;
+		} else if (match_option(arg, ret, "on")) {
+			spec2_print_if_secure("force enabled on command line.");
+			return SPECTRE_V2_CMD_FORCE;
+		} else if (match_option(arg, ret, "retpoline")) {
+			spec2_print_if_insecure("retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE;
+		} else if (match_option(arg, ret, "retpoline,amd")) {
+			if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
+				pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
+				return SPECTRE_V2_CMD_AUTO;
+			}
+			spec2_print_if_insecure("AMD retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE_AMD;
+		} else if (match_option(arg, ret, "retpoline,generic")) {
+			spec2_print_if_insecure("generic retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
+		} else if (match_option(arg, ret, "auto")) {
+			return SPECTRE_V2_CMD_AUTO;
+		}
+	}
+
+	if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
+		return SPECTRE_V2_CMD_AUTO;
+disable:
+	spec2_print_if_insecure("disabled on command line.");
+	return SPECTRE_V2_CMD_NONE;
+}
+
+static void __init spectre_v2_select_mitigation(void)
+{
+	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
+	enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
+
+	/*
+	 * If the CPU is not affected and the command line mode is NONE or AUTO
+	 * then nothing to do.
+	 */
+	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
+	    (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
+		return;
+
+	switch (cmd) {
+	case SPECTRE_V2_CMD_NONE:
+		return;
+
+	case SPECTRE_V2_CMD_FORCE:
+		/* FALLTRHU */
+	case SPECTRE_V2_CMD_AUTO:
+		goto retpoline_auto;
+
+	case SPECTRE_V2_CMD_RETPOLINE_AMD:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_amd;
+		break;
+	case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_generic;
+		break;
+	case SPECTRE_V2_CMD_RETPOLINE:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_auto;
+		break;
+	}
+	pr_err("kernel not compiled with retpoline; no mitigation available!");
+	return;
+
+retpoline_auto:
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+	retpoline_amd:
+		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
+			pr_err("LFENCE not serializing. Switching to generic retpoline\n");
+			goto retpoline_generic;
+		}
+		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
+					 SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+	} else {
+	retpoline_generic:
+		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
+					 SPECTRE_V2_RETPOLINE_MINIMAL;
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+	}
+
+	spectre_v2_enabled = mode;
+	pr_info("%s\n", spectre_v2_strings[mode]);
+}
+
+#undef pr_fmt
+
 #ifdef CONFIG_SYSFS
 ssize_t cpu_show_meltdown(struct device *dev,
 			  struct device_attribute *attr, char *buf)
@@ -86,6 +241,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 {
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 		return sprintf(buf, "Not affected\n");
-	return sprintf(buf, "Vulnerable\n");
+
+	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 #endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 7a671d1..372ba3f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,10 +905,6 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
-#ifdef CONFIG_RETPOLINE
-	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
-#endif
-
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
-- 
2.7.4

[tip:x86/pti] x86/spectre: Add boot time option to select Spectre v2 mitigation

[tip-bot for David Woodhouse]

Commit-ID:  da285121560e769cc31797bba6422eea71d473e0
Gitweb:     https://git.kernel.org/tip/da285121560e769cc31797bba6422eea71d473e0
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:26 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/spectre: Add boot time option to select Spectre v2 mitigation

Add a spectre_v2= option to select the mitigation used for the indirect
branch speculation vulnerability.

Currently, the only option available is retpoline, in its various forms.
This will be expanded to cover the new IBRS/IBPB microcode features.

The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
serializing instruction, which is indicated by the LFENCE_RDTSC feature.

[ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
  	integration becomes simple ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-5-git-send-email-dwmw@amazon.co.uk

---
 Documentation/admin-guide/kernel-parameters.txt |  28 +++++
 arch/x86/include/asm/nospec-branch.h            |  10 ++
 arch/x86/kernel/cpu/bugs.c                      | 158 +++++++++++++++++++++++-
 arch/x86/kernel/cpu/common.c                    |   4 -
 4 files changed, 195 insertions(+), 5 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 9059917..8122b5f 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2599,6 +2599,11 @@
 	nosmt		[KNL,S390] Disable symmetric multithreading (SMT).
 			Equivalent to smt=1.
 
+	nospectre_v2	[X86] Disable all mitigations for the Spectre variant 2
+			(indirect branch prediction) vulnerability. System may
+			allow data leaks with this option, which is equivalent
+			to spectre_v2=off.
+
 	noxsave		[BUGS=X86] Disables x86 extended register state save
 			and restore using xsave. The kernel will fallback to
 			enabling legacy floating-point and sse state.
@@ -3908,6 +3913,29 @@
 	sonypi.*=	[HW] Sony Programmable I/O Control Device driver
 			See Documentation/laptops/sonypi.txt
 
+	spectre_v2=	[X86] Control mitigation of Spectre variant 2
+			(indirect branch speculation) vulnerability.
+
+			on   - unconditionally enable
+			off  - unconditionally disable
+			auto - kernel detects whether your CPU model is
+			       vulnerable
+
+			Selecting 'on' will, and 'auto' may, choose a
+			mitigation method at run time according to the
+			CPU, the available microcode, the setting of the
+			CONFIG_RETPOLINE configuration option, and the
+			compiler with which the kernel was built.
+
+			Specific mitigations can also be selected manually:
+
+			retpoline	  - replace indirect branches
+			retpoline,generic - google's original retpoline
+			retpoline,amd     - AMD-specific minimal thunk
+
+			Not specifying this option is equivalent to
+			spectre_v2=auto.
+
 	spia_io_base=	[HW,MTD]
 	spia_fio_base=
 	spia_pedr=
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index e20e92e..ea034fa 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -124,5 +124,15 @@
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
 
+/* The Spectre V2 mitigation variants */
+enum spectre_v2_mitigation {
+	SPECTRE_V2_NONE,
+	SPECTRE_V2_RETPOLINE_MINIMAL,
+	SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
+	SPECTRE_V2_RETPOLINE_GENERIC,
+	SPECTRE_V2_RETPOLINE_AMD,
+	SPECTRE_V2_IBRS,
+};
+
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 76ad6cb..e4dc261 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,9 @@
 #include <linux/init.h>
 #include <linux/utsname.h>
 #include <linux/cpu.h>
+
+#include <asm/nospec-branch.h>
+#include <asm/cmdline.h>
 #include <asm/bugs.h>
 #include <asm/processor.h>
 #include <asm/processor-flags.h>
@@ -21,6 +24,8 @@
 #include <asm/pgtable.h>
 #include <asm/set_memory.h>
 
+static void __init spectre_v2_select_mitigation(void);
+
 void __init check_bugs(void)
 {
 	identify_boot_cpu();
@@ -30,6 +35,9 @@ void __init check_bugs(void)
 		print_cpu_info(&boot_cpu_data);
 	}
 
+	/* Select the proper spectre mitigation before patching alternatives */
+	spectre_v2_select_mitigation();
+
 #ifdef CONFIG_X86_32
 	/*
 	 * Check whether we are able to run this kernel safely on SMP.
@@ -62,6 +70,153 @@ void __init check_bugs(void)
 #endif
 }
 
+/* The kernel command line selection */
+enum spectre_v2_mitigation_cmd {
+	SPECTRE_V2_CMD_NONE,
+	SPECTRE_V2_CMD_AUTO,
+	SPECTRE_V2_CMD_FORCE,
+	SPECTRE_V2_CMD_RETPOLINE,
+	SPECTRE_V2_CMD_RETPOLINE_GENERIC,
+	SPECTRE_V2_CMD_RETPOLINE_AMD,
+};
+
+static const char *spectre_v2_strings[] = {
+	[SPECTRE_V2_NONE]			= "Vulnerable",
+	[SPECTRE_V2_RETPOLINE_MINIMAL]		= "Vulnerable: Minimal generic ASM retpoline",
+	[SPECTRE_V2_RETPOLINE_MINIMAL_AMD]	= "Vulnerable: Minimal AMD ASM retpoline",
+	[SPECTRE_V2_RETPOLINE_GENERIC]		= "Mitigation: Full generic retpoline",
+	[SPECTRE_V2_RETPOLINE_AMD]		= "Mitigation: Full AMD retpoline",
+};
+
+#undef pr_fmt
+#define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
+
+static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
+
+static void __init spec2_print_if_insecure(const char *reason)
+{
+	if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
+		pr_info("%s\n", reason);
+}
+
+static void __init spec2_print_if_secure(const char *reason)
+{
+	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
+		pr_info("%s\n", reason);
+}
+
+static inline bool retp_compiler(void)
+{
+	return __is_defined(RETPOLINE);
+}
+
+static inline bool match_option(const char *arg, int arglen, const char *opt)
+{
+	int len = strlen(opt);
+
+	return len == arglen && !strncmp(arg, opt, len);
+}
+
+static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
+{
+	char arg[20];
+	int ret;
+
+	ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
+				  sizeof(arg));
+	if (ret > 0)  {
+		if (match_option(arg, ret, "off")) {
+			goto disable;
+		} else if (match_option(arg, ret, "on")) {
+			spec2_print_if_secure("force enabled on command line.");
+			return SPECTRE_V2_CMD_FORCE;
+		} else if (match_option(arg, ret, "retpoline")) {
+			spec2_print_if_insecure("retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE;
+		} else if (match_option(arg, ret, "retpoline,amd")) {
+			if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
+				pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
+				return SPECTRE_V2_CMD_AUTO;
+			}
+			spec2_print_if_insecure("AMD retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE_AMD;
+		} else if (match_option(arg, ret, "retpoline,generic")) {
+			spec2_print_if_insecure("generic retpoline selected on command line.");
+			return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
+		} else if (match_option(arg, ret, "auto")) {
+			return SPECTRE_V2_CMD_AUTO;
+		}
+	}
+
+	if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
+		return SPECTRE_V2_CMD_AUTO;
+disable:
+	spec2_print_if_insecure("disabled on command line.");
+	return SPECTRE_V2_CMD_NONE;
+}
+
+static void __init spectre_v2_select_mitigation(void)
+{
+	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
+	enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
+
+	/*
+	 * If the CPU is not affected and the command line mode is NONE or AUTO
+	 * then nothing to do.
+	 */
+	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
+	    (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
+		return;
+
+	switch (cmd) {
+	case SPECTRE_V2_CMD_NONE:
+		return;
+
+	case SPECTRE_V2_CMD_FORCE:
+		/* FALLTRHU */
+	case SPECTRE_V2_CMD_AUTO:
+		goto retpoline_auto;
+
+	case SPECTRE_V2_CMD_RETPOLINE_AMD:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_amd;
+		break;
+	case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_generic;
+		break;
+	case SPECTRE_V2_CMD_RETPOLINE:
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_auto;
+		break;
+	}
+	pr_err("kernel not compiled with retpoline; no mitigation available!");
+	return;
+
+retpoline_auto:
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+	retpoline_amd:
+		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
+			pr_err("LFENCE not serializing. Switching to generic retpoline\n");
+			goto retpoline_generic;
+		}
+		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
+					 SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+	} else {
+	retpoline_generic:
+		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
+					 SPECTRE_V2_RETPOLINE_MINIMAL;
+		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+	}
+
+	spectre_v2_enabled = mode;
+	pr_info("%s\n", spectre_v2_strings[mode]);
+}
+
+#undef pr_fmt
+
 #ifdef CONFIG_SYSFS
 ssize_t cpu_show_meltdown(struct device *dev,
 			  struct device_attribute *attr, char *buf)
@@ -86,6 +241,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 {
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 		return sprintf(buf, "Not affected\n");
-	return sprintf(buf, "Vulnerable\n");
+
+	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 #endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 7a671d1..372ba3f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,10 +905,6 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
-#ifdef CONFIG_RETPOLINE
-	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
-#endif
-
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Borislav Petkov]

On Thu, Jan 11, 2018 at 09:46:26PM +0000, David Woodhouse wrote:
> Add a spectre_v2= option to select the mitigation used for the indirect
> branch speculation vulnerability.
> 
> Currently, the only option available is retpoline, in its various forms.
> This will be expanded to cover the new IBRS/IBPB microcode features.
> 
> The RETPOLINE_AMD feature relies on a serializing LFENCE for speculation
> control. For AMD hardware, only set RETPOLINE_AMD if LFENCE is a
> serializing instruction, which is indicated by the LFENCE_RDTSC feature.
> 
> [ tglx: Folded back the LFENCE/AMD fixes and reworked it so IBRS
>   	integration becomes simple ]

...

> +static inline bool retp_compiler(void)
> +{
> +	return __is_defined(RETPOLINE);
> +}

Btw, this came up today: do we have an idea how to detect objects built
with gcc which has retpoline support?

The only way I could think of is boot the respective kernel and stare at
dmesg:

[    0.064006] Spectre V2 mitigation: LFENCE not serializing. Switching to generic retpoline
[    0.068003] Spectre V2 mitigation: Vulnerable: Minimal generic ASM retpoline

and then deduce that it is not a retpoline-enabled compiler:

> +retpoline_auto:
> +	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
> +	retpoline_amd:
> +		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
> +			pr_err("LFENCE not serializing. Switching to generic retpoline\n");
> +			goto retpoline_generic;
> +		}
> +		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
> +					 SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
> +		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
> +		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
> +	} else {
> +	retpoline_generic:
> +		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
> +					 SPECTRE_V2_RETPOLINE_MINIMAL;
					^^^^^^^^^^^^^^^^^^^^^^^^^^^^

but that might not always be an option.

And it probably should be a more reliable method which we probably could
use to detect !retpolined modules too.

Hmmm.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 2165 bytes --]

On Tue, 2018-01-23 at 23:40 +0100, Borislav Petkov wrote:
> 
> Btw, this came up today: do we have an idea how to detect objects built
> with gcc which has retpoline support?
> 
> The only way I could think of is boot the respective kernel and stare at
> dmesg:
> 
> [    0.064006] Spectre V2 mitigation: LFENCE not serializing. Switching to generic retpoline
> [    0.068003] Spectre V2 mitigation: Vulnerable: Minimal generic ASM retpoline
> 
> and then deduce that it is not a retpoline-enabled compiler:

Right. There *was* a warning during build but we removed that. At the
time, the plan (at least in my head) was to use IBRS instead, if the
retpoline compiler wasn't available. For now we're just going to remain
vulnerable. I'm now wondering if we should reinstate that warning.

> > +retpoline_auto:
> > +     if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
> > +     retpoline_amd:
> > +             if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
> > +                     pr_err("LFENCE not serializing. Switching to generic retpoline\n");
> > +                     goto retpoline_generic;
> > +             }
> > +             mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
> > +                                      SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
> > +             setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
> > +             setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
> > +     } else {
> > +     retpoline_generic:
> > +             mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
> > +                                      SPECTRE_V2_RETPOLINE_MINIMAL;
>                                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> 
> but that might not always be an option.
> 
> And it probably should be a more reliable method which we probably could
> use to detect !retpolined modules too.

Andi actually implemented this, but it ended up being watered down
somewhat.

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Andi Kleen]

> > And it probably should be a more reliable method which we probably could
> > use to detect !retpolined modules too.
> 
> Andi actually implemented this, but it ended up being watered down
> somewhat.

It's enforced in mainline with the following patch

It's not fully bullet proof, but should be good enough to protect
against mistakes at least.

commit 6cfb521ac0d5b97470883ff9b7facae264b7ab12
Author: Andi Kleen <ak@linux.intel.com>
Date:   Tue Jan 16 12:52:28 2018 -0800

    module: Add retpoline tag to VERMAGIC
    
    Add a marker for retpoline to the module VERMAGIC. This catches the case
    when a non RETPOLINE compiled module gets loaded into a retpoline kernel,
    making it insecure.
    
    It doesn't handle the case when retpoline has been runtime disabled.  Even
    in this case the match of the retcompile status will be enforced.  This
    implies that even with retpoline run time disabled all modules loaded need
    to be recompiled.

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Tue, 23 Jan 2018, Borislav Petkov wrote:

> > +		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
> > +					 SPECTRE_V2_RETPOLINE_MINIMAL;
> 					^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> 
> but that might not always be an option.

I think we should start recording CFLAGS the kernel has been compiled with 
anyway; doesn't hurt and might come handy when debugging.

/proc/version is probably not the best place ... /proc/cflags?

> And it probably should be a more reliable method which we probably could
> use to detect !retpolined modules too.

That's the vermagic stuff Andi pushed. But that's not really acceptable 
for distros.

Distros have always been in the situation "we let the external modules to 
load, as it'll work when it comes to functionality, but then it's our 
duty/responsibility to explain to 3rd parties that they *really* should 
recompile". Mostly because of security fixes to static inlines, but not 
only that.

So that vermagic patch doesn't really help anything in real world (FWIW 
I've just dropped it from SLE kernel). "Potentially insecure" doesn't mean 
it shouldn't be loaded if the user wishes so. Only "functionally 
incorrect" (which is the kernel ABI compatibility check) should be the 
show stopper.

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Borislav Petkov]

On Tue, Jan 23, 2018 at 11:55:05PM +0100, Jiri Kosina wrote:
> I think we should start recording CFLAGS the kernel has been compiled with 
> anyway; doesn't hurt and might come handy when debugging.
> 
> /proc/version is probably not the best place ... /proc/cflags?

Yap, I guess I can find that string with hexdump on the kernel binary too :-)

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Kees Cook]

On Wed, Jan 24, 2018 at 10:05 AM, Borislav Petkov <bp@alien8.de> wrote:
> On Tue, Jan 23, 2018 at 11:55:05PM +0100, Jiri Kosina wrote:
>> I think we should start recording CFLAGS the kernel has been compiled with
>> anyway; doesn't hurt and might come handy when debugging.
>>
>> /proc/version is probably not the best place ... /proc/cflags?
>
> Yap, I guess I can find that string with hexdump on the kernel binary too :-)

I've wanted this for a while (especially for the coming detected
support for stack protector). Having more than just the clfags is, I
think, important. We'd likely want to record the entire environment
(compiler version, linker version, flags on both, etc).

-Kees

-- 
Kees Cook
Pixel Security

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Borislav Petkov]

On Wed, Jan 24, 2018 at 11:32:00AM +1100, Kees Cook wrote:
> I've wanted this for a while (especially for the coming detected
> support for stack protector). Having more than just the clfags is, I
> think, important. We'd likely want to record the entire environment
> (compiler version, linker version, flags on both, etc).

... which makes me think of /proc/config.gz

Can we do something like that here too?

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Tue, 23 Jan 2018, Jiri Kosina wrote:

> So that vermagic patch doesn't really help anything in real world (FWIW 
> I've just dropped it from SLE kernel). "Potentially insecure" doesn't mean 
> it shouldn't be loaded if the user wishes so. Only "functionally 
> incorrect" (which is the kernel ABI compatibility check) should be the 
> show stopper.

... one of the supporting arguments here obviously is: those external 
modules are quite often opening so many *other* holes into the system, 
that refusing to load it *just* because of kernel being retpolined while 
the module is not sounds more like not lettting a drunk and armed 
terrorist drive a plane, with the justification being the lack of a proper 
stamped license.

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Andi Kleen]

> Distros have always been in the situation "we let the external modules to 
> load, as it'll work when it comes to functionality, but then it's our 
> duty/responsibility to explain to 3rd parties that they *really* should 
> recompile". Mostly because of security fixes to static inlines, but not 
> only that.

You can use my previous patch which merely warns:

https://git.kernel.org/pub/scm/linux/kernel/git/ak/linux-misc.git/log/?h=spec/retpoline-module-2

-Andi

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Tue, 23 Jan 2018, Andi Kleen wrote:

> > Distros have always been in the situation "we let the external modules to 
> > load, as it'll work when it comes to functionality, but then it's our 
> > duty/responsibility to explain to 3rd parties that they *really* should 
> > recompile". Mostly because of security fixes to static inlines, but not 
> > only that.
> 
> You can use my previous patch which merely warns:
> 
> https://git.kernel.org/pub/scm/linux/kernel/git/ak/linux-misc.git/log/?h=spec/retpoline-module-2

Thanks for pointing this out, Andi. I've been just now writing more or 
less the same thing; ditching that and will reuse your patch instead.

Why was the more aggresive version (6cfb521ac0d5b) merged into Linus' tree 
instead of that?

Thanks,

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Andi Kleen]

> Thanks for pointing this out, Andi. I've been just now writing more or 
> less the same thing; ditching that and will reuse your patch instead.
> 
> Why was the more aggresive version (6cfb521ac0d5b) merged into Linus' tree 
> instead of that?

Greg prefered using modversions, and in the end simplicity won 
out for 4.15

-Andi

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Tue, 23 Jan 2018, Andi Kleen wrote:

> > Thanks for pointing this out, Andi. I've been just now writing more or 
> > less the same thing; ditching that and will reuse your patch instead.
> > 
> > Why was the more aggresive version (6cfb521ac0d5b) merged into Linus' tree 
> > instead of that?
> 
> Greg prefered using modversions

Ah. OK, Greg: I'll eat my hat (I've heard that's very popular these times) 
if there is a major enterprise distro that doesn't revert this patch 
immediately.

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Greg Kroah-Hartman]

On Wed, Jan 24, 2018 at 12:49:37AM +0100, Jiri Kosina wrote:
> On Tue, 23 Jan 2018, Andi Kleen wrote:
> 
> > > Thanks for pointing this out, Andi. I've been just now writing more or 
> > > less the same thing; ditching that and will reuse your patch instead.
> > > 
> > > Why was the more aggresive version (6cfb521ac0d5b) merged into Linus' tree 
> > > instead of that?
> > 
> > Greg prefered using modversions
> 
> Ah. OK, Greg: I'll eat my hat (I've heard that's very popular these times) 
> if there is a major enterprise distro that doesn't revert this patch 
> immediately.

But will Andi's patch work well for you?  Adding a MODULE_INFO() tag to
every module?  I just thought since you were already using modversions
in enterprise distros already, that adding it there would be the
simplest.

If I am wrong, great, let's revert this and add something that you
really will use, otherwise it's just pointless code :)

Perhaps adding a MODULE_INFO() tag for the entire gcc and ld flags would
be nice in the end, as others have pointed out in this thread, but I
don't know if that solves your issue here or not.

thanks,

greg k-h

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Wed, 24 Jan 2018, Greg Kroah-Hartman wrote:

> But will Andi's patch work well for you?  Adding a MODULE_INFO() tag to 
> every module?  

Yes, that would work -- all the modules that get built in tree, or out of 
tree but with retpolined compiler, would have that marker that could be 
optionally checked for by the kernel.

> I just thought since you were already using modversions in enterprise 
> distros already, that adding it there would be the simplest.

The patch as-is introduces immediate modversion mismatch between 
retpolined kernel and non-retpolined module, making each and every one 
fail to load.

Thanks,

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Greg Kroah-Hartman]

On Wed, Jan 24, 2018 at 10:56:49AM +0100, Jiri Kosina wrote:
> On Wed, 24 Jan 2018, Greg Kroah-Hartman wrote:
> 
> > But will Andi's patch work well for you?  Adding a MODULE_INFO() tag to 
> > every module?  
> 
> Yes, that would work -- all the modules that get built in tree, or out of 
> tree but with retpolined compiler, would have that marker that could be 
> optionally checked for by the kernel.

And what is the kernel supposed to do with that info?

> > I just thought since you were already using modversions in enterprise 
> > distros already, that adding it there would be the simplest.
> 
> The patch as-is introduces immediate modversion mismatch between 
> retpolined kernel and non-retpolined module, making each and every one 
> fail to load.

Good, the patch works then, because I thought that not loading
non-retpolined modules in a kernel that was built with retpoline was the
goal here.

If that is not the intended result, then yes, the patch should be
reverted and replaced with something that actually does whatever it is
that you all want.  As it is, I am totally confused as to the
requirement here...

thanks,

greg k-h

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Jiri Kosina]

On Wed, 24 Jan 2018, Greg Kroah-Hartman wrote:

> > > I just thought since you were already using modversions in enterprise 
> > > distros already, that adding it there would be the simplest.
> > 
> > The patch as-is introduces immediate modversion mismatch between 
> > retpolined kernel and non-retpolined module, making each and every one 
> > fail to load.
> 
> Good, the patch works then, because I thought that not loading
> non-retpolined modules in a kernel that was built with retpoline was the
> goal here.

No, we do not want to break loading of externally-built modules just 
because they might contain indirect calls.

Warning in such situations / tainting the kernel / reporting "might be 
vulnerable" in sysfs should be the proper way to go.

retpolines are not kernel ABI (towards modules) breaker, so let's not 
pretend it is.

Thanks,

-- 
Jiri Kosina
SUSE Labs

Re: [PATCH v8 04/12] x86/spectre: Add boot time option to select Spectre v2 mitigation

[Greg Kroah-Hartman]

On Wed, Jan 24, 2018 at 03:03:48PM +0100, Jiri Kosina wrote:
> On Wed, 24 Jan 2018, Greg Kroah-Hartman wrote:
> 
> > > > I just thought since you were already using modversions in enterprise 
> > > > distros already, that adding it there would be the simplest.
> > > 
> > > The patch as-is introduces immediate modversion mismatch between 
> > > retpolined kernel and non-retpolined module, making each and every one 
> > > fail to load.
> > 
> > Good, the patch works then, because I thought that not loading
> > non-retpolined modules in a kernel that was built with retpoline was the
> > goal here.
> 
> No, we do not want to break loading of externally-built modules just 
> because they might contain indirect calls.
> 
> Warning in such situations / tainting the kernel / reporting "might be 
> vulnerable" in sysfs should be the proper way to go.
> 
> retpolines are not kernel ABI (towards modules) breaker, so let's not 
> pretend it is.

Ok, my fault, I should not have suggested that Andi do the check this
way then.  I thought we wanted to make this part of the kernel ABI.

I'll go make up a patch to revert this now...

thanks,

greg k-h

[PATCH v8 05/12] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[David Woodhouse]

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-5-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/crypto/aesni-intel_asm.S            | 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S    | 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include <linux/linkage.h>
 #include <asm/inst.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
 	pxor INC, STATE4
 	movdqu IV, 0x30(OUTP)
 
-	call *%r11
+	CALL_NOSPEC %r11
 
 	movdqu 0x00(OUTP), INC
 	pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
 	_aesni_gf128mul_x_ble()
 	movups IV, (IVP)
 
-	call *%r11
+	CALL_NOSPEC %r11
 
 	movdqu 0x40(OUTP), INC
 	pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include <linux/linkage.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
 	vpxor 14 * 16(%rax), %xmm15, %xmm14;
 	vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-	call *%r9;
+	CALL_NOSPEC %r9;
 
 	addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include <linux/linkage.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
 	vpxor 14 * 32(%rax), %ymm15, %ymm14;
 	vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-	call *%r9;
+	CALL_NOSPEC %r9;
 
 	addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include <asm/inst.h>
 #include <linux/linkage.h>
+#include <asm/nospec-branch.h>
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
 	movzxw  (bufp, %rax, 2), len
 	lea	crc_array(%rip), bufp
 	lea     (bufp, len, 1), bufp
-	jmp     *bufp
+	JMP_NOSPEC bufp
 
 	################################################################
 	## 2a) PROCESS FULL BLOCKS:
-- 
2.7.4

[tip:x86/pti] x86/retpoline/crypto: Convert crypto assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9697fa39efd3fc3692f2949d4045f393ec58450b
Gitweb:     https://git.kernel.org/tip/9697fa39efd3fc3692f2949d4045f393ec58450b
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:27 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/crypto: Convert crypto assembler indirect jumps

Convert all indirect jumps in crypto assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-6-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/crypto/aesni-intel_asm.S            | 5 +++--
 arch/x86/crypto/camellia-aesni-avx-asm_64.S  | 3 ++-
 arch/x86/crypto/camellia-aesni-avx2-asm_64.S | 3 ++-
 arch/x86/crypto/crc32c-pcl-intel-asm_64.S    | 3 ++-
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 16627fe..3d09e3a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,7 @@
 #include <linux/linkage.h>
 #include <asm/inst.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -2884,7 +2885,7 @@ ENTRY(aesni_xts_crypt8)
 	pxor INC, STATE4
 	movdqu IV, 0x30(OUTP)
 
-	call *%r11
+	CALL_NOSPEC %r11
 
 	movdqu 0x00(OUTP), INC
 	pxor INC, STATE1
@@ -2929,7 +2930,7 @@ ENTRY(aesni_xts_crypt8)
 	_aesni_gf128mul_x_ble()
 	movups IV, (IVP)
 
-	call *%r11
+	CALL_NOSPEC %r11
 
 	movdqu 0x40(OUTP), INC
 	pxor INC, STATE1
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index f7c495e..a14af6e 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -17,6 +17,7 @@
 
 #include <linux/linkage.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1227,7 +1228,7 @@ camellia_xts_crypt_16way:
 	vpxor 14 * 16(%rax), %xmm15, %xmm14;
 	vpxor 15 * 16(%rax), %xmm15, %xmm15;
 
-	call *%r9;
+	CALL_NOSPEC %r9;
 
 	addq $(16 * 16), %rsp;
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index eee5b39..b66bbfa 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -12,6 +12,7 @@
 
 #include <linux/linkage.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -1343,7 +1344,7 @@ camellia_xts_crypt_32way:
 	vpxor 14 * 32(%rax), %ymm15, %ymm14;
 	vpxor 15 * 32(%rax), %ymm15, %ymm15;
 
-	call *%r9;
+	CALL_NOSPEC %r9;
 
 	addq $(16 * 32), %rsp;
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 7a7de27..d9b734d 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -45,6 +45,7 @@
 
 #include <asm/inst.h>
 #include <linux/linkage.h>
+#include <asm/nospec-branch.h>
 
 ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
 
@@ -172,7 +173,7 @@ continue_block:
 	movzxw  (bufp, %rax, 2), len
 	lea	crc_array(%rip), bufp
 	lea     (bufp, len, 1), bufp
-	jmp     *bufp
+	JMP_NOSPEC bufp
 
 	################################################################
 	## 2a) PROCESS FULL BLOCKS:

[PATCH v8 06/12] x86/retpoline/entry: Convert entry assembler indirect jumps

[David Woodhouse]

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-6-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +++++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include <asm/asm.h>
 #include <asm/smap.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 	.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
 	/* kernel thread */
 1:	movl	%edi, %eax
-	call	*%ebx
+	CALL_NOSPEC %ebx
 	/*
 	 * A kernel thread is allowed to return here after successfully
 	 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
 	movl	%ecx, %es
 	TRACE_IRQS_OFF
 	movl	%esp, %eax			# pt_regs pointer
-	call	*%edi
+	CALL_NOSPEC %edi
 	jmp	ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include <asm/pgtable_types.h>
 #include <asm/export.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 #include <linux/err.h>
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 	 */
 	pushq	%rdi
 	movq	$entry_SYSCALL_64_stage2, %rdi
-	jmp	*%rdi
+	JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
 	.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 	 * It might end up jumping to the slow path.  If it jumps, RAX
 	 * and all argument registers are clobbered.
 	 */
+#ifdef CONFIG_RETPOLINE
+	movq	sys_call_table(, %rax, 8), %rax
+	call	__x86_indirect_thunk_rax
+#else
 	call	*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
 	movq	%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
 	jmp	entry_SYSCALL64_slow_path
 
 1:
-	jmp	*%rax				/* Called from C */
+	JMP_NOSPEC %rax				/* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
 	/* kernel thread */
 	movq	%r12, %rdi
-	call	*%rbx
+	CALL_NOSPEC %rbx
 	/*
 	 * A kernel thread is allowed to return here after successfully
 	 * calling do_execve().  Exit to userspace to complete the execve()
-- 
2.7.4

[tip:x86/pti] x86/retpoline/entry: Convert entry assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  2641f08bb7fc63a636a2b18173221d7040a3512e
Gitweb:     https://git.kernel.org/tip/2641f08bb7fc63a636a2b18173221d7040a3512e
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:28 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:29 +0100

x86/retpoline/entry: Convert entry assembler indirect jumps

Convert indirect jumps in core 32/64bit entry assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Don't use CALL_NOSPEC in entry_SYSCALL_64_fastpath because the return
address after the 'call' instruction must be *precisely* at the
.Lentry_SYSCALL_64_after_fastpath label for stub_ptregs_64 to work,
and the use of alternatives will mess that up unless we play horrid
games to prepend with NOPs and make the variants the same length. It's
not worth it; in the case where we ALTERNATIVE out the retpoline, the
first instruction at __x86.indirect_thunk.rax is going to be a bare
jmp *%rax anyway.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-7-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/entry/entry_32.S |  5 +++--
 arch/x86/entry/entry_64.S | 12 +++++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ace8f32..a1f28a5 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -44,6 +44,7 @@
 #include <asm/asm.h>
 #include <asm/smap.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 
 	.section .entry.text, "ax"
 
@@ -290,7 +291,7 @@ ENTRY(ret_from_fork)
 
 	/* kernel thread */
 1:	movl	%edi, %eax
-	call	*%ebx
+	CALL_NOSPEC %ebx
 	/*
 	 * A kernel thread is allowed to return here after successfully
 	 * calling do_execve().  Exit to userspace to complete the execve()
@@ -919,7 +920,7 @@ common_exception:
 	movl	%ecx, %es
 	TRACE_IRQS_OFF
 	movl	%esp, %eax			# pt_regs pointer
-	call	*%edi
+	CALL_NOSPEC %edi
 	jmp	ret_from_exception
 END(common_exception)
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ed31d00..59874bc 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include <asm/pgtable_types.h>
 #include <asm/export.h>
 #include <asm/frame.h>
+#include <asm/nospec-branch.h>
 #include <linux/err.h>
 
 #include "calling.h"
@@ -187,7 +188,7 @@ ENTRY(entry_SYSCALL_64_trampoline)
 	 */
 	pushq	%rdi
 	movq	$entry_SYSCALL_64_stage2, %rdi
-	jmp	*%rdi
+	JMP_NOSPEC %rdi
 END(entry_SYSCALL_64_trampoline)
 
 	.popsection
@@ -266,7 +267,12 @@ entry_SYSCALL_64_fastpath:
 	 * It might end up jumping to the slow path.  If it jumps, RAX
 	 * and all argument registers are clobbered.
 	 */
+#ifdef CONFIG_RETPOLINE
+	movq	sys_call_table(, %rax, 8), %rax
+	call	__x86_indirect_thunk_rax
+#else
 	call	*sys_call_table(, %rax, 8)
+#endif
 .Lentry_SYSCALL_64_after_fastpath_call:
 
 	movq	%rax, RAX(%rsp)
@@ -438,7 +444,7 @@ ENTRY(stub_ptregs_64)
 	jmp	entry_SYSCALL64_slow_path
 
 1:
-	jmp	*%rax				/* Called from C */
+	JMP_NOSPEC %rax				/* Called from C */
 END(stub_ptregs_64)
 
 .macro ptregs_stub func
@@ -517,7 +523,7 @@ ENTRY(ret_from_fork)
 1:
 	/* kernel thread */
 	movq	%r12, %rdi
-	call	*%rbx
+	CALL_NOSPEC %rbx
 	/*
 	 * A kernel thread is allowed to return here after successfully
 	 * calling do_execve().  Exit to userspace to complete the execve()

[PATCH v8 07/12] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[David Woodhouse]

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-7-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/kernel/ftrace_32.S | 6 ++++--
 arch/x86/kernel/ftrace_64.S | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include <asm/segment.h>
 #include <asm/export.h>
 #include <asm/ftrace.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CC_USING_FENTRY
 # define function_hook	__fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
 	movl	0x4(%ebp), %edx
 	subl	$MCOUNT_INSN_SIZE, %eax
 
-	call	*ftrace_trace_function
+	movl	ftrace_trace_function, %ecx
+	CALL_NOSPEC %ecx
 
 	popl	%edx
 	popl	%ecx
@@ -241,5 +243,5 @@ return_to_handler:
 	movl	%eax, %ecx
 	popl	%edx
 	popl	%eax
-	jmp	*%ecx
+	JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include <asm/ptrace.h>
 #include <asm/ftrace.h>
 #include <asm/export.h>
-
+#include <asm/nospec-branch.h>
 
 	.code64
 	.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 	 * ip and parent ip are used and the list function is called when
 	 * function tracing is enabled.
 	 */
-	call   *ftrace_trace_function
-
+	movq ftrace_trace_function, %r8
+	CALL_NOSPEC %r8
 	restore_mcount_regs
 
 	jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
 	movq 8(%rsp), %rdx
 	movq (%rsp), %rax
 	addq $24, %rsp
-	jmp *%rdi
+	JMP_NOSPEC %rdi
 #endif
-- 
2.7.4

[tip:x86/pti] x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  9351803bd803cdbeb9b5a7850b7b6f464806e3db
Gitweb:     https://git.kernel.org/tip/9351803bd803cdbeb9b5a7850b7b6f464806e3db
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:29 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/ftrace: Convert ftrace assembler indirect jumps

Convert all indirect jumps in ftrace assembler code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-8-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/kernel/ftrace_32.S | 6 ++++--
 arch/x86/kernel/ftrace_64.S | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index b6c6468..4c8440d 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -8,6 +8,7 @@
 #include <asm/segment.h>
 #include <asm/export.h>
 #include <asm/ftrace.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CC_USING_FENTRY
 # define function_hook	__fentry__
@@ -197,7 +198,8 @@ ftrace_stub:
 	movl	0x4(%ebp), %edx
 	subl	$MCOUNT_INSN_SIZE, %eax
 
-	call	*ftrace_trace_function
+	movl	ftrace_trace_function, %ecx
+	CALL_NOSPEC %ecx
 
 	popl	%edx
 	popl	%ecx
@@ -241,5 +243,5 @@ return_to_handler:
 	movl	%eax, %ecx
 	popl	%edx
 	popl	%eax
-	jmp	*%ecx
+	JMP_NOSPEC %ecx
 #endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index c832291..7cb8ba0 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -7,7 +7,7 @@
 #include <asm/ptrace.h>
 #include <asm/ftrace.h>
 #include <asm/export.h>
-
+#include <asm/nospec-branch.h>
 
 	.code64
 	.section .entry.text, "ax"
@@ -286,8 +286,8 @@ trace:
 	 * ip and parent ip are used and the list function is called when
 	 * function tracing is enabled.
 	 */
-	call   *ftrace_trace_function
-
+	movq ftrace_trace_function, %r8
+	CALL_NOSPEC %r8
 	restore_mcount_regs
 
 	jmp fgraph_trace
@@ -329,5 +329,5 @@ GLOBAL(return_to_handler)
 	movq 8(%rsp), %rdx
 	movq (%rsp), %rax
 	addq $24, %rsp
-	jmp *%rdi
+	JMP_NOSPEC %rdi
 #endif

[PATCH v8 08/12] x86/retpoline/hyperv: Convert assembler indirect jumps

[David Woodhouse]

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-8-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/include/asm/mshyperv.h | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include <linux/nmi.h>
 #include <asm/io.h>
 #include <asm/hyperv.h>
+#include <asm/nospec-branch.h>
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 		return U64_MAX;
 
 	__asm__ __volatile__("mov %4, %%r8\n"
-			     "call *%5"
+			     CALL_NOSPEC
 			     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 			       "+c" (control), "+d" (input_address)
-			     :  "r" (output_address), "m" (hv_hypercall_pg)
+			     :  "r" (output_address),
+				THUNK_TARGET(hv_hypercall_pg)
 			     : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
 	u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 	if (!hv_hypercall_pg)
 		return U64_MAX;
 
-	__asm__ __volatile__("call *%7"
+	__asm__ __volatile__(CALL_NOSPEC
 			     : "=A" (hv_status),
 			       "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 			     : "A" (control),
 			       "b" (input_address_hi),
 			       "D"(output_address_hi), "S"(output_address_lo),
-			       "m" (hv_hypercall_pg)
+			       THUNK_TARGET(hv_hypercall_pg)
 			     : "cc", "memory");
 #endif /* !x86_64 */
 	return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 
 #ifdef CONFIG_X86_64
 	{
-		__asm__ __volatile__("call *%4"
+		__asm__ __volatile__(CALL_NOSPEC
 				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 				       "+c" (control), "+d" (input1)
-				     : "m" (hv_hypercall_pg)
+				     : THUNK_TARGET(hv_hypercall_pg)
 				     : "cc", "r8", "r9", "r10", "r11");
 	}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 		u32 input1_hi = upper_32_bits(input1);
 		u32 input1_lo = lower_32_bits(input1);
 
-		__asm__ __volatile__ ("call *%5"
+		__asm__ __volatile__ (CALL_NOSPEC
 				      : "=A"(hv_status),
 					"+c"(input1_lo),
 					ASM_CALL_CONSTRAINT
 				      :	"A" (control),
 					"b" (input1_hi),
-					"m" (hv_hypercall_pg)
+					THUNK_TARGET(hv_hypercall_pg)
 				      : "cc", "edi", "esi");
 	}
 #endif
-- 
2.7.4

[tip:x86/pti] x86/retpoline/hyperv: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  e70e5892b28c18f517f29ab6e83bd57705104b31
Gitweb:     https://git.kernel.org/tip/e70e5892b28c18f517f29ab6e83bd57705104b31
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:30 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:30 +0100

x86/retpoline/hyperv: Convert assembler indirect jumps

Convert all indirect jumps in hyperv inline asm code to use non-speculative
sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-9-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/include/asm/mshyperv.h | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 581bb54..5119e4b 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -7,6 +7,7 @@
 #include <linux/nmi.h>
 #include <asm/io.h>
 #include <asm/hyperv.h>
+#include <asm/nospec-branch.h>
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 		return U64_MAX;
 
 	__asm__ __volatile__("mov %4, %%r8\n"
-			     "call *%5"
+			     CALL_NOSPEC
 			     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 			       "+c" (control), "+d" (input_address)
-			     :  "r" (output_address), "m" (hv_hypercall_pg)
+			     :  "r" (output_address),
+				THUNK_TARGET(hv_hypercall_pg)
 			     : "cc", "memory", "r8", "r9", "r10", "r11");
 #else
 	u32 input_address_hi = upper_32_bits(input_address);
@@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 	if (!hv_hypercall_pg)
 		return U64_MAX;
 
-	__asm__ __volatile__("call *%7"
+	__asm__ __volatile__(CALL_NOSPEC
 			     : "=A" (hv_status),
 			       "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 			     : "A" (control),
 			       "b" (input_address_hi),
 			       "D"(output_address_hi), "S"(output_address_lo),
-			       "m" (hv_hypercall_pg)
+			       THUNK_TARGET(hv_hypercall_pg)
 			     : "cc", "memory");
 #endif /* !x86_64 */
 	return hv_status;
@@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 
 #ifdef CONFIG_X86_64
 	{
-		__asm__ __volatile__("call *%4"
+		__asm__ __volatile__(CALL_NOSPEC
 				     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 				       "+c" (control), "+d" (input1)
-				     : "m" (hv_hypercall_pg)
+				     : THUNK_TARGET(hv_hypercall_pg)
 				     : "cc", "r8", "r9", "r10", "r11");
 	}
 #else
@@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 		u32 input1_hi = upper_32_bits(input1);
 		u32 input1_lo = lower_32_bits(input1);
 
-		__asm__ __volatile__ ("call *%5"
+		__asm__ __volatile__ (CALL_NOSPEC
 				      : "=A"(hv_status),
 					"+c"(input1_lo),
 					ASM_CALL_CONSTRAINT
 				      :	"A" (control),
 					"b" (input1_hi),
-					"m" (hv_hypercall_pg)
+					THUNK_TARGET(hv_hypercall_pg)
 				      : "cc", "edi", "esi");
 	}
 #endif

[PATCH v8 09/12] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[David Woodhouse]

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Juergen Gross <jgross@suse.com>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-9-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/smap.h>
+#include <asm/nospec-branch.h>
 
 #include <xen/interface/xen.h>
 #include <xen/interface/sched.h>
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
 	__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
 	stac();
-	asm volatile("call *%[call]"
+	asm volatile(CALL_NOSPEC
 		     : __HYPERCALL_5PARAM
-		     : [call] "a" (&hypercall_page[call])
+		     : [thunk_target] "a" (&hypercall_page[call])
 		     : __HYPERCALL_CLOBBER5);
 	clac();
 
-- 
2.7.4

[tip:x86/pti] x86/retpoline/xen: Convert Xen hypercall indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  ea08816d5b185ab3d09e95e393f265af54560350
Gitweb:     https://git.kernel.org/tip/ea08816d5b185ab3d09e95e393f265af54560350
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:31 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/xen: Convert Xen hypercall indirect jumps

Convert indirect call in Xen hypercall to use non-speculative sequence,
when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Juergen Gross <jgross@suse.com>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-10-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/include/asm/xen/hypercall.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index 7cb282e..bfd8826 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -44,6 +44,7 @@
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/smap.h>
+#include <asm/nospec-branch.h>
 
 #include <xen/interface/xen.h>
 #include <xen/interface/sched.h>
@@ -217,9 +218,9 @@ privcmd_call(unsigned call,
 	__HYPERCALL_5ARG(a1, a2, a3, a4, a5);
 
 	stac();
-	asm volatile("call *%[call]"
+	asm volatile(CALL_NOSPEC
 		     : __HYPERCALL_5PARAM
-		     : [call] "a" (&hypercall_page[call])
+		     : [thunk_target] "a" (&hypercall_page[call])
 		     : __HYPERCALL_CLOBBER5);
 	clac();
 

[PATCH v8 10/12] x86/retpoline/checksum32: Convert assembler indirect jumps

[David Woodhouse]

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-10-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/lib/checksum_32.S | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include <asm/errno.h>
 #include <asm/asm.h>
 #include <asm/export.h>
-				
+#include <asm/nospec-branch.h>
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
 	negl %ebx
 	lea 45f(%ebx,%ebx,2), %ebx
 	testl %esi, %esi
-	jmp *%ebx
+	JMP_NOSPEC %ebx
 
 	# Handle 2-byte-aligned regions
 20:	addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
 	andl $-32,%edx
 	lea 3f(%ebx,%ebx), %ebx
 	testl %esi, %esi 
-	jmp *%ebx
+	JMP_NOSPEC %ebx
 1:	addl $64,%esi
 	addl $64,%edi 
 	SRC(movb -32(%edx),%bl)	; SRC(movb (%edx),%bl)
-- 
2.7.4

[tip:x86/pti] x86/retpoline/checksum32: Convert assembler indirect jumps

[tip-bot for David Woodhouse]

Commit-ID:  5096732f6f695001fa2d6f1335a2680b37912c69
Gitweb:     https://git.kernel.org/tip/5096732f6f695001fa2d6f1335a2680b37912c69
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:32 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:31 +0100

x86/retpoline/checksum32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit checksum assembler code to use
non-speculative sequences when CONFIG_RETPOLINE is enabled.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-11-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/lib/checksum_32.S | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/x86/lib/checksum_32.S b/arch/x86/lib/checksum_32.S
index 4d34bb5..46e71a7 100644
--- a/arch/x86/lib/checksum_32.S
+++ b/arch/x86/lib/checksum_32.S
@@ -29,7 +29,8 @@
 #include <asm/errno.h>
 #include <asm/asm.h>
 #include <asm/export.h>
-				
+#include <asm/nospec-branch.h>
+
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
  */
@@ -156,7 +157,7 @@ ENTRY(csum_partial)
 	negl %ebx
 	lea 45f(%ebx,%ebx,2), %ebx
 	testl %esi, %esi
-	jmp *%ebx
+	JMP_NOSPEC %ebx
 
 	# Handle 2-byte-aligned regions
 20:	addw (%esi), %ax
@@ -439,7 +440,7 @@ ENTRY(csum_partial_copy_generic)
 	andl $-32,%edx
 	lea 3f(%ebx,%ebx), %ebx
 	testl %esi, %esi 
-	jmp *%ebx
+	JMP_NOSPEC %ebx
 1:	addl $64,%esi
 	addl $64,%edi 
 	SRC(movb -32(%edx),%bl)	; SRC(movb (%edx),%bl)

[PATCH v8 11/12] x86/retpoline/irq32: Convert assembler indirect jumps

[David Woodhouse]

From: Andi Kleen <ak@linux.intel.com>

Convert all indirect jumps in 32bit irq inline asm code to use non
speculative sequences.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-11-git-send-email-dwmw@amazon.co.uk
---
 arch/x86/kernel/irq_32.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index a83b334..c1bdbd3 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -20,6 +20,7 @@
 #include <linux/mm.h>
 
 #include <asm/apic.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 
@@ -55,11 +56,11 @@ DEFINE_PER_CPU(struct irq_stack *, softirq_stack);
 static void call_on_stack(void *func, void *stack)
 {
 	asm volatile("xchgl	%%ebx,%%esp	\n"
-		     "call	*%%edi		\n"
+		     CALL_NOSPEC
 		     "movl	%%ebx,%%esp	\n"
 		     : "=b" (stack)
 		     : "0" (stack),
-		       "D"(func)
+		       [thunk_target] "D"(func)
 		     : "memory", "cc", "edx", "ecx", "eax");
 }
 
@@ -95,11 +96,11 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
 		call_on_stack(print_stack_overflow, isp);
 
 	asm volatile("xchgl	%%ebx,%%esp	\n"
-		     "call	*%%edi		\n"
+		     CALL_NOSPEC
 		     "movl	%%ebx,%%esp	\n"
 		     : "=a" (arg1), "=b" (isp)
 		     :  "0" (desc),   "1" (isp),
-			"D" (desc->handle_irq)
+			[thunk_target] "D" (desc->handle_irq)
 		     : "memory", "cc", "ecx");
 	return 1;
 }
-- 
2.7.4

[tip:x86/pti] x86/retpoline/irq32: Convert assembler indirect jumps

[tip-bot for Andi Kleen]

Commit-ID:  7614e913db1f40fff819b36216484dc3808995d4
Gitweb:     https://git.kernel.org/tip/7614e913db1f40fff819b36216484dc3808995d4
Author:     Andi Kleen <ak@linux.intel.com>
AuthorDate: Thu, 11 Jan 2018 21:46:33 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:32 +0100

x86/retpoline/irq32: Convert assembler indirect jumps

Convert all indirect jumps in 32bit irq inline asm code to use non
speculative sequences.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-12-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/kernel/irq_32.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index a83b334..c1bdbd3 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -20,6 +20,7 @@
 #include <linux/mm.h>
 
 #include <asm/apic.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 
@@ -55,11 +56,11 @@ DEFINE_PER_CPU(struct irq_stack *, softirq_stack);
 static void call_on_stack(void *func, void *stack)
 {
 	asm volatile("xchgl	%%ebx,%%esp	\n"
-		     "call	*%%edi		\n"
+		     CALL_NOSPEC
 		     "movl	%%ebx,%%esp	\n"
 		     : "=b" (stack)
 		     : "0" (stack),
-		       "D"(func)
+		       [thunk_target] "D"(func)
 		     : "memory", "cc", "edx", "ecx", "eax");
 }
 
@@ -95,11 +96,11 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
 		call_on_stack(print_stack_overflow, isp);
 
 	asm volatile("xchgl	%%ebx,%%esp	\n"
-		     "call	*%%edi		\n"
+		     CALL_NOSPEC
 		     "movl	%%ebx,%%esp	\n"
 		     : "=a" (arg1), "=b" (isp)
 		     :  "0" (desc),   "1" (isp),
-			"D" (desc->handle_irq)
+			[thunk_target] "D" (desc->handle_irq)
 		     : "memory", "cc", "ecx");
 	return 1;
 }

[PATCH v8 12/12] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 arch/x86/include/asm/nospec-branch.h | 73 +++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm.c                   |  4 ++
 arch/x86/kvm/vmx.c                   |  4 ++
 3 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..475ab0c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,43 @@
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS		16	/* To avoid underflow */
+
+#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)	\
+	mov	$(nr/2), reg;			\
+.Ldo_call1_ ## uniq:				\
+	call	.Ldo_call2_ ## uniq;		\
+.Ltrap1_ ## uniq:				\
+	pause;					\
+	jmp	.Ltrap1_ ## uniq;		\
+.Ldo_call2_ ## uniq:				\
+	call	.Ldo_loop_ ## uniq;		\
+.Ltrap2_ ## uniq:				\
+	pause;					\
+	jmp	.Ltrap2_ ## uniq;		\
+.Ldo_loop_ ## uniq:				\
+	dec	reg;				\
+	jnz	.Ldo_call1_ ## uniq;		\
+	add	$(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +113,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP,\@))	\
+		\ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
@@ -119,7 +170,7 @@
 	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +185,25 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+	unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+		      ALTERNATIVE("jmp 910f",
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1, __LINE__)),
+				  X86_FEATURE_RETPOLINE)
+		      "910:"
+		      : "=&r" (loops), ASM_CALL_CONSTRAINT
+		      : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 #ifdef CONFIG_X86_64
 	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 
 #include "trace.h"
 #include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
 	if (debugctlmsr)
 		update_debugctlmsr(debugctlmsr);
-- 
2.7.4

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  85ec967c1dc04bde16d783ea04428bef3c00a171
Gitweb:     https://git.kernel.org/tip/85ec967c1dc04bde16d783ea04428bef3c00a171
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Thu, 11 Jan 2018 21:46:34 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 00:14:32 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, all entries in the RSB
must be overwrite on exiting a guest, to prevent malicious branch target
predictions from affecting the host kernel.

This is needed both for retpoline and for IBRS.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-13-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 73 +++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm.c                   |  4 ++
 arch/x86/kvm/vmx.c                   |  4 ++
 3 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..475ab0c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,43 @@
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS		16	/* To avoid underflow */
+
+#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)	\
+	mov	$(nr/2), reg;			\
+.Ldo_call1_ ## uniq:				\
+	call	.Ldo_call2_ ## uniq;		\
+.Ltrap1_ ## uniq:				\
+	pause;					\
+	jmp	.Ltrap1_ ## uniq;		\
+.Ldo_call2_ ## uniq:				\
+	call	.Ldo_loop_ ## uniq;		\
+.Ltrap2_ ## uniq:				\
+	pause;					\
+	jmp	.Ltrap2_ ## uniq;		\
+.Ldo_loop_ ## uniq:				\
+	dec	reg;				\
+	jnz	.Ldo_call1_ ## uniq;		\
+	add	$(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +113,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP,\@))	\
+		\ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
@@ -119,7 +170,7 @@
 	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +185,25 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+	unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+		      ALTERNATIVE("jmp 910f",
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1, __LINE__)),
+				  X86_FEATURE_RETPOLINE)
+		      "910:"
+		      : "=&r" (loops), ASM_CALL_CONSTRAINT
+		      : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 #ifdef CONFIG_X86_64
 	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 
 #include "trace.h"
 #include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
 	if (debugctlmsr)
 		update_debugctlmsr(debugctlmsr);

Re: [PATCH v8 12/12] x86/retpoline: Fill return stack buffer on vmexit

[Andi Kleen]

> +/*
> + * On VMEXIT we must ensure that no RSB predictions learned in the guest
> + * can be followed in the host, by overwriting the RSB completely. Both
> + * retpoline and IBRS mitigations for Spectre v2 need this; only on future
> + * CPUs with IBRS_ATT *might* it be avoided.
> + */
> +static inline void vmexit_fill_RSB(void)
> +{
> +#ifdef CONFIG_RETPOLINE
> +	unsigned long loops = RSB_CLEAR_LOOPS / 2;
> +
> +	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
> +		      ALTERNATIVE("jmp 910f",
> +				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1, __LINE__)),

This __LINE__ thing is broken. The expanded assembler doesn't use the number,
but ends up with a label with __LINE__, and if you actually use the inline
multiple times in a file you end up with

/home/ak/lsrc/git/linux/arch/x86/include/asm/nospec-branch.h:241: Error: symbol `.Ldo_call1___LINE__' is already defined
/home/ak/lsrc/git/linux/arch/x86/include/asm/nospec-branch.h:241: Error: symbol `.Ltrap1___LINE__' is already defined
/home/ak/lsrc/git/linux/arch/x86/include/asm/nospec-branch.h:241: Error: symbol `.Ldo_call2___LINE__' is already defined
/home/ak/lsrc/git/linux/arch/x86/include/asm/nospec-branch.h:241: Error: symbol `.Ltrap2___LINE__' is already defined
/home/ak/lsrc/git/linux/arch/x86/include/asm/nospec-branch.h:241: Error: symbol `.Ldo_loop___LINE__' is already defined

I used this incremential patch, which seems to fix that:


diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 475ab0cb80c0..53ea99bf5ed5 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -27,21 +27,21 @@
 #define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
 #define RSB_FILL_LOOPS		16	/* To avoid underflow */
 
-#define __FILL_RETURN_BUFFER(reg, nr, sp, uniq)	\
+#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
 	mov	$(nr/2), reg;			\
-.Ldo_call1_ ## uniq:				\
-	call	.Ldo_call2_ ## uniq;		\
-.Ltrap1_ ## uniq:				\
+771:						\
+	call	772f;				\
+773:						\
 	pause;					\
-	jmp	.Ltrap1_ ## uniq;		\
-.Ldo_call2_ ## uniq:				\
-	call	.Ldo_loop_ ## uniq;		\
-.Ltrap2_ ## uniq:				\
+	jmp	773b;				\
+772:						\
+	call	774f;				\
+775:						\
 	pause;					\
-	jmp	.Ltrap2_ ## uniq;		\
-.Ldo_loop_ ## uniq:				\
+	jmp	775b;				\
+774:						\
 	dec	reg;				\
-	jnz	.Ldo_call1_ ## uniq;		\
+	jnz	771b;				\
 	add	$(BITS_PER_LONG/8) * nr, sp;
 
 #ifdef __ASSEMBLY__
@@ -121,7 +121,7 @@
 #ifdef CONFIG_RETPOLINE
 	ANNOTATE_NOSPEC_ALTERNATIVE
 	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
-		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP,\@))	\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
 		\ftr
 .Lskip_rsb_\@:
 #endif
@@ -198,10 +198,10 @@ static inline void vmexit_fill_RSB(void)
 
 	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
 		      ALTERNATIVE("jmp 910f",
-				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1, __LINE__)),
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
 				  X86_FEATURE_RETPOLINE)
 		      "910:"
 		      : "=&r" (loops), ASM_CALL_CONSTRAINT
 		      : "r" (loops) : "memory" );
 #endif
 }

[PATCH v8.1 12/12] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
I love the smell of bikeshed paint in the morning. But to be fair, this
one was actually an issue which might possibly have bitten in the future.

Can we please stop arguing about asm labels now though? Let's get this
stuff done, and we can set about the oh-so-important task of persuading
Linus to eliminate all numeric labels and rely on human-readable labels
with %= and \@ to make them unique, some time after the dust settles.

 arch/x86/include/asm/nospec-branch.h | 78 +++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm.c                   |  4 ++
 arch/x86/kvm/vmx.c                   |  4 ++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS		16	/* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
+	mov	$(nr/2), reg;			\
+771:						\
+	call	772f;				\
+773:	/* speculation trap */			\
+	pause;					\
+	jmp	773b;				\
+772:						\
+	call	774f;				\
+775:	/* speculation trap */			\
+	pause;					\
+	jmp	775b;				\
+774:						\
+	dec	reg;				\
+	jnz	771b;				\
+	add	$(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +118,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
+		\ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
@@ -119,7 +175,7 @@
 	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+	unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+		      ALTERNATIVE("jmp 910f",
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+				  X86_FEATURE_RETPOLINE)
+		      "910:"
+		      : "=&r" (loops), ASM_CALL_CONSTRAINT
+		      : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 #ifdef CONFIG_X86_64
 	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 
 #include "trace.h"
 #include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
 	if (debugctlmsr)
 		update_debugctlmsr(debugctlmsr);
-- 
2.7.4

Re: [PATCH v8.1 12/12] x86/retpoline: Fill return stack buffer on vmexit

[Thomas Gleixner]

On Fri, 12 Jan 2018, David Woodhouse wrote:

> In accordance with the Intel and AMD documentation, we need to overwrite
> all entries in the RSB on exiting a guest, to prevent malicious branch
> target predictions from affecting the host kernel. This is needed both
> for retpoline and for IBRS.
> 
> [ak: numbers again for the RSB stuffing labels]
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
> ---
> I love the smell of bikeshed paint in the morning. But to be fair, this
> one was actually an issue which might possibly have bitten in the future.
> 
> Can we please stop arguing about asm labels now though? Let's get this
> stuff done, and we can set about the oh-so-important task of persuading
> Linus to eliminate all numeric labels and rely on human-readable labels
> with %= and \@ to make them unique, some time after the dust settles.

Fair enough. I surely like the below way more than the sloppy hackery from
Andi which completely removed any form of documentation.

> +#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
> +	mov	$(nr/2), reg;			\
> +771:						\
> +	call	772f;				\
> +773:	/* speculation trap */			\
> +	pause;					\
> +	jmp	773b;				\
> +772:						\
> +	call	774f;				\
> +775:	/* speculation trap */			\
> +	pause;					\
> +	jmp	775b;				\
> +774:						\
> +	dec	reg;				\
> +	jnz	771b;				\
> +	add	$(BITS_PER_LONG/8) * nr, sp;
> +

Re: [PATCH v8.1 12/12] x86/retpoline: Fill return stack buffer on vmexit

[Woodhouse, David]

[-- Attachment #1: Type: text/plain, Size: 662 bytes --]

> On Fri, 2018-01-12 at 12:15 +0100, Thomas Gleixner wrote:
> Fair enough. I surely like the below way more than the sloppy hackery from
> Andi which completely removed any form of documentation.

Be nice. Andi has been extremely helpful in testing and finding corner
cases here, and generally keeping me honest — thanks, Andi.

And I'd have done it *precisely* the same way if you hadn't been
whining on IRC about documenting it. There's a ten-line comment right
above it, saying what it's doing and why. The addition of a couple of
tiny /* speculation trap */ comments inline is really not that much of
a benefit. I just did it to shut you up :)

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5210 bytes --]

[tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[tip-bot for David Woodhouse]

Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
Gitweb:     https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Fri, 12 Jan 2018 11:11:27 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Fri, 12 Jan 2018 12:33:37 +0100

x86/retpoline: Fill return stack buffer on vmexit

In accordance with the Intel and AMD documentation, we need to overwrite
all entries in the RSB on exiting a guest, to prevent malicious branch
target predictions from affecting the host kernel. This is needed both
for retpoline and for IBRS.

[ak: numbers again for the RSB stuffing labels]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515755487-8524-1-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/include/asm/nospec-branch.h | 78 +++++++++++++++++++++++++++++++++++-
 arch/x86/kvm/svm.c                   |  4 ++
 arch/x86/kvm/vmx.c                   |  4 ++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ea034fa..402a11c 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -7,6 +7,48 @@
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
 
+/*
+ * Fill the CPU return stack buffer.
+ *
+ * Each entry in the RSB, if used for a speculative 'ret', contains an
+ * infinite 'pause; jmp' loop to capture speculative execution.
+ *
+ * This is required in various cases for retpoline and IBRS-based
+ * mitigations for the Spectre variant 2 vulnerability. Sometimes to
+ * eliminate potentially bogus entries from the RSB, and sometimes
+ * purely to ensure that it doesn't get empty, which on some CPUs would
+ * allow predictions from other (unwanted!) sources to be used.
+ *
+ * We define a CPP macro such that it can be used from both .S files and
+ * inline assembly. It's possible to do a .macro and then include that
+ * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
+ */
+
+#define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
+#define RSB_FILL_LOOPS		16	/* To avoid underflow */
+
+/*
+ * Google experimented with loop-unrolling and this turned out to be
+ * the optimal version — two calls, each with their own speculation
+ * trap should their return address end up getting used, in a loop.
+ */
+#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
+	mov	$(nr/2), reg;			\
+771:						\
+	call	772f;				\
+773:	/* speculation trap */			\
+	pause;					\
+	jmp	773b;				\
+772:						\
+	call	774f;				\
+775:	/* speculation trap */			\
+	pause;					\
+	jmp	775b;				\
+774:						\
+	dec	reg;				\
+	jnz	771b;				\
+	add	$(BITS_PER_LONG/8) * nr, sp;
+
 #ifdef __ASSEMBLY__
 
 /*
@@ -76,6 +118,20 @@
 #endif
 .endm
 
+ /*
+  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
+  * monstrosity above, manually.
+  */
+.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
+#ifdef CONFIG_RETPOLINE
+	ANNOTATE_NOSPEC_ALTERNATIVE
+	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
+		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
+		\ftr
+.Lskip_rsb_\@:
+#endif
+.endm
+
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
@@ -119,7 +175,7 @@
 	X86_FEATURE_RETPOLINE)
 
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
-#else /* No retpoline */
+#else /* No retpoline for C / inline asm */
 # define CALL_NOSPEC "call *%[thunk_target]\n"
 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 #endif
@@ -134,5 +190,25 @@ enum spectre_v2_mitigation {
 	SPECTRE_V2_IBRS,
 };
 
+/*
+ * On VMEXIT we must ensure that no RSB predictions learned in the guest
+ * can be followed in the host, by overwriting the RSB completely. Both
+ * retpoline and IBRS mitigations for Spectre v2 need this; only on future
+ * CPUs with IBRS_ATT *might* it be avoided.
+ */
+static inline void vmexit_fill_RSB(void)
+{
+#ifdef CONFIG_RETPOLINE
+	unsigned long loops = RSB_CLEAR_LOOPS / 2;
+
+	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
+		      ALTERNATIVE("jmp 910f",
+				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
+				  X86_FEATURE_RETPOLINE)
+		      "910:"
+		      : "=&r" (loops), ASM_CALL_CONSTRAINT
+		      : "r" (loops) : "memory" );
+#endif
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b..2744b973 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -45,6 +45,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
 #include "trace.h"
@@ -4985,6 +4986,9 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 #ifdef CONFIG_X86_64
 	wrmsrl(MSR_GS_BASE, svm->host.gs_base);
 #else
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 62ee436..d1e25db 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -50,6 +50,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 
 #include "trace.h"
 #include "pmu.h"
@@ -9403,6 +9404,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/* Eliminate branch target predictions from guest mode */
+	vmexit_fill_RSB();
+
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
 	if (debugctlmsr)
 		update_debugctlmsr(debugctlmsr);

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Fri, Jan 12, 2018 at 03:37:49AM -0800, tip-bot for David Woodhouse wrote:
> Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
> Gitweb:     https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
> Author:     David Woodhouse <dwmw@amazon.co.uk>
> AuthorDate: Fri, 12 Jan 2018 11:11:27 +0000
> Committer:  Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Fri, 12 Jan 2018 12:33:37 +0100
> 
> x86/retpoline: Fill return stack buffer on vmexit

...

> + /*
> +  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
> +  * monstrosity above, manually.
> +  */
> +.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
> +#ifdef CONFIG_RETPOLINE
> +	ANNOTATE_NOSPEC_ALTERNATIVE
> +	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
> +		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
> +		\ftr
> +.Lskip_rsb_\@:
> +#endif
> +.endm

Looks unused to me. Was it ever gound to be used? If not:

---
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -121,20 +121,6 @@
 #endif
 .endm
 
- /*
-  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
-  * monstrosity above, manually.
-  */
-.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
-#ifdef CONFIG_RETPOLINE
-	ANNOTATE_NOSPEC_ALTERNATIVE
-	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
-		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
-		\ftr
-.Lskip_rsb_\@:
-#endif
-.endm
-
 #else /* __ASSEMBLY__ */
 
 #define ANNOTATE_NOSPEC_ALTERNATIVE				\

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Thomas Gleixner]

On Sun, 14 Jan 2018, Borislav Petkov wrote:

> On Fri, Jan 12, 2018 at 03:37:49AM -0800, tip-bot for David Woodhouse wrote:
> > Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
> > Gitweb:     https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
> > Author:     David Woodhouse <dwmw@amazon.co.uk>
> > AuthorDate: Fri, 12 Jan 2018 11:11:27 +0000
> > Committer:  Thomas Gleixner <tglx@linutronix.de>
> > CommitDate: Fri, 12 Jan 2018 12:33:37 +0100
> > 
> > x86/retpoline: Fill return stack buffer on vmexit
> 
> ...
> 
> > + /*
> > +  * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
> > +  * monstrosity above, manually.
> > +  */
> > +.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
> > +#ifdef CONFIG_RETPOLINE
> > +	ANNOTATE_NOSPEC_ALTERNATIVE
> > +	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
> > +		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
> > +		\ftr
> > +.Lskip_rsb_\@:
> > +#endif
> > +.endm
> 
> Looks unused to me. Was it ever gound to be used? If not:

It's for the outstanding RSB fill after context switch.

Thanks,

	tglx

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Fri, Jan 12, 2018 at 03:37:49AM -0800, tip-bot for David Woodhouse wrote:
> Commit-ID:  117cc7a908c83697b0b737d15ae1eb5943afe35b
> Gitweb:     https://git.kernel.org/tip/117cc7a908c83697b0b737d15ae1eb5943afe35b
> Author:     David Woodhouse <dwmw@amazon.co.uk>
> AuthorDate: Fri, 12 Jan 2018 11:11:27 +0000
> Committer:  Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Fri, 12 Jan 2018 12:33:37 +0100
> 
> x86/retpoline: Fill return stack buffer on vmexit

...

> +/*
> + * Google experimented with loop-unrolling and this turned out to be
> + * the optimal version — two calls, each with their own speculation
> + * trap should their return address end up getting used, in a loop.
> + */
> +#define __FILL_RETURN_BUFFER(reg, nr, sp)	\
> +	mov	$(nr/2), reg;			\
> +771:						\
> +	call	772f;				\
> +773:	/* speculation trap */			\
> +	pause;					\
> +	jmp	773b;				\
> +772:						\
> +	call	774f;				\
> +775:	/* speculation trap */			\
> +	pause;					\
> +	jmp	775b;				\
> +774:						\
> +	dec	reg;				\
> +	jnz	771b;				\
> +	add	$(BITS_PER_LONG/8) * nr, sp;
> +

Btw, just a minor nit: one could finish the lines with \n\t - the lines
preceding labels only with \n - so that the asm output looks readable.

Right now it is a single line of instructions. But I can fix that later.

---
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -33,23 +33,23 @@
  * trap should their return address end up getting used, in a loop.
  */
 #define __FILL_RETURN_BUFFER(reg, nr, sp)	\
-	mov	$(nr/2), reg;			\
-771:						\
-	call	772f;				\
-773:	/* speculation trap */			\
-	pause;					\
-	lfence;					\
-	jmp	773b;				\
-772:						\
-	call	774f;				\
-775:	/* speculation trap */			\
-	pause;					\
-	lfence;					\
-	jmp	775b;				\
-774:						\
-	dec	reg;				\
-	jnz	771b;				\
-	add	$(BITS_PER_LONG/8) * nr, sp;
+	mov	$(nr/2), reg\n			\
+771:\n\t					\
+	call	772f\n				\
+773:/* speculation trap */\n\t			\
+	pause\n\t				\
+	lfence\n\t				\
+	jmp	773b\n				\
+772:\n\t					\
+	call	774f\n				\
+775:	/* speculation trap */\n\t		\
+	pause\n\t				\
+	lfence\n\t				\
+	jmp	775b\n				\
+774:\n\t					\
+	dec	reg\n\t				\
+	jnz	771b\n\t			\
+	add	$(BITS_PER_LONG/8) * nr, sp\n
 
 #ifdef __ASSEMBLY__

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Fri, Jan 12, 2018 at 03:37:49AM -0800, tip-bot for David Woodhouse wrote:
> +/*
> + * On VMEXIT we must ensure that no RSB predictions learned in the guest
> + * can be followed in the host, by overwriting the RSB completely. Both
> + * retpoline and IBRS mitigations for Spectre v2 need this; only on future
> + * CPUs with IBRS_ATT *might* it be avoided.
> + */
> +static inline void vmexit_fill_RSB(void)
> +{
> +#ifdef CONFIG_RETPOLINE
> +	unsigned long loops = RSB_CLEAR_LOOPS / 2;
> +
> +	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
> +		      ALTERNATIVE("jmp 910f",
> +				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
> +				  X86_FEATURE_RETPOLINE)
> +		      "910:"
> +		      : "=&r" (loops), ASM_CALL_CONSTRAINT
> +		      : "r" (loops) : "memory" );
> +#endif
> +}

Btw,

this thing is a real pain to backport to older kernels without breaking
kABI (I know, I know, latter sucks but it is what it is) so I'm thinking
we could simplify that thing regardless.

As it is, 41 bytes get replaced currently:

[    0.437005] apply_alternatives: feat: 7*32+12, old: (        (ptrval), len: 43), repl: (        (ptrval), len: 43), pad: 41
[    0.438885]         (ptrval): old_insn: eb 29 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90
[    0.440001]         (ptrval): rpl_insn: 48 c7 c0 10 00 00 00 e8 07 00 00 00 f3 90 0f ae e8 eb f9 e8 07 00 00 00 f3 90 0f ae e8 eb f9 48 ff c8 75 e3 48 81 c4 00 01 00 00
[    0.444002]         (ptrval): final_insn: 48 c7 c0 10 00 00 00 e8 07 00 00 00 f3 90 0f ae e8 eb f9 e8 07 00 00 00 f3 90 0f ae e8 eb f9 48 ff c8 75 e3 48 81 c4 00 01 00 00

Not that it doesn't work but the less bytes we replace, the better.

So it could be made to simply call two functions. The replacing then
turns into:

[    0.438154] apply_alternatives: feat: 7*32+12, old: (ffffffff81060b60 len: 5), repl: (ffffffff82434692, len: 5), pad: 0
[    0.440002] ffffffff81060b60: old_insn: e8 ab 73 01 00
[    0.441003] ffffffff82434692: rpl_insn: e8 89 38 c4 fe
[    0.441966] apply_alternatives: Fix CALL offset: 0x173bb, CALL 0xffffffff81077f20
[    0.444002] ffffffff81060b60: final_insn: e8 bb 73 01 00

The "old_insn" above is:
ffffffff81060b60:       e8 ab 73 01 00          callq  ffffffff81077f10 <__fill_rsb_nop>

and final_insn is
ffffffff82434692:       e8 89 38 c4 fe          callq  ffffffff81077f20 <__fill_rsb>

so both CALLs with immediates for which there's no speculation going on.

I had to add a new alternative_void_call() macro for that but that's
straight-forward. Also, this causes objtool to segfault so Josh and I
need to look at that first.

Other than that, it gets a lot simpler this way:

--
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index cf5961ca8677..a84863c1b7d3 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -210,6 +210,11 @@ static inline int alternatives_text_reserved(void *start, void *end)
 	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
 		: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
 
+/* Like alternative_io, but for replacing a direct call with another one. */
+#define alternative_void_call(oldfunc, newfunc, feature, input...)		\
+	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature)	\
+		: : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
+
 /*
  * Like alternative_call, but there are two features and respective functions.
  * If CPU has feature2, function2 is used.
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 4ad41087ce0e..5ba951c208af 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
-#ifndef __NOSPEC_BRANCH_H__
-#define __NOSPEC_BRANCH_H__
+#ifndef __X86_ASM_NOSPEC_BRANCH_H__
+#define __X86_ASM_NOSPEC_BRANCH_H__
 
 #include <asm/alternative.h>
 #include <asm/alternative-asm.h>
@@ -206,17 +206,9 @@ extern char __indirect_thunk_end[];
 static inline void vmexit_fill_RSB(void)
 {
 #ifdef CONFIG_RETPOLINE
-	unsigned long loops;
-
-	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
-		      ALTERNATIVE("jmp 910f",
-				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
-				  X86_FEATURE_RETPOLINE)
-		      "910:"
-		      : "=r" (loops), ASM_CALL_CONSTRAINT
-		      : : "memory" );
+	alternative_void_call(__fill_rsb_nop, __fill_rsb, X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory"));
 #endif
 }
 
 #endif /* __ASSEMBLY__ */
-#endif /* __NOSPEC_BRANCH_H__ */
+#endif /* __X86_ASM_NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d3a67fba200a..b6a002bf893b 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -971,4 +971,9 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
+
+#ifdef CONFIG_RETPOLINE
+void __fill_rsb_nop(void);
+void __fill_rsb(void);
+#endif
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 390b3dc3d438..16cc2e73d17d 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -281,3 +281,19 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 #endif
+
+#ifdef CONFIG_RETPOLINE
+void __fill_rsb_nop(void)
+{
+	cpu_relax();
+}
+
+void __fill_rsb(void)
+{
+	unsigned long loops;
+
+	asm volatile (__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1))
+		      : "=r" (loops), ASM_CALL_CONSTRAINT
+		      : : "memory" );
+}
+#endif

Thx.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 4790 bytes --]

On Thu, 2018-01-25 at 13:07 +0100, Borislav Petkov wrote:
> On Fri, Jan 12, 2018 at 03:37:49AM -0800, tip-bot for David Woodhouse wrote:
> > 
> > +/*
> > + * On VMEXIT we must ensure that no RSB predictions learned in the guest
> > + * can be followed in the host, by overwriting the RSB completely. Both
> > + * retpoline and IBRS mitigations for Spectre v2 need this; only on future
> > + * CPUs with IBRS_ATT *might* it be avoided.
> > + */
> > +static inline void vmexit_fill_RSB(void)
> > +{
> > +#ifdef CONFIG_RETPOLINE
> > +	unsigned long loops = RSB_CLEAR_LOOPS / 2;
> > +
> > +	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
> > +		      ALTERNATIVE("jmp 910f",
> > +				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
> > +				  X86_FEATURE_RETPOLINE)
> > +		      "910:"
> > +		      : "=&r" (loops), ASM_CALL_CONSTRAINT
> > +		      : "r" (loops) : "memory" );
> > +#endif
> > +}
> Btw,
> 
> this thing is a real pain to backport to older kernels without breaking
> kABI (I know, I know, latter sucks but it is what it is) 

I haven't had lunch yet, so I don't feel queasy and I'm vaguely
interested... *why* does it break kABI?

> so I'm thinking we could simplify that thing regardless.
> 
> As it is, 41 bytes get replaced currently:
> 
> [    0.437005] apply_alternatives: feat: 7*32+12, old: (        (ptrval), len: 43), repl: (        (ptrval), len: 43), pad: 41
> [    0.438885]         (ptrval): old_insn: eb 29 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90
> [    0.440001]         (ptrval): rpl_insn: 48 c7 c0 10 00 00 00 e8 07 00 00 00 f3 90 0f ae e8 eb f9 e8 07 00 00 00 f3 90 0f ae e8 eb f9 48 ff c8 75 e3 48 81 c4 00 01 00 00
> [    0.444002]         (ptrval): final_insn: 48 c7 c0 10 00 00 00 e8 07 00 00 00 f3 90 0f ae e8 eb f9 e8 07 00 00 00 f3 90 0f ae e8 eb f9 48 ff c8 75 e3 48 81 c4 00 01 00 00
> 
> Not that it doesn't work but the less bytes we replace, the better.
> 
> So it could be made to simply call two functions. The replacing then
> turns into:
> 
> [    0.438154] apply_alternatives: feat: 7*32+12, old: (ffffffff81060b60 len: 5), repl: (ffffffff82434692, len: 5), pad: 0
> [    0.440002] ffffffff81060b60: old_insn: e8 ab 73 01 00
> [    0.441003] ffffffff82434692: rpl_insn: e8 89 38 c4 fe
> [    0.441966] apply_alternatives: Fix CALL offset: 0x173bb, CALL 0xffffffff81077f20
> [    0.444002] ffffffff81060b60: final_insn: e8 bb 73 01 00
> 
> The "old_insn" above is:
> ffffffff81060b60:       e8 ab 73 01
> 00          callq  ffffffff81077f10 <__fill_rsb_nop>
> 
> and final_insn is
> ffffffff82434692:       e8 89 38 c4
> fe          callq  ffffffff81077f20 <__fill_rsb>
> 
> so both CALLs with immediates for which there's no speculation going
> on.
> 
> I had to add a new alternative_void_call() macro for that but that's
> straight-forward. Also, this causes objtool to segfault so Josh and I
> need to look at that first.
> 
> Other than that, it gets a lot simpler this way:
> 
> --
> diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
> index cf5961ca8677..a84863c1b7d3 100644
> --- a/arch/x86/include/asm/alternative.h
> +++ b/arch/x86/include/asm/alternative.h
> @@ -210,6 +210,11 @@ static inline int alternatives_text_reserved(void *start, void *end)
>  	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
>  		: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
>  
> +/* Like alternative_io, but for replacing a direct call with another one. */
> +#define alternative_void_call(oldfunc, newfunc, feature, input...)		\
> +	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature)	\
> +		: : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)

But you aren't doing the call at all in the other case, and
alternatives *always* handled the case where the first 'alternative'
instruction was a branch, didn't it?

So couldn't it just be alternative(nop, call __fill_rsb_func)?

But I still don't understand why it matters.

> +void __fill_rsb(void)
> +{
> +	unsigned long loops;
> +
> +	asm volatile (__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1))
> +		      : "=r" (loops), ASM_CALL_CONSTRAINT
> +		      : : "memory" );
> +}
> +#endif

The out-of-line function should be __clear_rsb() if it's using
RSB_CLEAR_LOOPS, and __fill_rsb() if it's using RSB_FILL_LOOPS. I think
we're only using one right now but Andi at least is posting patches
which use the other, as part of the Skylake clusterfuck.

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Thu, Jan 25, 2018 at 12:20:49PM +0000, David Woodhouse wrote:
> I haven't had lunch yet, so I don't feel queasy

Oh, I caught you on time then :-)

> and I'm vaguely interested... *why* does it break kABI?

Kernels < 4.1 don't have the insn padding I did back then so when one
tries to backport this, struct alt_instr size changes. Fun.

> > +/* Like alternative_io, but for replacing a direct call with another one. */
> > +#define alternative_void_call(oldfunc, newfunc, feature, input...)		\
> > +	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature)	\
> > +		: : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
> 
> But you aren't doing the call at all in the other case, and
> alternatives *always* handled the case where the first 'alternative'
> instruction was a branch, didn't it?
> 
> So couldn't it just be alternative(nop, call __fill_rsb_func)?
> 
> But I still don't understand why it matters.

You need for both to be CALL instructions there so that gcc can manage
the callee clobbers properly - i.e., recognize that there's a function
call there.

Otherwise you need to do uglies like the hunk in arch/x86/Kconfig in

d61931d89be5 ("x86: Add optimized popcnt variants")

which I killed later after all as it broke profiling and other tools.

And besides, calling a NOPpy function on CONFIG_RETPOLINE=n is the least
of our troubles. Also, the installations running CONFIG_RETPOLINE=n are
going to be a very very very small number.

> > +void __fill_rsb(void)
> > +{
> > +	unsigned long loops;
> > +
> > +	asm volatile (__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1))
> > +		      : "=r" (loops), ASM_CALL_CONSTRAINT
> > +		      : : "memory" );
> > +}
> > +#endif
> 
> The out-of-line function should be __clear_rsb() if it's using
> RSB_CLEAR_LOOPS, and __fill_rsb() if it's using RSB_FILL_LOOPS. I think
> we're only using one right now but Andi at least is posting patches
> which use the other, as part of the Skylake clusterfuck.

Ok.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Josh Poimboeuf]

On Thu, Jan 25, 2018 at 01:45:54PM +0100, Borislav Petkov wrote:
> > > +/* Like alternative_io, but for replacing a direct call with another one. */
> > > +#define alternative_void_call(oldfunc, newfunc, feature, input...)		\
> > > +	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature)	\
> > > +		: : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
> > 
> > But you aren't doing the call at all in the other case, and
> > alternatives *always* handled the case where the first 'alternative'
> > instruction was a branch, didn't it?
> > 
> > So couldn't it just be alternative(nop, call __fill_rsb_func)?
> > 
> > But I still don't understand why it matters.
> 
> You need for both to be CALL instructions there so that gcc can manage
> the callee clobbers properly - i.e., recognize that there's a function
> call there.

Huh?  GCC doesn't even look inside the inline asm.  That's why we had to
implement ASM_CALL_CONSTRAINT.

And the seg fault is objtool's way of telling you you need a
ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)

(I know that's not the best answer, will fix it soon.)

-- 
Josh

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Thu, Jan 25, 2018 at 09:10:24AM -0600, Josh Poimboeuf wrote:
> Huh?  GCC doesn't even look inside the inline asm.  That's why we had to
> implement ASM_CALL_CONSTRAINT.

That wasn't very correct. What I meant was: *we* need to tell gcc that
the inline asm *might* clobber registers and which those might be. What
David suggested with ASM_NO_INPUT_CLOBBER is the proper thing to do.

> And the seg fault is objtool's way of telling you you need a
> ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)

Except that it blew up when I did this which doesn't have ALTERNATIVE
(it's the diff I saved :-))

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 390b3dc3d438..16cc2e73d17d 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -281,3 +281,19 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 #endif
+
+#ifdef CONFIG_RETPOLINE
+void __fill_rsb_nop(void)
+{
+	cpu_relax();
+}
+
+void __fill_rsb(void)
+{
+	unsigned long loops;
+
+	asm volatile (__stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1))
+		      : "=r" (loops), ASM_CALL_CONSTRAINT
+		      : : "memory" );
+}
+#endif
--

> (I know that's not the best answer, will fix it soon.)

Thx.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 603 bytes --]

On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> 
> > And the seg fault is objtool's way of telling you you need a
> > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> 
> Except that it blew up when I did this which doesn't have ALTERNATIVE
> (it's the diff I saved :-))

Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
alternative; you're not going to like it".

If you start putting a __fill_rsb() function out of line somewhere and
only *calling* it from alternatives, then objtool is going to shit
itself when it sees that function, regardless.

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Josh Poimboeuf]

On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
> On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> > 
> > > And the seg fault is objtool's way of telling you you need a
> > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> > 
> > Except that it blew up when I did this which doesn't have ALTERNATIVE
> > (it's the diff I saved :-))
> 
> Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
> alternative; you're not going to like it".
> 
> If you start putting a __fill_rsb() function out of line somewhere and
> only *calling* it from alternatives, then objtool is going to shit
> itself when it sees that function, regardless.

Right, if you *really* want it always inline, the short term solution is
to just patch it in with X86_FEATURE_ALWAYS.

And don't ask me what the long term solution is, I don't even know what
I'm having for lunch today.

-- 
Josh

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 1183 bytes --]

On Thu, 2018-01-25 at 10:56 -0600, Josh Poimboeuf wrote:
> On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
> > On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> > > 
> > > > And the seg fault is objtool's way of telling you you need a
> > > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> > > 
> > > Except that it blew up when I did this which doesn't have ALTERNATIVE
> > > (it's the diff I saved :-))
> > 
> > Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
> > alternative; you're not going to like it".
> > 
> > If you start putting a __fill_rsb() function out of line somewhere and
> > only *calling* it from alternatives, then objtool is going to shit
> > itself when it sees that function, regardless.
> 
> Right, if you *really* want it always inline, the short term solution is
> to just patch it in with X86_FEATURE_ALWAYS.

And the whole problem here is that patching it in with alternatives is
painful on kernels < 4.1 because back then, we didn't cope with
oldinstr and altinstr being different lengths.

And they don't want to fix *that* because kABI...

I just stopped caring.

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Andy Lutomirski]

On Thu, Jan 25, 2018 at 9:00 AM, David Woodhouse <dwmw2@infradead.org> wrote:
> On Thu, 2018-01-25 at 10:56 -0600, Josh Poimboeuf wrote:
>> On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
>> > On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
>> > >
>> > > > And the seg fault is objtool's way of telling you you need a
>> > > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
>> > >
>> > > Except that it blew up when I did this which doesn't have ALTERNATIVE
>> > > (it's the diff I saved :-))
>> >
>> > Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
>> > alternative; you're not going to like it".
>> >
>> > If you start putting a __fill_rsb() function out of line somewhere and
>> > only *calling* it from alternatives, then objtool is going to shit
>> > itself when it sees that function, regardless.
>>
>> Right, if you *really* want it always inline, the short term solution is
>> to just patch it in with X86_FEATURE_ALWAYS.
>
> And the whole problem here is that patching it in with alternatives is
> painful on kernels < 4.1 because back then, we didn't cope with
> oldinstr and altinstr being different lengths.
>
> And they don't want to fix *that* because kABI...
>
> I just stopped caring.

Screw kABI.

Distros that use retpolines need their driver vendors to recompile no
matter what.  Distros that use IBRS and refuse to use retpolines
should get put on a list of "didn't actually adequately mitigate
spectre".

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Josh Poimboeuf]

On Thu, Jan 25, 2018 at 09:05:37AM -0800, Andy Lutomirski wrote:
> On Thu, Jan 25, 2018 at 9:00 AM, David Woodhouse <dwmw2@infradead.org> wrote:
> > On Thu, 2018-01-25 at 10:56 -0600, Josh Poimboeuf wrote:
> >> On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
> >> > On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> >> > >
> >> > > > And the seg fault is objtool's way of telling you you need a
> >> > > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> >> > >
> >> > > Except that it blew up when I did this which doesn't have ALTERNATIVE
> >> > > (it's the diff I saved :-))
> >> >
> >> > Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
> >> > alternative; you're not going to like it".
> >> >
> >> > If you start putting a __fill_rsb() function out of line somewhere and
> >> > only *calling* it from alternatives, then objtool is going to shit
> >> > itself when it sees that function, regardless.
> >>
> >> Right, if you *really* want it always inline, the short term solution is
> >> to just patch it in with X86_FEATURE_ALWAYS.
> >
> > And the whole problem here is that patching it in with alternatives is
> > painful on kernels < 4.1 because back then, we didn't cope with
> > oldinstr and altinstr being different lengths.
> >
> > And they don't want to fix *that* because kABI...
> >
> > I just stopped caring.
> 
> Screw kABI.

There are *many* real world users who depend on kABI, so it's a fact of
life with no better known solution for the given constraints.

If you have a better idea then I suggest you build your own enterprise
distro.

> Distros that use retpolines need their driver vendors to recompile no
> matter what.  Distros that use IBRS and refuse to use retpolines
> should get put on a list of "didn't actually adequately mitigate
> spectre".

Retpolines don't need to break kABI.  We can just detect when they're
missing and report it to the user.

And I don't think there's anybody *refusing* to use retpolines, is
there?  But they're still not fully baked, especially for Skylake+.

IBRS was just easier to implement out of the gate.  And it mitigates
Spectre just fine, but it's too slow to stick with long term.

-- 
Josh

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Jiri Kosina]

On Thu, 25 Jan 2018, Andy Lutomirski wrote:

> Distros that use retpolines need their driver vendors to recompile no 
> matter what.  

Absolutely. Tainting a kernel, issuing a warning, or even voluntarily 
deciding to not load modules loaded without retpolines, that all sounds 
like reasonable aproaches.

Artificially introducing kernel ABI breakage which is not there (as 
retpolines are fully compatible when it comes to ABI between modules and 
kernel ... the fact that it potentially brings non-retpolined indirect 
jump into the kernel is a security concent, but not ABI issue) sounds like 
a bad idea.

Those two things (ABI and security concerns) are independent.

> Distros that use IBRS and refuse to use retpolines should get put on a 
> list of "didn't actually adequately mitigate spectre".

Oh absolutely, especially on archs where there is no IBRS. But how is this 
relevant to ABI?

Thanks,

-- 
Jiri Kosina
SUSE Labs

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Thomas Gleixner]

[-- Attachment #1: Type: text/plain, Size: 1401 bytes --]

On Thu, 25 Jan 2018, David Woodhouse wrote:
> On Thu, 2018-01-25 at 10:56 -0600, Josh Poimboeuf wrote:
> > On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
> > > On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> > > > 
> > > > > And the seg fault is objtool's way of telling you you need a
> > > > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> > > > 
> > > > Except that it blew up when I did this which doesn't have ALTERNATIVE
> > > > (it's the diff I saved :-))
> > > 
> > > Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
> > > alternative; you're not going to like it".
> > > 
> > > If you start putting a __fill_rsb() function out of line somewhere and
> > > only *calling* it from alternatives, then objtool is going to shit
> > > itself when it sees that function, regardless.
> > 
> > Right, if you *really* want it always inline, the short term solution is
> > to just patch it in with X86_FEATURE_ALWAYS.
> 
> And the whole problem here is that patching it in with alternatives is
> painful on kernels < 4.1 because back then, we didn't cope with
> oldinstr and altinstr being different lengths.
> 
> And they don't want to fix *that* because kABI...
> 
> I just stopped caring.

Rightfully so. KABI has often enough proven to be complete garbage. If
people still insist on it, it's _their_ problem not ours.

Thanks,

	tglx

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Josh Poimboeuf]

On Thu, Jan 25, 2018 at 05:00:39PM +0000, David Woodhouse wrote:
> On Thu, 2018-01-25 at 10:56 -0600, Josh Poimboeuf wrote:
> > On Thu, Jan 25, 2018 at 04:03:18PM +0000, David Woodhouse wrote:
> > > On Thu, 2018-01-25 at 16:51 +0100, Borislav Petkov wrote:
> > > > 
> > > > > And the seg fault is objtool's way of telling you you need a
> > > > > ANNOTATE_NOSPEC_ALTERNATIVE above the alternative ;-)
> > > > 
> > > > Except that it blew up when I did this which doesn't have ALTERNATIVE
> > > > (it's the diff I saved :-))
> > > 
> > > Yeah, ANNOTATE_NOSPEC_ALTERNATIVE just tells objtool "don't look at the
> > > alternative; you're not going to like it".
> > > 
> > > If you start putting a __fill_rsb() function out of line somewhere and
> > > only *calling* it from alternatives, then objtool is going to shit
> > > itself when it sees that function, regardless.
> > 
> > Right, if you *really* want it always inline, the short term solution is
> > to just patch it in with X86_FEATURE_ALWAYS.
> 
> And the whole problem here is that patching it in with alternatives is
> painful on kernels < 4.1 because back then, we didn't cope with
> oldinstr and altinstr being different lengths.

We just manually added the nops for the in-line path, that should be
good enough.

-- 
Josh

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Thu, Jan 25, 2018 at 05:00:39PM +0000, David Woodhouse wrote:
> And the whole problem here is that patching it in with alternatives is
> painful on kernels < 4.1 because back then, we didn't cope with
> oldinstr and altinstr being different lengths.
> 
> And they don't want to fix *that* because kABI...

So if it were only because of the KABI, I would never sent a mail on the
list but would've done it in our tress and forgotten about it.

 [ And just to set one thing straight: I'm not the right person to
   complain to about KABI. ]

Now, I happen to think that those macros could be simplified regardless.
And I don't see *anything* wrong with that. Like making them more
readable, simpler, etc, etc. That's the only reason why I raised the
issue here. I mentioned the KABI because I didn't want to leave anything
out from the whole picture.

So forget the KABI angle and think: simpler, cleaner, more readable
macros.

Oh, and David, if while doing so I manage to add the alignment, then
*that* is even better.

Win-win-effing-win situation!

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[David Woodhouse]

[-- Attachment #1: Type: text/plain, Size: 527 bytes --]

On Thu, 2018-01-25 at 18:53 +0100, Borislav Petkov wrote:
> 
> So forget the KABI angle and think: simpler, cleaner, more readable
> macros.
> 
> Oh, and David, if while doing so I manage to add the alignment, then
> *that* is even better.
> 
> Win-win-effing-win situation!

Yep, I'll buy that. But first we need Josh to work out what he's having
for lunch.

Although just another marker to tell objtool "ignore this whole
function" might be sufficient to allow us to have an out-of-line RSB-
stuffing function.

[-- Attachment #2: smime.p7s --]
[-- Type: application/x-pkcs7-signature, Size: 5213 bytes --]

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Josh Poimboeuf]

On Thu, Jan 25, 2018 at 06:04:23PM +0000, David Woodhouse wrote:
> On Thu, 2018-01-25 at 18:53 +0100, Borislav Petkov wrote:
> > 
> > So forget the KABI angle and think: simpler, cleaner, more readable
> > macros.
> > 
> > Oh, and David, if while doing so I manage to add the alignment, then
> > *that* is even better.
> > 
> > Win-win-effing-win situation!
> 
> Yep, I'll buy that. But first we need Josh to work out what he's having
> for lunch.

This is what I've been eating every day:

  https://www.urbandictionary.com/define.php?term=Shit%20Sandwich

:-)

> Although just another marker to tell objtool "ignore this whole
> function" might be sufficient to allow us to have an out-of-line RSB-
> stuffing function.

If the function is only in arch/x86/lib/retpoline.S, we can just tell
objtool to ignore the entire file for now, as the file has no useful
information anyway (until I get around to adding ORC hints to it).  Just
add

  OBJECT_FILES_NON_STANDARD_retpoline.o :=y
  
to the Makefile.

-- 
Josh

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Thu, Jan 25, 2018 at 06:04:23PM +0000, David Woodhouse wrote:
> Yep, I'll buy that. But first we need Josh to work out what he's having
> for lunch.
> 
> Although just another marker to tell objtool "ignore this whole
> function" might be sufficient to allow us to have an out-of-line RSB-
> stuffing function.

Ok, I think we solved it all on IRC.

I'm adding a conglomerate patch at the end. It builds and boots here in
a vm. Here's what I did:

The barrier:

static inline void indirect_branch_prediction_barrier(void)
{
        alternative_input("", "call __ibp_barrier", X86_FEATURE_IBPB, ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
}

with

void __ibp_barrier(void)
{
        wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
}

Alternatives applies correctly:

[    1.040965] apply_alternatives: feat: 7*32+16, old: (ffffffff8259d806 len: 5), repl: (ffffffff826aa88b, len: 5), pad: 5
[    1.042798] ffffffff8259d806: old_insn: 90 90 90 90 90
[    1.044001] ffffffff826aa88b: rpl_insn: e8 80 c3 9c fe
[    1.044938] apply_alternatives: Fix CALL offset: 0xfead9405, CALL 0xffffffff81076c10
[    1.046343] ffffffff8259d806: final_insn: e8 05 94 ad fe

with final_insn becoming:

ffffffff8259d806:       e8 80 c3 9c fe          callq  ffffffff81076c10 <__ibp_barrier>

When we look at the __ibp_barrier:

ffffffff81076c10 <__ibp_barrier>:
ffffffff81076c10:       e8 bb ac 98 00          callq  ffffffff81a018d0 <__fentry__>
ffffffff81076c15:       b9 49 00 00 00          mov    $0x49,%ecx
ffffffff81076c1a:       b8 01 00 00 00          mov    $0x1,%eax
ffffffff81076c1f:       31 d2                   xor    %edx,%edx
ffffffff81076c21:       0f 30                   wrmsr  
ffffffff81076c23:       0f 1f 44 00 00          nopl   0x0(%rax,%rax,1)
ffffffff81076c28:       c3                      retq   
ffffffff81076c29:       31 d2                   xor    %edx,%edx
ffffffff81076c2b:       be 01 00 00 00          mov    $0x1,%esi
ffffffff81076c30:       bf 49 00 00 00          mov    $0x49,%edi
ffffffff81076c35:       e9 76 64 3f 00          jmpq   ffffffff8146d0b0 <do_trace_write_msr>
ffffffff81076c3a:       90                      nop
ffffffff81076c3b:       90                      nop
ffffffff81076c3c:       90                      nop
ffffffff81076c3d:       90                      nop
ffffffff81076c3e:       90                      nop

It has the tracing shit too. We could replace that with __wrmsr() which
does exception handling. And I think we should do that instead.

/me does it.

The RSB filler became:

static inline void vmexit_fill_RSB(void)
{
#ifdef CONFIG_RETPOLINE
        alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory"));
#endif
}

and __fill_rsb_clobber_ax looks like:

ENTRY(__fill_rsb_clobber_ax)
        ___FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, %_ASM_SP
END(__fill_rsb_clobber_ax)

where:

.macro  ___FILL_RETURN_BUFFER reg:req nr:req sp:req
        mov     (\nr / 2), \reg
        .align 16
771:
        call    772f
773:                                            /* speculation trap */
        pause
        lfence
        jmp     773b
        .align 16
772:
        call    774f
775:                                            /* speculation trap */
        pause
        lfence
        jmp     775b
        .align 16
774:
        dec     \reg
        jnz     771b
        add     (BITS_PER_LONG /8) * \nr, \sp
.endm

Please check the alignment directives are at the right places.

Application looks correct too:

[    0.428974] apply_alternatives: feat: 7*32+12, old: (ffffffff8105fe50 len: 5), repl: (ffffffff826aa69f, len: 5), pad: 5
[    0.432003] ffffffff8105fe50: old_insn: 90 90 90 90 90
[    0.432977] ffffffff826aa69f: rpl_insn: e8 5c 8b 55 ff
[    0.433943] apply_alternatives: Fix CALL offset: 0xba33ab, CALL 0xffffffff81c03200
[    0.435368] ffffffff8105fe50: final_insn: e8 ab 33 ba 00

final_insn becomes

ffffffff8105fe50:       e8 5c 8b 55 ff          callq  ffffffff81c03200 <__fill_rsb_clobber_ax>

and the __fill_rsb_clobber_ax asm look good to me too:

ffffffff81c03200 <__fill_rsb_clobber_ax>:
ffffffff81c03200:       48 8b 04 25 10 00 00    mov    0x10,%rax
ffffffff81c03207:       00 
ffffffff81c03208:       0f 1f 84 00 00 00 00    nopl   0x0(%rax,%rax,1)
ffffffff81c0320f:       00 
ffffffff81c03210:       e8 0b 00 00 00          callq  ffffffff81c03220 <__fill_rsb_clobber_ax+0x20>
ffffffff81c03215:       f3 90                   pause  
ffffffff81c03217:       0f ae e8                lfence 
ffffffff81c0321a:       eb f9                   jmp    ffffffff81c03215 <__fill_rsb_clobber_ax+0x15>
ffffffff81c0321c:       0f 1f 40 00             nopl   0x0(%rax)
ffffffff81c03220:       e8 0b 00 00 00          callq  ffffffff81c03230 <__fill_rsb_clobber_ax+0x30>
ffffffff81c03225:       f3 90                   pause  
ffffffff81c03227:       0f ae e8                lfence 
ffffffff81c0322a:       eb f9                   jmp    ffffffff81c03225 <__fill_rsb_clobber_ax+0x25>
ffffffff81c0322c:       0f 1f 40 00             nopl   0x0(%rax)
ffffffff81c03230:       48 ff c8                dec    %rax
ffffffff81c03233:       75 db                   jne    ffffffff81c03210 <__fill_rsb_clobber_ax+0x10>
ffffffff81c03235:       48 03 24 25 00 01 00    add    0x100,%rsp
ffffffff81c0323c:       00

with the respective alignment.

Btw, I've forced

	 setup_force_cpu_cap(X86_FEATURE_IBPB);

so that I can see it applies only - nothing else.

Here's the full diff I have so far:

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 67bbfaa1448b..a7c5602847b1 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -210,6 +210,7 @@
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
 #define X86_FEATURE_AVX512_4FMAPS	( 7*32+17) /* AVX-512 Multiply Accumulation Single precision */
 
+#define X86_FEATURE_IBPB		( 7*32+16) /* Using Indirect Branch Prediction Barrier */
 #define X86_FEATURE_MBA			( 7*32+18) /* Memory Bandwidth Allocation */
 #define X86_FEATURE_RSB_CTXSW		( 7*32+19) /* Fill RSB on context switches */
 
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index e7b983a35506..1c4e87b73789 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -39,6 +39,9 @@
 
 /* Intel MSRs. Some also available on other CPUs */
 
+#define MSR_IA32_PRED_CMD              0x00000049 /* Prediction Command */
+#define PRED_CMD_IBPB                  (1 << 0)   /* Indirect Branch Prediction Barrier */
+
 #define MSR_PPIN_CTL			0x0000004e
 #define MSR_PPIN			0x0000004f
 
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 4ad41087ce0e..3e4d4996374e 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
-#ifndef __NOSPEC_BRANCH_H__
-#define __NOSPEC_BRANCH_H__
+#ifndef __X86_ASM_NOSPEC_BRANCH_H__
+#define __X86_ASM_NOSPEC_BRANCH_H__
 
 #include <asm/alternative.h>
 #include <asm/alternative-asm.h>
@@ -53,6 +53,9 @@
 
 #ifdef __ASSEMBLY__
 
+#include <asm/bitsperlong.h>
+
+
 /*
  * This should be used immediately before a retpoline alternative.  It tells
  * objtool where the retpolines are so that it can make sense of the control
@@ -121,6 +124,29 @@
 #endif
 .endm
 
+.macro  ___FILL_RETURN_BUFFER reg:req nr:req sp:req
+	mov	(\nr / 2), \reg
+	.align 16
+771:
+	call	772f
+773:						/* speculation trap */
+	pause
+	lfence
+	jmp	773b
+	.align 16
+772:
+	call	774f
+775:						/* speculation trap */
+	pause
+	lfence
+	jmp	775b
+	.align 16
+774:
+	dec	\reg
+	jnz	771b
+	add	(BITS_PER_LONG /8) * \nr, \sp
+.endm
+
  /*
   * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
   * monstrosity above, manually.
@@ -206,17 +232,14 @@ extern char __indirect_thunk_end[];
 static inline void vmexit_fill_RSB(void)
 {
 #ifdef CONFIG_RETPOLINE
-	unsigned long loops;
-
-	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
-		      ALTERNATIVE("jmp 910f",
-				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
-				  X86_FEATURE_RETPOLINE)
-		      "910:"
-		      : "=r" (loops), ASM_CALL_CONSTRAINT
-		      : : "memory" );
+	alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory"));
 #endif
 }
 
+static inline void indirect_branch_prediction_barrier(void)
+{
+	alternative_input("", "call __ibp_barrier", X86_FEATURE_IBPB, ASM_NO_INPUT_CLOBBER("eax", "ecx", "edx", "memory"));
+}
+
 #endif /* __ASSEMBLY__ */
-#endif /* __NOSPEC_BRANCH_H__ */
+#endif /* __X86_ASM_NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d3a67fba200a..624bc37dc696 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -971,4 +971,9 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
+
+#ifdef CONFIG_RETPOLINE
+asmlinkage void __fill_rsb_clobber_ax(void);
+void __ibp_barrier(void);
+#endif
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 390b3dc3d438..35f60e5b93bf 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -249,6 +249,10 @@ static void __init spectre_v2_select_mitigation(void)
 		setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 		pr_info("Filling RSB on context switch\n");
 	}
+
+	setup_force_cpu_cap(X86_FEATURE_IBPB);
+
+	indirect_branch_prediction_barrier();
 }
 
 #undef pr_fmt
@@ -281,3 +285,10 @@ ssize_t cpu_show_spectre_v2(struct device *dev,
 	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
 #endif
+
+#ifdef CONFIG_RETPOLINE
+void __ibp_barrier(void)
+{
+	__wrmsr(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, 0);
+}
+#endif
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index f23934bbaf4e..69a473919260 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -27,6 +27,7 @@ lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
 lib-$(CONFIG_RETPOLINE) += retpoline.o
+OBJECT_FILES_NON_STANDARD_retpoline.o :=y
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index dfb2ba91b670..62eaa8f80d30 100644
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -47,3 +47,7 @@ GENERATE_THUNK(r13)
 GENERATE_THUNK(r14)
 GENERATE_THUNK(r15)
 #endif
+
+ENTRY(__fill_rsb_clobber_ax)
+	___FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, %_ASM_SP
+END(__fill_rsb_clobber_ax)
-- 
2.13.0

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [tip:x86/pti] x86/retpoline: Fill return stack buffer on vmexit

[Borislav Petkov]

On Thu, Jan 25, 2018 at 08:07:29PM +0100, Borislav Petkov wrote:
> static inline void vmexit_fill_RSB(void)
> {
> #ifdef CONFIG_RETPOLINE
>         alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER("memory"));

Whoops, forgot the clobbers:

        alternative_input("", "call __fill_rsb_clobber_ax", X86_FEATURE_RETPOLINE, ASM_NO_INPUT_CLOBBER(_ASM_AX, "memory"));

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

[PATCH v7 01/11] x86/retpoline: Add initial retpoline support

[David Woodhouse]

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Acked-By: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 ++++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 112 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++++
 8 files changed, 215 insertions(+)
 create mode 100644 arch/x86/include/asm/nospec-branch.h
 create mode 100644 arch/x86/lib/retpoline.S

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..918e550 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning Retpoline not supported in compiler. System may be insecure.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..da407df
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	.align	16
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+	.align	16
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+	.align	16
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call	*%[thunk_target]\n",				\
+	""							\
+	"       jmp    do_call%=;\n"				\
+	"       .align 16\n"					\
+	"do_retpoline%=:\n"					\
+	"	call   do_rop%=;\n"				\
+	"spec_trap%=:\n"					\
+	"	pause;\n"					\
+	"       jmp    spec_trap%=;\n"				\
+	"       .align 16\n"					\
+	"do_rop%=:\n"						\
+	"	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"do_call%=:\n"						\
+	"	call   do_retpoline%=;\n",			\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif
-- 
2.7.4

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  ce004e1cb670f0bc134be82b24b0d0db16bccc37
Gitweb:     https://git.kernel.org/tip/ce004e1cb670f0bc134be82b24b0d0db16bccc37
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Tue, 9 Jan 2018 14:43:07 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 9 Jan 2018 16:17:51 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk

---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 +++
 arch/x86/include/asm/asm-prototypes.h |  25 ++++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 112 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++++
 8 files changed, 215 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..918e550 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning Retpoline not supported in compiler. System may be insecure.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..da407df
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	.align	16
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+	.align	16
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+	.align	16
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call	*%[thunk_target]\n",				\
+	""							\
+	"       jmp    do_call%=;\n"				\
+	"       .align 16\n"					\
+	"do_retpoline%=:\n"					\
+	"	call   do_rop%=;\n"				\
+	"spec_trap%=:\n"					\
+	"	pause;\n"					\
+	"       jmp    spec_trap%=;\n"				\
+	"       .align 16\n"					\
+	"do_rop%=:\n"						\
+	"	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"do_call%=:\n"						\
+	"	call   do_retpoline%=;\n",			\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  533ae606cfdbcded9878291761eb4dc0ab19cdb3
Gitweb:     https://git.kernel.org/tip/533ae606cfdbcded9878291761eb4dc0ab19cdb3
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Tue, 9 Jan 2018 14:43:07 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 10 Jan 2018 18:28:21 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk


---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 ++++
 arch/x86/include/asm/asm-prototypes.h |  25 ++++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..7f58713
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call	*%[thunk_target]\n",				\
+	""							\
+	"       jmp    do_call%=;\n"				\
+	"       .align 16\n"					\
+	"do_retpoline%=:\n"					\
+	"	call   do_rop%=;\n"				\
+	"spec_trap%=:\n"					\
+	"	pause;\n"					\
+	"       jmp    spec_trap%=;\n"				\
+	"       .align 16\n"					\
+	"do_rop%=:\n"						\
+	"	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"do_call%=:\n"						\
+	"	call   do_retpoline%=;\n",			\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  3c0c41a7bc03932d02d7796b4735f9077eb2b320
Gitweb:     https://git.kernel.org/tip/3c0c41a7bc03932d02d7796b4735f9077eb2b320
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Tue, 9 Jan 2018 14:43:07 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 10 Jan 2018 19:09:09 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk


---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 ++++
 arch/x86/include/asm/asm-prototypes.h |  25 ++++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..7f58713
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call	*%[thunk_target]\n",				\
+	""							\
+	"       jmp    do_call%=;\n"				\
+	"       .align 16\n"					\
+	"do_retpoline%=:\n"					\
+	"	call   do_rop%=;\n"				\
+	"spec_trap%=:\n"					\
+	"	pause;\n"					\
+	"       jmp    spec_trap%=;\n"				\
+	"       .align 16\n"					\
+	"do_rop%=:\n"						\
+	"	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"do_call%=:\n"						\
+	"	call   do_retpoline%=;\n",			\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif

[tip:x86/pti] x86/retpoline: Add initial retpoline support

[tip-bot for David Woodhouse]

Commit-ID:  d46717c610dcfa2cba5c87500c928993371ef1ad
Gitweb:     https://git.kernel.org/tip/d46717c610dcfa2cba5c87500c928993371ef1ad
Author:     David Woodhouse <dwmw@amazon.co.uk>
AuthorDate: Tue, 9 Jan 2018 14:43:07 +0000
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 10 Jan 2018 19:36:22 +0100

x86/retpoline: Add initial retpoline support

Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Arjan van de Ven <arjan@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515508997-6154-2-git-send-email-dwmw@amazon.co.uk


---
 arch/x86/Kconfig                      |  13 ++++
 arch/x86/Makefile                     |  10 ++++
 arch/x86/include/asm/asm-prototypes.h |  25 ++++++++
 arch/x86/include/asm/cpufeatures.h    |   2 +
 arch/x86/include/asm/nospec-branch.h  | 109 ++++++++++++++++++++++++++++++++++
 arch/x86/kernel/cpu/common.c          |   4 ++
 arch/x86/lib/Makefile                 |   1 +
 arch/x86/lib/retpoline.S              |  48 +++++++++++++++
 8 files changed, 212 insertions(+)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e23d21a..d181916 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -429,6 +429,19 @@ config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
 
+config RETPOLINE
+	bool "Avoid speculative indirect branches in kernel"
+	default y
+	help
+	  Compile kernel with the retpoline compiler options to guard against
+	  kernel-to-user data leaks by avoiding speculative indirect
+	  branches. Requires a compiler with -mindirect-branch=thunk-extern
+	  support for full protection. The kernel may run slower.
+
+	  Without compiler support, at least indirect branches in assembler
+	  code are eliminated. Since this includes the syscall entry path,
+	  it is not entirely pointless.
+
 config INTEL_RDT
 	bool "Intel Resource Director Technology support"
 	default n
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index a20eacd..974c618 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare
 #
 KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
+# Avoid indirect branches in kernel to deal with Spectre
+ifdef CONFIG_RETPOLINE
+    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
+    ifneq ($(RETPOLINE_CFLAGS),)
+        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+    else
+        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)
+    endif
+endif
+
 archscripts: scripts_basic
 	$(Q)$(MAKE) $(build)=arch/x86/tools relocs
 
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index ff700d8..0927cdc 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -11,7 +11,32 @@
 #include <asm/pgtable.h>
 #include <asm/special_insns.h>
 #include <asm/preempt.h>
+#include <asm/asm.h>
 
 #ifndef CONFIG_X86_CMPXCHG64
 extern void cmpxchg8b_emu(void);
 #endif
+
+#ifdef CONFIG_RETPOLINE
+#ifdef CONFIG_X86_32
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);
+#else
+#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);
+INDIRECT_THUNK(8)
+INDIRECT_THUNK(9)
+INDIRECT_THUNK(10)
+INDIRECT_THUNK(11)
+INDIRECT_THUNK(12)
+INDIRECT_THUNK(13)
+INDIRECT_THUNK(14)
+INDIRECT_THUNK(15)
+#endif
+INDIRECT_THUNK(ax)
+INDIRECT_THUNK(bx)
+INDIRECT_THUNK(cx)
+INDIRECT_THUNK(dx)
+INDIRECT_THUNK(si)
+INDIRECT_THUNK(di)
+INDIRECT_THUNK(bp)
+INDIRECT_THUNK(sp)
+#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1641c2f..f275447 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -203,6 +203,8 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
 #define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */
 #define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
new file mode 100644
index 0000000..7f58713
--- /dev/null
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -0,0 +1,109 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#include <asm/alternative.h>
+#include <asm/alternative-asm.h>
+#include <asm/cpufeatures.h>
+
+#ifdef __ASSEMBLY__
+
+/*
+ * These are the bare retpoline primitives for indirect jmp and call.
+ * Do not use these directly; they only exist to make the ALTERNATIVE
+ * invocation below less ugly.
+ */
+.macro RETPOLINE_JMP reg:req
+	call	.Ldo_rop_\@
+.Lspec_trap_\@:
+	pause
+	jmp	.Lspec_trap_\@
+.Ldo_rop_\@:
+	mov	\reg, (%_ASM_SP)
+	ret
+.endm
+
+/*
+ * This is a wrapper around RETPOLINE_JMP so the called function in reg
+ * returns to the instruction after the macro.
+ */
+.macro RETPOLINE_CALL reg:req
+	jmp	.Ldo_call_\@
+.Ldo_retpoline_jmp_\@:
+	RETPOLINE_JMP \reg
+.Ldo_call_\@:
+	call	.Ldo_retpoline_jmp_\@
+.endm
+
+/*
+ * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
+ * indirect jmp/call which may be susceptible to the Spectre variant 2
+ * attack.
+ */
+.macro JMP_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
+		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	jmp	*\reg
+#endif
+.endm
+
+.macro CALL_NOSPEC reg:req
+#ifdef CONFIG_RETPOLINE
+	ALTERNATIVE_2 __stringify(call *\reg),				\
+		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
+		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+#else
+	call	*\reg
+#endif
+.endm
+
+#else /* __ASSEMBLY__ */
+
+#if defined(CONFIG_X86_64) && defined(RETPOLINE)
+/*
+ * Since the inline asm uses the %V modifier which is only in newer GCC,
+ * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call *%[thunk_target]\n",				\
+	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
+	X86_FEATURE_RETPOLINE)
+# define THUNK_TARGET(addr) [thunk_target] "r" (addr)
+#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
+/*
+ * For i386 we use the original ret-equivalent retpoline, because
+ * otherwise we'll run out of registers. We don't care about CET
+ * here, anyway.
+ */
+# define CALL_NOSPEC ALTERNATIVE(				\
+	"call	*%[thunk_target]\n",				\
+	""							\
+	"       jmp    do_call%=;\n"				\
+	"       .align 16\n"					\
+	"do_retpoline%=:\n"					\
+	"	call   do_rop%=;\n"				\
+	"spec_trap%=:\n"					\
+	"	pause;\n"					\
+	"       jmp    spec_trap%=;\n"				\
+	"       .align 16\n"					\
+	"do_rop%=:\n"						\
+	"	addl   $4, %%esp;\n"				\
+	"       pushl  %[thunk_target];\n"			\
+	"       ret;\n"						\
+	"       .align 16\n"					\
+	"do_call%=:\n"						\
+	"	call   do_retpoline%=;\n",			\
+	X86_FEATURE_RETPOLINE)
+
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#else /* No retpoline */
+# define CALL_NOSPEC "call *%[thunk_target]\n"
+# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __NOSPEC_BRANCH_H__ */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 372ba3f..7a671d1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -905,6 +905,10 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
 	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
 
+#ifdef CONFIG_RETPOLINE
+	setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+#endif
+
 	fpu__init_system(c);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 457f681..d435c89 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
 
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
new file mode 100644
index 0000000..cb45c6c
--- /dev/null
+++ b/arch/x86/lib/retpoline.S
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/stringify.h>
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/cpufeatures.h>
+#include <asm/alternative-asm.h>
+#include <asm/export.h>
+#include <asm/nospec-branch.h>
+
+.macro THUNK reg
+	.section .text.__x86.indirect_thunk.\reg
+
+ENTRY(__x86_indirect_thunk_\reg)
+	CFI_STARTPROC
+	JMP_NOSPEC %\reg
+	CFI_ENDPROC
+ENDPROC(__x86_indirect_thunk_\reg)
+.endm
+
+/*
+ * Despite being an assembler file we can't just use .irp here
+ * because __KSYM_DEPS__ only uses the C preprocessor and would
+ * only see one instance of "__x86_indirect_thunk_\reg" rather
+ * than one per register with the correct names. So we do it
+ * the simple and nasty way...
+ */
+#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg)
+#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg)
+
+GENERATE_THUNK(_ASM_AX)
+GENERATE_THUNK(_ASM_BX)
+GENERATE_THUNK(_ASM_CX)
+GENERATE_THUNK(_ASM_DX)
+GENERATE_THUNK(_ASM_SI)
+GENERATE_THUNK(_ASM_DI)
+GENERATE_THUNK(_ASM_BP)
+GENERATE_THUNK(_ASM_SP)
+#ifdef CONFIG_64BIT
+GENERATE_THUNK(r8)
+GENERATE_THUNK(r9)
+GENERATE_THUNK(r10)
+GENERATE_THUNK(r11)
+GENERATE_THUNK(r12)
+GENERATE_THUNK(r13)
+GENERATE_THUNK(r14)
+GENERATE_THUNK(r15)
+#endif