From: "tip-bot2 for Peter Zijlstra" <tip-bot2@linutronix.de>
To: linux-tip-commits@vger.kernel.org
Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org>,
Borislav Petkov <bp@suse.de>, Miroslav Benes <mbenes@suse.cz>,
x86@kernel.org, linux-kernel@vger.kernel.org
Subject: [tip: x86/core] objtool/x86: Rewrite retpoline thunk calls
Date: Thu, 01 Apr 2021 15:08:53 -0000 [thread overview]
Message-ID: <161728973397.29796.9283971977578914711.tip-bot2@tip-bot2> (raw)
In-Reply-To: <20210326151300.320177914@infradead.org>
The following commit has been merged into the x86/core branch of tip:
Commit-ID: f31390437ce984118215169d75570e365457ec23
Gitweb: https://git.kernel.org/tip/f31390437ce984118215169d75570e365457ec23
Author: Peter Zijlstra <peterz@infradead.org>
AuthorDate: Fri, 26 Mar 2021 16:12:15 +01:00
Committer: Borislav Petkov <bp@suse.de>
CommitterDate: Thu, 01 Apr 2021 14:30:45 +02:00
objtool/x86: Rewrite retpoline thunk calls
When the compiler emits: "CALL __x86_indirect_thunk_\reg" for an
indirect call, have objtool rewrite it to:
ALTERNATIVE "call __x86_indirect_thunk_\reg",
"call *%reg", ALT_NOT(X86_FEATURE_RETPOLINE)
Additionally, in order to not emit endless identical
.altinst_replacement chunks, use a global symbol for them, see
__x86_indirect_alt_*.
This also avoids objtool from having to do code generation.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Link: https://lkml.kernel.org/r/20210326151300.320177914@infradead.org
---
arch/x86/include/asm/asm-prototypes.h | 12 ++-
arch/x86/lib/retpoline.S | 41 ++++++++-
tools/objtool/arch/x86/decode.c | 117 +++++++++++++++++++++++++-
3 files changed, 167 insertions(+), 3 deletions(-)
diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h
index 0545b07..4cb726c 100644
--- a/arch/x86/include/asm/asm-prototypes.h
+++ b/arch/x86/include/asm/asm-prototypes.h
@@ -19,11 +19,19 @@ extern void cmpxchg8b_emu(void);
#ifdef CONFIG_RETPOLINE
-#define DECL_INDIRECT_THUNK(reg) \
+#undef GEN
+#define GEN(reg) \
extern asmlinkage void __x86_indirect_thunk_ ## reg (void);
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) \
+ extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);
+#include <asm/GEN-for-each-reg.h>
#undef GEN
-#define GEN(reg) DECL_INDIRECT_THUNK(reg)
+#define GEN(reg) \
+ extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);
#include <asm/GEN-for-each-reg.h>
#endif /* CONFIG_RETPOLINE */
diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index d2c0d14..4d32cb0 100644
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -10,6 +10,8 @@
#include <asm/unwind_hints.h>
#include <asm/frame.h>
+ .section .text.__x86.indirect_thunk
+
.macro RETPOLINE reg
ANNOTATE_INTRA_FUNCTION_CALL
call .Ldo_rop_\@
@@ -25,9 +27,9 @@
.endm
.macro THUNK reg
- .section .text.__x86.indirect_thunk
.align 32
+
SYM_FUNC_START(__x86_indirect_thunk_\reg)
ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
@@ -39,6 +41,32 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg)
.endm
/*
+ * This generates .altinstr_replacement symbols for use by objtool. They,
+ * however, must not actually live in .altinstr_replacement since that will be
+ * discarded after init, but module alternatives will also reference these
+ * symbols.
+ *
+ * Their names matches the "__x86_indirect_" prefix to mark them as retpolines.
+ */
+.macro ALT_THUNK reg
+
+ .align 1
+
+SYM_FUNC_START_NOALIGN(__x86_indirect_alt_call_\reg)
+ ANNOTATE_RETPOLINE_SAFE
+1: call *%\reg
+2: .skip 5-(2b-1b), 0x90
+SYM_FUNC_END(__x86_indirect_alt_call_\reg)
+
+SYM_FUNC_START_NOALIGN(__x86_indirect_alt_jmp_\reg)
+ ANNOTATE_RETPOLINE_SAFE
+1: jmp *%\reg
+2: .skip 5-(2b-1b), 0x90
+SYM_FUNC_END(__x86_indirect_alt_jmp_\reg)
+
+.endm
+
+/*
* Despite being an assembler file we can't just use .irp here
* because __KSYM_DEPS__ only uses the C preprocessor and would
* only see one instance of "__x86_indirect_thunk_\reg" rather
@@ -61,3 +89,14 @@ SYM_FUNC_END(__x86_indirect_thunk_\reg)
#define GEN(reg) EXPORT_THUNK(reg)
#include <asm/GEN-for-each-reg.h>
+#undef GEN
+#define GEN(reg) ALT_THUNK reg
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_call_ ## reg)
+#include <asm/GEN-for-each-reg.h>
+
+#undef GEN
+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_alt_jmp_ ## reg)
+#include <asm/GEN-for-each-reg.h>
diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
index e5fa3a5..44375fa 100644
--- a/tools/objtool/arch/x86/decode.c
+++ b/tools/objtool/arch/x86/decode.c
@@ -16,6 +16,7 @@
#include <objtool/elf.h>
#include <objtool/arch.h>
#include <objtool/warn.h>
+#include <arch/elf.h>
static unsigned char op_to_cfi_reg[][2] = {
{CFI_AX, CFI_R8},
@@ -610,6 +611,122 @@ const char *arch_nop_insn(int len)
return nops[len-1];
}
+/* asm/alternative.h ? */
+
+#define ALTINSTR_FLAG_INV (1 << 15)
+#define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV)
+
+struct alt_instr {
+ s32 instr_offset; /* original instruction */
+ s32 repl_offset; /* offset to replacement instruction */
+ u16 cpuid; /* cpuid bit set for replacement */
+ u8 instrlen; /* length of original instruction */
+ u8 replacementlen; /* length of new instruction */
+} __packed;
+
+static int elf_add_alternative(struct elf *elf,
+ struct instruction *orig, struct symbol *sym,
+ int cpuid, u8 orig_len, u8 repl_len)
+{
+ const int size = sizeof(struct alt_instr);
+ struct alt_instr *alt;
+ struct section *sec;
+ Elf_Scn *s;
+
+ sec = find_section_by_name(elf, ".altinstructions");
+ if (!sec) {
+ sec = elf_create_section(elf, ".altinstructions",
+ SHF_WRITE, size, 0);
+
+ if (!sec) {
+ WARN_ELF("elf_create_section");
+ return -1;
+ }
+ }
+
+ s = elf_getscn(elf->elf, sec->idx);
+ if (!s) {
+ WARN_ELF("elf_getscn");
+ return -1;
+ }
+
+ sec->data = elf_newdata(s);
+ if (!sec->data) {
+ WARN_ELF("elf_newdata");
+ return -1;
+ }
+
+ sec->data->d_size = size;
+ sec->data->d_align = 1;
+
+ alt = sec->data->d_buf = malloc(size);
+ if (!sec->data->d_buf) {
+ perror("malloc");
+ return -1;
+ }
+ memset(sec->data->d_buf, 0, size);
+
+ if (elf_add_reloc_to_insn(elf, sec, sec->sh.sh_size,
+ R_X86_64_PC32, orig->sec, orig->offset)) {
+ WARN("elf_create_reloc: alt_instr::instr_offset");
+ return -1;
+ }
+
+ if (elf_add_reloc(elf, sec, sec->sh.sh_size + 4,
+ R_X86_64_PC32, sym, 0)) {
+ WARN("elf_create_reloc: alt_instr::repl_offset");
+ return -1;
+ }
+
+ alt->cpuid = cpuid;
+ alt->instrlen = orig_len;
+ alt->replacementlen = repl_len;
+
+ sec->sh.sh_size += size;
+ sec->changed = true;
+
+ return 0;
+}
+
+#define X86_FEATURE_RETPOLINE ( 7*32+12)
+
+int arch_rewrite_retpolines(struct objtool_file *file)
+{
+ struct instruction *insn;
+ struct reloc *reloc;
+ struct symbol *sym;
+ char name[32] = "";
+
+ list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
+
+ if (!strcmp(insn->sec->name, ".text.__x86.indirect_thunk"))
+ continue;
+
+ reloc = insn->reloc;
+
+ sprintf(name, "__x86_indirect_alt_%s_%s",
+ insn->type == INSN_JUMP_DYNAMIC ? "jmp" : "call",
+ reloc->sym->name + 21);
+
+ sym = find_symbol_by_name(file->elf, name);
+ if (!sym) {
+ sym = elf_create_undef_symbol(file->elf, name);
+ if (!sym) {
+ WARN("elf_create_undef_symbol");
+ return -1;
+ }
+ }
+
+ if (elf_add_alternative(file->elf, insn, sym,
+ ALT_NOT(X86_FEATURE_RETPOLINE), 5, 5)) {
+ WARN("elf_add_alternative");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
int arch_decode_hint_reg(struct instruction *insn, u8 sp_reg)
{
struct cfi_reg *cfa = &insn->cfi.cfa;
next prev parent reply other threads:[~2021-04-01 17:57 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-26 15:11 [PATCH v3 00/16] x86,objtool: Optimize !RETPOLINE Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 01/16] x86: Add insn_decode_kernel() Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 02/16] x86/alternatives: Optimize optimize_nops() Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:11 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 03/16] x86/retpoline: Simplify retpolines Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-04-06 8:56 ` David Laight
2021-03-26 15:12 ` [PATCH v3 04/16] objtool: Correctly handle retpoline thunk calls Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 05/16] objtool: Per arch retpoline naming Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] objtool: Handle per " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 06/16] objtool: Fix static_call list generation Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 07/16] objtool: Rework rebuild_reloc logic Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` [tip: x86/core] objtool: Rework the elf_rebuild_reloc_section() logic tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 08/16] objtool: Add elf_create_reloc() helper Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 09/16] objtool: Implicitly create reloc sections Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` [tip: x86/core] objtool: Create reloc sections implicitly tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 10/16] objtool: Extract elf_strtab_concat() Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 11/16] objtool: Extract elf_symbol_add() Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 12/16] objtool: Add elf_create_undef_symbol() Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 13/16] objtool: Keep track of retpoline call sites Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 14/16] objtool: Cache instruction relocs Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 15/16] objtool: Skip magical retpoline .altinstr_replacement Peter Zijlstra
2021-04-01 15:08 ` [tip: x86/core] " tip-bot2 for Peter Zijlstra
2021-04-03 11:10 ` tip-bot2 for Peter Zijlstra
2021-03-26 15:12 ` [PATCH v3 16/16] objtool,x86: Rewrite retpoline thunk calls Peter Zijlstra
2021-03-29 16:38 ` Josh Poimboeuf
2021-06-02 15:51 ` Lukasz Majczak
2021-06-02 16:56 ` Peter Zijlstra
2021-06-02 17:10 ` Peter Zijlstra
2021-06-02 20:43 ` Josh Poimboeuf
2021-06-04 20:50 ` Nick Desaulniers
2021-06-04 23:27 ` Nick Desaulniers
2021-06-04 23:50 ` Fangrui Song
2021-06-05 10:38 ` Peter Zijlstra
2021-06-06 1:58 ` Fāng-ruì Sòng
2021-06-07 7:56 ` Peter Zijlstra
2021-06-07 9:22 ` Peter Zijlstra
2021-06-07 9:45 ` Peter Zijlstra
2021-06-07 17:23 ` Fāng-ruì Sòng
2021-06-07 18:25 ` Peter Zijlstra
2021-06-07 20:54 ` Nick Desaulniers
2021-06-08 9:56 ` Peter Zijlstra
2021-06-08 16:58 ` Nathan Chancellor
2021-06-08 17:22 ` Peter Zijlstra
2021-06-08 17:29 ` Nathan Chancellor
2021-06-08 18:17 ` Peter Zijlstra
2021-06-08 18:49 ` Nathan Chancellor
2021-06-09 7:11 ` Lukasz Majczak
2021-06-09 7:20 ` Peter Zijlstra
2021-06-09 12:23 ` Lukasz Majczak
2021-06-09 15:08 ` Peter Zijlstra
2021-06-09 15:11 ` Peter Zijlstra
2021-06-09 15:56 ` Nathan Chancellor
2021-06-08 18:18 ` Nick Desaulniers
2021-06-07 18:19 ` Peter Zijlstra
2021-06-07 18:27 ` Fāng-ruì Sòng
2021-06-07 18:47 ` Peter Zijlstra
2021-04-01 15:08 ` tip-bot2 for Peter Zijlstra [this message]
2021-04-03 11:10 ` [tip: x86/core] objtool/x86: " tip-bot2 for Peter Zijlstra
2021-03-30 15:02 ` [PATCH v3 00/16] x86,objtool: Optimize !RETPOLINE Miroslav Benes
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=161728973397.29796.9283971977578914711.tip-bot2@tip-bot2 \
--to=tip-bot2@linutronix.de \
--cc=bp@suse.de \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=mbenes@suse.cz \
--cc=peterz@infradead.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).