From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org, jpoimboe@redhat.com, jbaron@akamai.com,
rostedt@goodmis.org, ardb@kernel.org
Cc: linux-kernel@vger.kernel.org, peterz@infradead.org
Subject: [PATCH 07/13] jump_label, x86: Add variable length patching support
Date: Thu, 06 May 2021 21:33:59 +0200 [thread overview]
Message-ID: <20210506194157.846870383@infradead.org> (raw)
In-Reply-To: 20210506193352.719596001@infradead.org
This allows the patching to to emit 2 byte JMP/NOP instruction in
addition to the 5 byte JMP/NOP we already did. This allows for more
compact code.
This code is not yet used, as we don't emit shorter code at compile
time yet.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/kernel/jump_label.c | 53 ++++++++++++++++++++++++++++---------------
1 file changed, 35 insertions(+), 18 deletions(-)
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -23,44 +23,63 @@ int arch_jump_entry_size(struct jump_ent
return JMP32_INSN_SIZE;
}
-static const void *
-__jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type)
+struct jump_label_patch {
+ const void *code;
+ int size;
+};
+
+static struct jump_label_patch
+__jump_label_patch(struct jump_entry *entry, enum jump_label_type type)
{
- const void *expect, *code;
+ const void *expect, *code, *nop;
const void *addr, *dest;
+ int size;
addr = (void *)jump_entry_code(entry);
dest = (void *)jump_entry_target(entry);
- code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
+ size = arch_jump_entry_size(entry);
+ switch (size) {
+ case JMP8_INSN_SIZE:
+ code = text_gen_insn(JMP8_INSN_OPCODE, addr, dest);
+ nop = x86_nops[size];
+ break;
+
+ case JMP32_INSN_SIZE:
+ code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
+ nop = x86_nops[size];
+ break;
+
+ default: BUG();
+ }
if (type == JUMP_LABEL_JMP)
- expect = x86_nops[5];
+ expect = nop;
else
expect = code;
- if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE)) {
+ if (memcmp(addr, expect, size)) {
/*
* The location is not an op that we were expecting.
* Something went wrong. Crash the box, as something could be
* corrupting the kernel.
*/
- pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) type:%d\n",
- addr, addr, addr, expect, type);
+ pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) size:%d type:%d\n",
+ addr, addr, addr, expect, size, type);
BUG();
}
if (type == JUMP_LABEL_NOP)
- code = x86_nops[5];
+ code = nop;
- return code;
+ return (struct jump_label_patch){.code = code, .size = size};
}
static inline void __jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
int init)
{
- const void *opcode = __jump_label_set_jump_code(entry, type);
+ const struct jump_label_patch jlp = __jump_label_patch(entry, type);
/*
* As long as only a single processor is running and the code is still
@@ -74,12 +93,11 @@ static inline void __jump_label_transfor
* always nop being the 'currently valid' instruction
*/
if (init || system_state == SYSTEM_BOOTING) {
- text_poke_early((void *)jump_entry_code(entry), opcode,
- JUMP_LABEL_NOP_SIZE);
+ text_poke_early((void *)jump_entry_code(entry), jlp.code, jlp.size);
return;
}
- text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
+ text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
}
static void __ref jump_label_transform(struct jump_entry *entry,
@@ -100,7 +118,7 @@ void arch_jump_label_transform(struct ju
bool arch_jump_label_transform_queue(struct jump_entry *entry,
enum jump_label_type type)
{
- const void *opcode;
+ struct jump_label_patch jlp;
if (system_state == SYSTEM_BOOTING) {
/*
@@ -111,9 +129,8 @@ bool arch_jump_label_transform_queue(str
}
mutex_lock(&text_mutex);
- opcode = __jump_label_set_jump_code(entry, type);
- text_poke_queue((void *)jump_entry_code(entry),
- opcode, JUMP_LABEL_NOP_SIZE, NULL);
+ jlp = __jump_label_patch(entry, type);
+ text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
mutex_unlock(&text_mutex);
return true;
}
next prev parent reply other threads:[~2021-05-06 19:42 UTC|newest]
Thread overview: 48+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-06 19:33 [PATCH 00/13] jump_label: Yet another attempt at variable sized jump_labels Peter Zijlstra
2021-05-06 19:33 ` [PATCH 01/13] objtool: Rewrite hashtable sizing Peter Zijlstra
2021-05-12 10:41 ` Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-06-10 18:14 ` [PATCH 01/13] " Nathan Chancellor
2021-06-10 18:43 ` Peter Zijlstra
2021-06-10 18:54 ` Nathan Chancellor
2021-06-10 18:50 ` Sami Tolvanen
2021-06-10 19:33 ` Peter Zijlstra
2021-06-10 19:43 ` Sami Tolvanen
2021-06-10 20:59 ` Nathan Chancellor
2021-06-14 13:19 ` [tip: objtool/core] objtool: Improve reloc hash size guestimate tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` [PATCH 02/13] x86,objtool: Dont exclude arch/x86/realmode/ Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] x86, objtool: " tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` [PATCH 03/13] jump_label, x86: Strip ASM jump_label support Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` [PATCH 04/13] jump_label, x86: Factor out the __jump_table generation Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` [PATCH 05/13] jump_label, x86: Improve error when we fail expected text Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` [PATCH 06/13] jump_label, x86: Introduce jump_entry_size() Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:33 ` Peter Zijlstra [this message]
2021-05-12 13:19 ` [tip: objtool/core] jump_label, x86: Add variable length patching support tip-bot2 for Peter Zijlstra
2021-05-13 14:16 ` [PATCH 07.5/13] jump_label,x86: Remove unused JUMP_LABEL_NOP_SIZE Peter Zijlstra
2021-05-14 7:01 ` [tip: objtool/core] jump_label/x86: " tip-bot2 for Peter Zijlstra
2021-05-06 19:34 ` [PATCH 08/13] jump_label: Free jump_entry::key bit1 for build use Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:34 ` [PATCH 09/13] jump_label,x86: Emit short JMP Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] jump_label, x86: " tip-bot2 for Peter Zijlstra
2021-05-06 19:34 ` [PATCH 10/13] objtool: Decode jump_entry::key addend Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:34 ` [PATCH 11/13] objtool: Rewrite jump_label instructions Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-06 19:34 ` [PATCH 12/13] objtool: Provide stats for jump_labels Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] " tip-bot2 for Peter Zijlstra
2021-05-13 14:15 ` [PATCH 12.5/13] objtool: Reflow handle_jump_alt() Peter Zijlstra
2021-05-06 19:34 ` [PATCH 13/13] jump_label,x86: Allow short NOPs Peter Zijlstra
2021-05-06 19:49 ` Peter Zijlstra
2021-05-12 13:19 ` [tip: objtool/core] jump_label, x86: " tip-bot2 for Peter Zijlstra
2021-05-18 19:50 ` Peter Zijlstra
2021-05-18 20:24 ` Peter Zijlstra
2021-05-19 0:44 ` Josh Poimboeuf
2021-05-19 6:56 ` Peter Zijlstra
2021-06-29 20:00 ` Matthew Wilcox
2021-06-29 20:35 ` Matthew Wilcox
2021-06-30 7:07 ` Peter Zijlstra
2021-06-30 7:38 ` Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210506194157.846870383@infradead.org \
--to=peterz@infradead.org \
--cc=ardb@kernel.org \
--cc=jbaron@akamai.com \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).