From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org, peterz@infradead.org, linux-kernel@vger.kernel.org
Cc: Josh Poimboeuf <jpoimboe@redhat.com>,
Steven Rostedt <rostedt@goodmis.org>,
Masami Hiramatsu <mhiramat@kernel.org>,
Daniel Bristot de Oliveira <bristot@redhat.com>,
Jason Baron <jbaron@akamai.com>, Nadav Amit <namit@vmware.com>,
Andy Lutomirski <luto@kernel.org>,
Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>,
Vineet Gupta <Vineet.Gupta1@synopsys.com>
Subject: [RFC][PATCH 6/8] jump_label, x86: Add variable length patching support
Date: Fri, 28 Jun 2019 12:21:19 +0200 [thread overview]
Message-ID: <20190628103224.773901511@infradead.org> (raw)
In-Reply-To: 20190628102113.360432762@infradead.org
This allows the patching to to emit 2 byte JMP/NOP instruction in
addition to the 5 byte JMP/NOP we already did. This allows for more
compact code.
This code is not yet used, as we don't emit shorter code at compile
time yet.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/jump_label.h | 6 +-
arch/x86/include/asm/nops.h | 1
arch/x86/kernel/jump_label.c | 77 +++++++++++++++++++++++---------------
3 files changed, 53 insertions(+), 31 deletions(-)
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -7,9 +7,11 @@
#define JUMP_LABEL_NOP_SIZE 5
#ifdef CONFIG_X86_64
-# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
+# define STATIC_KEY_NOP2 P6_NOP2
+# define STATIC_KEY_NOP5 P6_NOP5_ATOMIC
#else
-# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
+# define STATIC_KEY_NOP2 GENERIC_NOP2
+# define STATIC_KEY_NOP5 GENERIC_NOP5_ATOMIC
#endif
#include <asm/asm.h>
--- a/arch/x86/include/asm/nops.h
+++ b/arch/x86/include/asm/nops.h
@@ -5,6 +5,7 @@
/*
* Define nops for use with alternative() and for tracing.
*
+ * *_NOP2 must be a single instruction
* *_NOP5_ATOMIC must be a single instruction.
*/
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -17,49 +17,67 @@
#include <asm/text-patching.h>
union jump_code_union {
- char code[JUMP_LABEL_NOP_SIZE];
+ char code[JMP32_INSN_SIZE];
struct {
- char jump;
- int offset;
+ char opcode;
+ union {
+ s8 d8;
+ s32 d32;
+ };
} __attribute__((packed));
};
-static void __jump_label_set_jump_code(struct jump_entry *entry,
- enum jump_label_type type,
- union jump_code_union *code,
- int init)
+static inline bool __jump_disp_is_byte(s32 disp)
{
- const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
- const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
- unsigned char *ip = (void *)jump_entry_code(entry);
+ return false;
+}
+
+static int __jump_label_set_jump_code(struct jump_entry *entry,
+ enum jump_label_type type,
+ union jump_code_union *code,
+ int init)
+{
+ static unsigned char default_nop2[] = { STATIC_KEY_NOP2 };
+ static unsigned char default_nop5[] = { STATIC_KEY_NOP5 };
+ s32 disp = jump_entry_target(entry) - jump_entry_code(entry);
+ void *ip = (void *)jump_entry_code(entry);
+ const unsigned char *nop;
const void *expect;
- int line;
+ int line, size;
- code->jump = 0xe9;
- code->offset = jump_entry_target(entry) -
- (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
-
- if (init) {
- expect = default_nop; line = __LINE__;
- } else if (type == JUMP_LABEL_JMP) {
- expect = ideal_nop; line = __LINE__;
+ if (__jump_disp_is_byte(disp)) {
+ size = JMP8_INSN_SIZE;
+ code->opcode = JMP8_INSN_OPCODE;
+ code->d8 = disp - size;
+ nop = init ? default_nop2 : ideal_nops[2];
+ } else {
+ size = JMP32_INSN_SIZE;
+ code->opcode = JMP32_INSN_OPCODE;
+ code->d32 = disp - size;
+ nop = init ? default_nop5 : ideal_nops[NOP_ATOMIC5];
+ }
+
+ if (init || type == JUMP_LABEL_JMP) {
+ expect = nop; line = __LINE__;
} else {
expect = code->code; line = __LINE__;
}
- if (memcmp(ip, expect, JUMP_LABEL_NOP_SIZE)) {
+ if (memcmp(ip, expect, size)) {
/*
* The location is not an op that we were expecting.
* Something went wrong. Crash the box, as something could be
* corrupting the kernel.
*/
- pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) line:%d init:%d type:%d\n",
- ip, ip, ip, expect, line, init, type);
+ pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) line:%d init:%d size:%d type:%d\n",
+ ip, ip, ip, expect, line, init, size, type);
BUG();
}
if (type == JUMP_LABEL_NOP)
- memcpy(code, ideal_nop, JUMP_LABEL_NOP_SIZE);
+ memcpy(code, nop, size);
+
+ return size;
}
static void __ref __jump_label_transform(struct jump_entry *entry,
@@ -67,8 +85,9 @@ static void __ref __jump_label_transform
int init)
{
union jump_code_union code;
+ int size;
- __jump_label_set_jump_code(entry, type, &code, init);
+ size = __jump_label_set_jump_code(entry, type, &code, init);
/*
* As long as only a single processor is running and the code is still
@@ -82,12 +101,11 @@ static void __ref __jump_label_transform
* always nop being the 'currently valid' instruction
*/
if (init || system_state == SYSTEM_BOOTING) {
- text_poke_early((void *)jump_entry_code(entry), &code,
- JUMP_LABEL_NOP_SIZE);
+ text_poke_early((void *)jump_entry_code(entry), &code, size);
return;
}
- text_poke_bp((void *)jump_entry_code(entry), &code, JUMP_LABEL_NOP_SIZE, NULL);
+ text_poke_bp((void *)jump_entry_code(entry), &code, size, NULL);
}
void arch_jump_label_transform(struct jump_entry *entry,
@@ -107,6 +125,7 @@ bool arch_jump_label_transform_queue(str
{
struct text_poke_loc *tp;
void *entry_code;
+ int size;
if (system_state == SYSTEM_BOOTING) {
/*
@@ -143,10 +162,10 @@ bool arch_jump_label_transform_queue(str
return false;
}
- __jump_label_set_jump_code(entry, type,
+ size = __jump_label_set_jump_code(entry, type,
(union jump_code_union *)&tp->text, 0);
- text_poke_loc_init(tp, entry_code, NULL, JUMP_LABEL_NOP_SIZE, NULL);
+ text_poke_loc_init(tp, entry_code, NULL, size, NULL);
tp_vec_nr++;
next prev parent reply other threads:[~2019-06-28 13:36 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-28 10:21 [RFC][PATCH 0/8] jump_label, x86: Support variable sized JMP instructions Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 1/8] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 2/8] jump_label, x86: Strip ASM jump_label support Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 3/8] jump_label, x86: Factor out the __jump_table generation Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 4/8] jump_label, x86: Remove init NOP optimization Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 5/8] jump_label, x86: Improve error when we fail expected text Peter Zijlstra
2019-06-28 10:21 ` Peter Zijlstra [this message]
2019-06-28 10:21 ` [RFC][PATCH 7/8] jump_label, x86: Introduce jump_entry_size() Peter Zijlstra
2019-06-28 10:21 ` [RFC][PATCH 8/8] jump_label, x86: Enable JMP8/NOP2 support Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190628103224.773901511@infradead.org \
--to=peterz@infradead.org \
--cc=Eugeniy.Paltsev@synopsys.com \
--cc=Vineet.Gupta1@synopsys.com \
--cc=bristot@redhat.com \
--cc=jbaron@akamai.com \
--cc=jpoimboe@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=mhiramat@kernel.org \
--cc=namit@vmware.com \
--cc=rostedt@goodmis.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.