From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org
Cc: linux-kernel@vger.kernel.org,
Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Andy Lutomirski <luto@kernel.org>,
Steven Rostedt <rostedt@goodmis.org>,
Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
Linus Torvalds <torvalds@linux-foundation.org>,
Masami Hiramatsu <mhiramat@kernel.org>,
Jason Baron <jbaron@akamai.com>, Jiri Kosina <jkosina@suse.cz>,
David Laight <David.Laight@ACULAB.COM>,
Borislav Petkov <bp@alien8.de>, Julia Cartwright <julia@ni.com>,
Jessica Yu <jeyu@kernel.org>, "H. Peter Anvin" <hpa@zytor.com>,
Nadav Amit <namit@vmware.com>,
Rasmus Villemoes <linux@rasmusvillemoes.dk>,
Edward Cree <ecree@solarflare.com>,
Daniel Bristot de Oliveira <bristot@redhat.com>
Subject: [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions
Date: Wed, 05 Jun 2019 15:08:01 +0200 [thread overview]
Message-ID: <20190605131945.005681046@infradead.org> (raw)
In-Reply-To: 20190605130753.327195108@infradead.org
In preparation for static_call support, teach text_poke_bp() to
emulate instructions, including CALL.
The current text_poke_bp() takes a @handler argument which is used as
a jump target when the temporary INT3 is hit by a different CPU.
When patching CALL instructions, this doesn't work because we'd miss
the PUSH of the return address. Instead, teach poke_int3_handler() to
emulate an instruction, typically the instruction we're patching in.
This fits almost all text_poke_bp() users, except
arch_unoptimize_kprobe() which restores random text, and for that site
we have to build an explicit emulate instruction.
Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
Cc: Nadav Amit <namit@vmware.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/include/asm/text-patching.h | 2 -
arch/x86/kernel/alternative.c | 47 ++++++++++++++++++++++++++---------
arch/x86/kernel/jump_label.c | 3 --
arch/x86/kernel/kprobes/opt.c | 11 +++++---
4 files changed, 46 insertions(+), 17 deletions(-)
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -37,7 +37,7 @@ extern void text_poke_early(void *addr,
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_kgdb(void *addr, const void *opcode, size_t len);
extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler);
+extern void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate);
extern int after_bootmem;
extern __ro_after_init struct mm_struct *poking_mm;
extern __ro_after_init unsigned long poking_addr;
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -921,19 +921,25 @@ static void do_sync_core(void *info)
}
static bool bp_patching_in_progress;
-static void *bp_int3_handler, *bp_int3_addr;
+static const void *bp_int3_opcode, *bp_int3_addr;
int poke_int3_handler(struct pt_regs *regs)
{
+ long ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+ struct opcode {
+ u8 insn;
+ s32 rel;
+ } __packed opcode;
+
/*
* Having observed our INT3 instruction, we now must observe
* bp_patching_in_progress.
*
- * in_progress = TRUE INT3
- * WMB RMB
- * write INT3 if (in_progress)
+ * in_progress = TRUE INT3
+ * WMB RMB
+ * write INT3 if (in_progress)
*
- * Idem for bp_int3_handler.
+ * Idem for bp_int3_opcode.
*/
smp_rmb();
@@ -943,8 +949,21 @@ int poke_int3_handler(struct pt_regs *re
if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
return 0;
- /* set up the specified breakpoint handler */
- regs->ip = (unsigned long) bp_int3_handler;
+ opcode = *(struct opcode *)bp_int3_opcode;
+
+ switch (opcode.insn) {
+ case 0xE8: /* CALL */
+ int3_emulate_call(regs, ip + opcode.rel);
+ break;
+
+ case 0xE9: /* JMP */
+ int3_emulate_jmp(regs, ip + opcode.rel);
+ break;
+
+ default: /* assume NOP */
+ int3_emulate_jmp(regs, ip);
+ break;
+ }
return 1;
}
@@ -955,7 +974,7 @@ NOKPROBE_SYMBOL(poke_int3_handler);
* @addr: address to patch
* @opcode: opcode of new instruction
* @len: length to copy
- * @handler: address to jump to when the temporary breakpoint is hit
+ * @emulate: opcode to emulate, when NULL use @opcode
*
* Modify multi-byte instruction by using int3 breakpoint on SMP.
* We completely avoid stop_machine() here, and achieve the
@@ -970,19 +989,25 @@ NOKPROBE_SYMBOL(poke_int3_handler);
* replacing opcode
* - sync cores
*/
-void text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
+void text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
{
unsigned char int3 = 0xcc;
- bp_int3_handler = handler;
+ bp_int3_opcode = emulate ?: opcode;
bp_int3_addr = (u8 *)addr + sizeof(int3);
bp_patching_in_progress = true;
lockdep_assert_held(&text_mutex);
/*
+ * poke_int3_handler() relies on @opcode being a 5 byte instruction;
+ * notably a JMP, CALL or NOP5_ATOMIC.
+ */
+ BUG_ON(len != 5);
+
+ /*
* Corresponding read barrier in int3 notifier for making sure the
- * in_progress and handler are correctly ordered wrt. patching.
+ * in_progress and opcode are correctly ordered wrt. patching.
*/
smp_wmb();
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -87,8 +87,7 @@ static void __ref __jump_label_transform
return;
}
- text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
- (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
+ text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, NULL);
}
void arch_jump_label_transform(struct jump_entry *entry,
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -437,8 +437,7 @@ void arch_optimize_kprobes(struct list_h
insn_buff[0] = RELATIVEJUMP_OPCODE;
*(s32 *)(&insn_buff[1]) = rel;
- text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE, NULL);
list_del_init(&op->list);
}
@@ -448,12 +447,18 @@ void arch_optimize_kprobes(struct list_h
void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
u8 insn_buff[RELATIVEJUMP_SIZE];
+ u8 emulate_buff[RELATIVEJUMP_SIZE];
/* Set int3 to first byte for kprobes */
insn_buff[0] = BREAKPOINT_INSTRUCTION;
memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+
+ emulate_buff[0] = RELATIVEJUMP_OPCODE;
+ *(s32 *)(&emulate_buff[1]) = (s32)((long)op->optinsn.insn -
+ ((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
- op->optinsn.insn);
+ emulate_buff);
}
/*
next prev parent reply other threads:[~2019-06-05 13:23 UTC|newest]
Thread overview: 87+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-05 13:07 [PATCH 00/15] x86 cleanups and static_call() Peter Zijlstra
2019-06-05 13:07 ` [PATCH 01/15] x86/entry/32: Clean up return from interrupt preemption path Peter Zijlstra
2019-06-07 14:21 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 02/15] x86: Move ENCODE_FRAME_POINTER to asm/frame.h Peter Zijlstra
2019-06-07 14:24 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 03/15] x86/kprobes: Fix frame pointer annotations Peter Zijlstra
2019-06-07 13:02 ` Masami Hiramatsu
2019-06-07 13:36 ` Josh Poimboeuf
2019-06-07 15:21 ` Masami Hiramatsu
2019-06-11 8:12 ` Peter Zijlstra
2019-06-05 13:07 ` [PATCH 04/15] x86/ftrace: Add pt_regs frame annotations Peter Zijlstra
2019-06-07 14:45 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 05/15] x86_32: Provide consistent pt_regs Peter Zijlstra
2019-06-07 13:13 ` Masami Hiramatsu
2019-06-07 19:32 ` Josh Poimboeuf
2019-06-11 8:14 ` Peter Zijlstra
2019-06-05 13:07 ` [PATCH 06/15] x86_32: Allow int3_emulate_push() Peter Zijlstra
2019-06-05 13:08 ` [PATCH 07/15] x86: Add int3_emulate_call() selftest Peter Zijlstra
2019-06-10 16:52 ` Josh Poimboeuf
2019-06-10 16:57 ` Andy Lutomirski
2019-06-11 8:17 ` Peter Zijlstra
2019-06-05 13:08 ` Peter Zijlstra [this message]
2019-06-07 5:41 ` [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions Nadav Amit
2019-06-07 8:20 ` Peter Zijlstra
2019-06-07 14:27 ` Masami Hiramatsu
2019-06-07 15:47 ` Masami Hiramatsu
2019-06-07 17:34 ` Peter Zijlstra
2019-06-07 17:48 ` Linus Torvalds
2019-06-11 10:44 ` Peter Zijlstra
2019-06-07 18:10 ` Andy Lutomirski
2019-06-07 20:22 ` hpa
2019-06-11 8:03 ` Peter Zijlstra
2019-06-11 12:08 ` Peter Zijlstra
2019-06-11 12:34 ` Peter Zijlstra
2019-06-11 12:42 ` Peter Zijlstra
2019-06-11 15:22 ` Steven Rostedt
2019-06-11 15:52 ` Steven Rostedt
2019-06-11 15:55 ` Peter Zijlstra
2019-06-12 19:44 ` Nadav Amit
2019-06-17 14:42 ` Peter Zijlstra
2019-06-17 17:06 ` Nadav Amit
2019-06-17 17:25 ` Andy Lutomirski
2019-06-17 19:26 ` Peter Zijlstra
2019-06-11 15:54 ` Andy Lutomirski
2019-06-11 16:11 ` Steven Rostedt
2019-06-17 14:31 ` Peter Zijlstra
2019-06-12 17:09 ` Peter Zijlstra
2019-06-10 16:57 ` Josh Poimboeuf
2019-06-11 15:14 ` Steven Rostedt
2019-06-11 15:52 ` Peter Zijlstra
2019-06-11 16:21 ` Peter Zijlstra
2019-06-12 14:44 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 09/15] compiler.h: Make __ADDRESSABLE() symbol truly unique Peter Zijlstra
2019-06-05 13:08 ` [PATCH 10/15] static_call: Add basic static call infrastructure Peter Zijlstra
2019-06-06 22:44 ` Nadav Amit
2019-06-07 8:28 ` Peter Zijlstra
2019-06-07 8:49 ` Ard Biesheuvel
2019-06-07 16:33 ` Andy Lutomirski
2019-06-07 16:58 ` Nadav Amit
2019-10-02 13:54 ` Peter Zijlstra
2019-10-02 20:48 ` Josh Poimboeuf
2019-06-05 13:08 ` [PATCH 11/15] static_call: Add inline " Peter Zijlstra
2019-06-06 22:24 ` Nadav Amit
2019-06-07 8:37 ` Peter Zijlstra
2019-06-07 16:35 ` Nadav Amit
2019-06-07 17:41 ` Peter Zijlstra
2019-06-10 17:19 ` Josh Poimboeuf
2019-06-10 18:33 ` Nadav Amit
2019-06-10 18:42 ` Josh Poimboeuf
2019-10-01 12:00 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 12/15] x86/static_call: Add out-of-line static call implementation Peter Zijlstra
2019-06-07 6:13 ` Nadav Amit
2019-06-07 7:51 ` Steven Rostedt
2019-06-07 8:38 ` Peter Zijlstra
2019-06-07 8:52 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64 Peter Zijlstra
2019-06-07 5:50 ` Nadav Amit
2019-06-10 18:33 ` Josh Poimboeuf
2019-06-10 18:45 ` Nadav Amit
2019-06-10 18:55 ` Josh Poimboeuf
2019-06-10 19:20 ` Nadav Amit
2019-10-01 14:43 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 14/15] static_call: Simple self-test module Peter Zijlstra
2019-06-10 17:24 ` Josh Poimboeuf
2019-06-11 8:29 ` Peter Zijlstra
2019-06-11 13:02 ` Josh Poimboeuf
2019-06-05 13:08 ` [PATCH 15/15] tracepoints: Use static_call Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190605131945.005681046@infradead.org \
--to=peterz@infradead.org \
--cc=David.Laight@ACULAB.COM \
--cc=ard.biesheuvel@linaro.org \
--cc=bp@alien8.de \
--cc=bristot@redhat.com \
--cc=ecree@solarflare.com \
--cc=hpa@zytor.com \
--cc=jbaron@akamai.com \
--cc=jeyu@kernel.org \
--cc=jkosina@suse.cz \
--cc=julia@ni.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux@rasmusvillemoes.dk \
--cc=luto@kernel.org \
--cc=mhiramat@kernel.org \
--cc=mingo@kernel.org \
--cc=namit@vmware.com \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).