From: Chen Guokai <chenguokai17@mails.ucas.ac.cn>
To: paul.walmsley@sifive.com, palmer@dabbelt.com,
aou@eecs.berkeley.edu, rostedt@goodmis.org, mingo@redhat.com,
sfr@canb.auug.org.au
Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org,
liaochang1@huawei.com,
Chen Guokai <chenguokai17@mails.ucas.ac.cn>
Subject: [PATCH 8/8] riscv/kprobe: Patch AUIPC/JALR pair to optimize kprobe
Date: Sun, 30 Oct 2022 17:01:41 +0800 [thread overview]
Message-ID: <20221030090141.2550837-9-chenguokai17@mails.ucas.ac.cn> (raw)
In-Reply-To: <20221030090141.2550837-1-chenguokai17@mails.ucas.ac.cn>
From: Liao Chang <liaochang1@huawei.com>
The patch optimizes 'EBREAK' with 'AUIPC/JALR', introduce new patching
function to modify multiple instructions.
Co-developed-by: Chen Guokai <chenguokai17@mails.ucas.ac.cn>
Signed-off-by: Chen Guokai <chenguokai17@mails.ucas.ac.cn>
Signed-off-by: Liao Chang <liaochang1@huawei.com>
---
arch/riscv/include/asm/patch.h | 1 +
arch/riscv/kernel/patch.c | 22 +++++++++---
arch/riscv/kernel/probes/opt.c | 63 ++++++++++++++++++++++++++++++++--
3 files changed, 80 insertions(+), 6 deletions(-)
diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h
index 9a7d7346001e..ee31539de65f 100644
--- a/arch/riscv/include/asm/patch.h
+++ b/arch/riscv/include/asm/patch.h
@@ -8,5 +8,6 @@
int patch_text_nosync(void *addr, const void *insns, size_t len);
int patch_text(void *addr, u32 insn);
+int patch_text_batch(void *addr, const void *insn, size_t size);
#endif /* _ASM_RISCV_PATCH_H */
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index 765004b60513..02c43bb1fad3 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -15,7 +15,8 @@
struct patch_insn {
void *addr;
- u32 insn;
+ const void *insn;
+ size_t size;
atomic_t cpu_count;
};
@@ -106,8 +107,7 @@ static int patch_text_cb(void *data)
if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
ret =
- patch_text_nosync(patch->addr, &patch->insn,
- GET_INSN_LENGTH(patch->insn));
+ patch_text_nosync(patch->addr, patch->insn, patch->size);
atomic_inc(&patch->cpu_count);
} else {
while (atomic_read(&patch->cpu_count) <= num_online_cpus())
@@ -123,7 +123,8 @@ int patch_text(void *addr, u32 insn)
{
struct patch_insn patch = {
.addr = addr,
- .insn = insn,
+ .insn = &insn,
+ .size = GET_INSN_LENGTH(insn),
.cpu_count = ATOMIC_INIT(0),
};
@@ -131,3 +132,17 @@ int patch_text(void *addr, u32 insn)
&patch, cpu_online_mask);
}
NOKPROBE_SYMBOL(patch_text);
+
+int patch_text_batch(void *addr, const void *insn, size_t size)
+{
+ struct patch_insn patch = {
+ .addr = addr,
+ .insn = insn,
+ .size = size,
+ .cpu_count = ATOMIC_INIT(0),
+ };
+
+ return stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask);
+}
+
+NOKPROBE_SYMBOL(patch_text_batch);
diff --git a/arch/riscv/kernel/probes/opt.c b/arch/riscv/kernel/probes/opt.c
index 77248ed7d4e8..947bc015da7e 100644
--- a/arch/riscv/kernel/probes/opt.c
+++ b/arch/riscv/kernel/probes/opt.c
@@ -448,11 +448,19 @@ static bool can_optimize(unsigned long paddr, struct optimized_kprobe *op)
int arch_prepared_optinsn(struct arch_optimized_insn *optinsn)
{
- return 0;
+ return optinsn->length;
}
int arch_check_optimized_kprobe(struct optimized_kprobe *op)
{
+ unsigned long i;
+ struct kprobe *p;
+
+ for (i = RVC_INSN_LEN; i < op->optinsn.length; i += RVC_INSN_LEN) {
+ p = get_kprobe(op->kp.addr + i);
+ if (p && !kprobe_disabled(p))
+ return -EEXIST;
+ }
return 0;
}
@@ -521,23 +529,74 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe *op,
void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
{
+ if (op->optinsn.insn) {
+ free_optinsn_slot(op->optinsn.insn, 1);
+ op->optinsn.insn = NULL;
+ op->optinsn.length = 0;
+ }
}
void arch_optimize_kprobes(struct list_head *oplist)
{
+ long offs;
+ kprobe_opcode_t insn[3];
+ struct optimized_kprobe *op, *tmp;
+
+ list_for_each_entry_safe(op, tmp, oplist, list) {
+ WARN_ON(kprobe_disabled(&op->kp));
+
+ /* Backup instructions which will be replaced by jump address */
+ memcpy(op->optinsn.copied_insn,
+ DETOUR_ADDR(op->kp.addr, GET_INSN_LENGTH(op->kp.opcode)),
+ op->optinsn.length - GET_INSN_LENGTH(op->kp.opcode));
+
+ /*
+ * After patch, it should be:
+ * auipc free_register, %hi(detour_buffer)
+ * jalr free_register, free_register, %lo(detour_buffer)
+ * where free_register will eventually save the return address
+ */
+ offs = (unsigned long)op->optinsn.insn -
+ (unsigned long)op->kp.addr;
+ insn[0] = rv_auipc(op->optinsn.rd, (offs + (1 << 11)) >> 12);
+ insn[1] = rv_jalr(op->optinsn.rd, op->optinsn.rd, offs & 0xFFF);
+ /* For 3 RVC + 1 RVI scenario, need C.NOP for padding */
+ if (op->optinsn.length > 2 * RVI_INSN_LEN)
+ insn[2] = rvc_addi(0, 0);
+
+ patch_text_batch(op->kp.addr, insn, op->optinsn.length);
+ if (memcmp(op->kp.addr, insn, op->optinsn.length))
+ continue;
+
+ list_del_init(&op->list);
+ }
}
void arch_unoptimize_kprobes(struct list_head *oplist,
struct list_head *done_list)
{
+ struct optimized_kprobe *op, *tmp;
+
+ list_for_each_entry_safe(op, tmp, oplist, list) {
+ arch_unoptimize_kprobe(op);
+ list_move(&op->list, done_list);
+ }
}
void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
+ kprobe_opcode_t buf[MAX_COPIED_INSN];
+ unsigned long offset = GET_INSN_LENGTH(op->kp.opcode);
+
+ buf[0] = (offset == RVI_INSN_LEN) ? __BUG_INSN_32 : __BUG_INSN_16;
+ memcpy(DETOUR_ADDR(buf, offset), op->optinsn.copied_insn,
+ op->optinsn.length - offset);
+ patch_text_batch(op->kp.addr, buf, op->optinsn.length);
}
int arch_within_optimized_kprobe(struct optimized_kprobe *op,
kprobe_opcode_t *addr)
{
- return 0;
+ return (op->kp.addr <= addr &&
+ op->kp.addr + op->optinsn.length > addr);
}
--
2.25.1
_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv
prev parent reply other threads:[~2022-10-30 9:02 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-30 9:01 [PATCH v3 0/8] Add OPTPROBES feature on RISCV Chen Guokai
2022-10-30 9:01 ` [PATCH 1/8] riscv/kprobe: Prepare the skeleton to implement RISCV OPTPROBES feature Chen Guokai
2022-10-31 19:42 ` Conor Dooley
2022-11-01 11:07 ` liaochang (A)
2022-11-01 23:30 ` Conor Dooley
2022-11-03 1:23 ` liaochang (A)
2022-10-30 9:01 ` [PATCH 2/8] riscv/kprobe: Allocate detour buffer from module area Chen Guokai
2022-10-30 9:01 ` [PATCH 3/8] riscv/kprobe: Prepare the skeleton to prepare optimized kprobe Chen Guokai
2022-10-30 9:01 ` [PATCH 4/8] riscv/kprobe: Add common RVI and RVC instruction decoder code Chen Guokai
2022-10-30 9:01 ` [PATCH 5/8] riscv/kprobe: Search free register(s) to clobber for 'AUIPC/JALR' Chen Guokai
2022-10-30 9:01 ` [PATCH 6/8] riscv/kprobe: Add code to check if kprobe can be optimized Chen Guokai
2022-10-30 9:01 ` [PATCH 7/8] riscv/kprobe: Prepare detour buffer for optimized kprobe Chen Guokai
2022-10-30 9:01 ` Chen Guokai [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221030090141.2550837-9-chenguokai17@mails.ucas.ac.cn \
--to=chenguokai17@mails.ucas.ac.cn \
--cc=aou@eecs.berkeley.edu \
--cc=liaochang1@huawei.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-riscv@lists.infradead.org \
--cc=mingo@redhat.com \
--cc=palmer@dabbelt.com \
--cc=paul.walmsley@sifive.com \
--cc=rostedt@goodmis.org \
--cc=sfr@canb.auug.org.au \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).