From: forrestniu@foxmail.com To: peterz@infradead.org, paul.walmsley@sifive.com, palmer@dabbelt.com, aou@eecs.berkeley.edu, linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org Cc: falcon@tinylab.org, forrestniu@foxmail.com, suagrfillet@gmail.com, bmeng@tinylab.org, wangliming@walimis.org, jpoimboe@kernel.org, christophe.leroy@csgroup.eu Subject: [RFC PATCH] riscv: Add static call implementation Date: Sun, 2 Apr 2023 13:49:27 +0800 [thread overview] Message-ID: <tencent_A8A256967B654625AEE1DB222514B0613B07@qq.com> (raw) From: ForrestNiu <forrestniu@foxmail.com> Add the riscv static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes. The "inline" version is under development. Signed-off-by: Falcon <falcon@tinylab.org> Signed-off-by: ForrestNiu <forrestniu@foxmail.com> --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/static_call.h | 30 +++++++ arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/static_call.c | 117 +++++++++++++++++++++++++++ arch/riscv/kernel/vmlinux.lds.S | 1 + 5 files changed, 150 insertions(+) create mode 100644 arch/riscv/include/asm/static_call.h create mode 100644 arch/riscv/kernel/static_call.c diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5b182d1c196c..06f541c6d824 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -115,6 +115,7 @@ config RISCV select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_STACKPROTECTOR + select HAVE_STATIC_CALL select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select IRQ_FORCED_THREADING diff --git a/arch/riscv/include/asm/static_call.h b/arch/riscv/include/asm/static_call.h new file mode 100644 index 000000000000..617c00adf45c --- /dev/null +++ b/arch/riscv/include/asm/static_call.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H +#include <linux/ftrace.h> + +#define __ARCH_DEFINE_STATIC_CALL_TRAMP(name,insns) \ + asm(".pushsection .static_call.text, \"ax\" \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + insns " \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name,"la t0,"#func";jalr t1,0(t0);") + +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name,"ret;") + +#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name,__static_call_return0;) + +#define ARCH_ADD_TRAMP_KEY(name) \ + asm(".pushsection .static_call_tramp_key, \"a\" \n" \ + ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \ + ".long " STATIC_CALL_KEY_STR(name) " - . \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 4cf303a779ab..3d86edfedb7e 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -44,6 +44,7 @@ obj-y += setup.o obj-y += signal.o obj-y += syscall_table.o obj-y += sys_riscv.o +obj-y += static_call.o obj-y += time.o obj-y += traps.o obj-y += riscv_ksyms.o diff --git a/arch/riscv/kernel/static_call.c b/arch/riscv/kernel/static_call.c new file mode 100644 index 000000000000..58bf86f0ac00 --- /dev/null +++ b/arch/riscv/kernel/static_call.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/patch.h> +#include <linux/ftrace.h> + +#define RISCV_INSN_JALR 0x00000067U +#define RISCV_INSN_ADDI 0x00000013U +#define RISCV_INSN_AUIPC 0x00000017U +#define AUIPC_T0_MASK 0x00000280U +#define JALR_T1_T0_MASK 0x00028300U + +enum insn_type { + CALL = 0, /* site call */ + NOP = 1, /* site cond-call */ + JMP = 2, /* tramp / site tail-call */ + RET = 3, /* tramp / site cond-tail-call */ +}; + +static void __ref __static_call_transform(void *insn, enum insn_type type, void *func,bool modinit) +{ + u32 call[2]={0}; + u32 jump_code = (*(u32*)(insn+sizeof(call[0])) & GENMASK(6,0)); + + switch (type) { + case CALL: + break; + case NOP: + break; + case JMP: + if(jump_code == RISCV_INSN_ADDI ) + { + /* + * When the trampoland is initialized, three instructions are as follows: + * "auipc t0,imm20" + * "addi t0,imm12" + * "jalr t1,0(t0)" + * First, use the "jalr t1,imm12(t0)" replace the "addi t0,imm12" + * To ensure that two instructions are caused in the case of multiple nuclear + * conditions to lead to unknown errors. + */ + make_call_t0(insn, func, call); + patch_text_nosync(insn+sizeof(call[0]), &call[1], sizeof(call[1])); + + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * "jalr t1,0(t0)" + * The first two instructions have guaranteed that the third instruction will + * not be ordered, so the third instruction can be updated.After the update is completed, + * the second instruction can update to make the third jump instruction take effect. + * Use the "auipc t0,imm20" replace "jalr t0,imm12",the third jump instruction will take effect + */ + make_call_t0(insn+sizeof(call[0]), func, call); + + patch_text_nosync(insn+sizeof(call), &call[1], sizeof(call[1])); + patch_text_nosync(insn+sizeof(call[0]), &call[0], sizeof(call[0])); + } + else if( jump_code == RISCV_INSN_AUIPC ) + { + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * The first auipc instruction will not take effect and can be updated easily. + * After updating the first auipc instruction, update the second auipc instruction + * to the jalr instruction. + */ + make_call_t0(insn, func, call); + patch_text_nosync(insn, &call[0], sizeof(call[0])); + patch_text_nosync(insn+sizeof(call[0]), &call[1], sizeof(call[1])); + } + else if( jump_code == RISCV_INSN_JALR ) + { + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * "jalr t1,imm12(t0)" + * This looks like "jump_code == RISCV_INSN_ADDI". + */ + make_call_t0(insn+sizeof(call[0]), func, call); + + patch_text_nosync(insn+sizeof(call), &call[1], sizeof(call[1])); + patch_text_nosync(insn+sizeof(call[0]), &call[0], sizeof(call[0])); + } + break; + case RET://If func is NULL, the upgrade is not performed + break; + } +} + +static inline enum insn_type __sc_insn(bool null, bool tail) +{ + /* + * Encode the following table without branches: + * + * tail null insn + * -----+-------+------ + * 0 | 0 | CALL + * 0 | 1 | NOP + * 1 | 0 | JMP + * 1 | 1 | RET + */ + return 2*tail + null; +} + +void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) +{ + mutex_lock(&text_mutex); + __static_call_transform(tramp, __sc_insn(!func, true), func,false); + mutex_unlock(&text_mutex); +} +EXPORT_SYMBOL_GPL(arch_static_call_transform); diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 53a8ad65b255..1b06ac1a511d 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -48,6 +48,7 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT + STATIC_CALL_TEXT _etext = .; } -- 2.39.2
WARNING: multiple messages have this Message-ID (diff)
From: forrestniu@foxmail.com To: peterz@infradead.org, paul.walmsley@sifive.com, palmer@dabbelt.com, aou@eecs.berkeley.edu, linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org Cc: falcon@tinylab.org, forrestniu@foxmail.com, suagrfillet@gmail.com, bmeng@tinylab.org, wangliming@walimis.org, jpoimboe@kernel.org, christophe.leroy@csgroup.eu Subject: [RFC PATCH] riscv: Add static call implementation Date: Sun, 2 Apr 2023 13:49:27 +0800 [thread overview] Message-ID: <tencent_A8A256967B654625AEE1DB222514B0613B07@qq.com> (raw) From: ForrestNiu <forrestniu@foxmail.com> Add the riscv static call implementation. For each key, a permanent trampoline is created which is the destination for all static calls for the given key. The trampoline has a direct jump which gets patched by static_call_update() when the destination function changes. The "inline" version is under development. Signed-off-by: Falcon <falcon@tinylab.org> Signed-off-by: ForrestNiu <forrestniu@foxmail.com> --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/static_call.h | 30 +++++++ arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/static_call.c | 117 +++++++++++++++++++++++++++ arch/riscv/kernel/vmlinux.lds.S | 1 + 5 files changed, 150 insertions(+) create mode 100644 arch/riscv/include/asm/static_call.h create mode 100644 arch/riscv/kernel/static_call.c diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5b182d1c196c..06f541c6d824 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -115,6 +115,7 @@ config RISCV select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_STACKPROTECTOR + select HAVE_STATIC_CALL select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select IRQ_FORCED_THREADING diff --git a/arch/riscv/include/asm/static_call.h b/arch/riscv/include/asm/static_call.h new file mode 100644 index 000000000000..617c00adf45c --- /dev/null +++ b/arch/riscv/include/asm/static_call.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_STATIC_CALL_H +#define _ASM_STATIC_CALL_H +#include <linux/ftrace.h> + +#define __ARCH_DEFINE_STATIC_CALL_TRAMP(name,insns) \ + asm(".pushsection .static_call.text, \"ax\" \n" \ + ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ + STATIC_CALL_TRAMP_STR(name) ": \n" \ + insns " \n" \ + ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ + ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \ + ".popsection \n") + +#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name,"la t0,"#func";jalr t1,0(t0);") + +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \ + __ARCH_DEFINE_STATIC_CALL_TRAMP(name,"ret;") + +#define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) \ + ARCH_DEFINE_STATIC_CALL_TRAMP(name,__static_call_return0;) + +#define ARCH_ADD_TRAMP_KEY(name) \ + asm(".pushsection .static_call_tramp_key, \"a\" \n" \ + ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \ + ".long " STATIC_CALL_KEY_STR(name) " - . \n" \ + ".popsection \n") + +#endif /* _ASM_STATIC_CALL_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 4cf303a779ab..3d86edfedb7e 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -44,6 +44,7 @@ obj-y += setup.o obj-y += signal.o obj-y += syscall_table.o obj-y += sys_riscv.o +obj-y += static_call.o obj-y += time.o obj-y += traps.o obj-y += riscv_ksyms.o diff --git a/arch/riscv/kernel/static_call.c b/arch/riscv/kernel/static_call.c new file mode 100644 index 000000000000..58bf86f0ac00 --- /dev/null +++ b/arch/riscv/kernel/static_call.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/static_call.h> +#include <linux/memory.h> +#include <linux/bug.h> +#include <asm/patch.h> +#include <linux/ftrace.h> + +#define RISCV_INSN_JALR 0x00000067U +#define RISCV_INSN_ADDI 0x00000013U +#define RISCV_INSN_AUIPC 0x00000017U +#define AUIPC_T0_MASK 0x00000280U +#define JALR_T1_T0_MASK 0x00028300U + +enum insn_type { + CALL = 0, /* site call */ + NOP = 1, /* site cond-call */ + JMP = 2, /* tramp / site tail-call */ + RET = 3, /* tramp / site cond-tail-call */ +}; + +static void __ref __static_call_transform(void *insn, enum insn_type type, void *func,bool modinit) +{ + u32 call[2]={0}; + u32 jump_code = (*(u32*)(insn+sizeof(call[0])) & GENMASK(6,0)); + + switch (type) { + case CALL: + break; + case NOP: + break; + case JMP: + if(jump_code == RISCV_INSN_ADDI ) + { + /* + * When the trampoland is initialized, three instructions are as follows: + * "auipc t0,imm20" + * "addi t0,imm12" + * "jalr t1,0(t0)" + * First, use the "jalr t1,imm12(t0)" replace the "addi t0,imm12" + * To ensure that two instructions are caused in the case of multiple nuclear + * conditions to lead to unknown errors. + */ + make_call_t0(insn, func, call); + patch_text_nosync(insn+sizeof(call[0]), &call[1], sizeof(call[1])); + + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * "jalr t1,0(t0)" + * The first two instructions have guaranteed that the third instruction will + * not be ordered, so the third instruction can be updated.After the update is completed, + * the second instruction can update to make the third jump instruction take effect. + * Use the "auipc t0,imm20" replace "jalr t0,imm12",the third jump instruction will take effect + */ + make_call_t0(insn+sizeof(call[0]), func, call); + + patch_text_nosync(insn+sizeof(call), &call[1], sizeof(call[1])); + patch_text_nosync(insn+sizeof(call[0]), &call[0], sizeof(call[0])); + } + else if( jump_code == RISCV_INSN_AUIPC ) + { + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * The first auipc instruction will not take effect and can be updated easily. + * After updating the first auipc instruction, update the second auipc instruction + * to the jalr instruction. + */ + make_call_t0(insn, func, call); + patch_text_nosync(insn, &call[0], sizeof(call[0])); + patch_text_nosync(insn+sizeof(call[0]), &call[1], sizeof(call[1])); + } + else if( jump_code == RISCV_INSN_JALR ) + { + /* + * After the last dynamic adjustment, the instructions are as follows: + * "auipc t0,imm20" + * "jalr t1,imm12(t0)" + * "jalr t1,imm12(t0)" + * This looks like "jump_code == RISCV_INSN_ADDI". + */ + make_call_t0(insn+sizeof(call[0]), func, call); + + patch_text_nosync(insn+sizeof(call), &call[1], sizeof(call[1])); + patch_text_nosync(insn+sizeof(call[0]), &call[0], sizeof(call[0])); + } + break; + case RET://If func is NULL, the upgrade is not performed + break; + } +} + +static inline enum insn_type __sc_insn(bool null, bool tail) +{ + /* + * Encode the following table without branches: + * + * tail null insn + * -----+-------+------ + * 0 | 0 | CALL + * 0 | 1 | NOP + * 1 | 0 | JMP + * 1 | 1 | RET + */ + return 2*tail + null; +} + +void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) +{ + mutex_lock(&text_mutex); + __static_call_transform(tramp, __sc_insn(!func, true), func,false); + mutex_unlock(&text_mutex); +} +EXPORT_SYMBOL_GPL(arch_static_call_transform); diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 53a8ad65b255..1b06ac1a511d 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -48,6 +48,7 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT + STATIC_CALL_TEXT _etext = .; } -- 2.39.2 _______________________________________________ linux-riscv mailing list linux-riscv@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-riscv
next reply other threads:[~2023-04-02 5:51 UTC|newest] Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top 2023-04-02 5:49 forrestniu [this message] 2023-04-02 5:49 ` [RFC PATCH] riscv: Add static call implementation forrestniu 2023-04-02 6:49 ` kernel test robot 2023-04-02 6:49 ` kernel test robot
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=tencent_A8A256967B654625AEE1DB222514B0613B07@qq.com \ --to=forrestniu@foxmail.com \ --cc=aou@eecs.berkeley.edu \ --cc=bmeng@tinylab.org \ --cc=christophe.leroy@csgroup.eu \ --cc=falcon@tinylab.org \ --cc=jpoimboe@kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-riscv@lists.infradead.org \ --cc=palmer@dabbelt.com \ --cc=paul.walmsley@sifive.com \ --cc=peterz@infradead.org \ --cc=suagrfillet@gmail.com \ --cc=wangliming@walimis.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.