Re: [PATCH 2/4] arm64: implement support for static call trampolines

[Ard Biesheuvel <ardb@kernel.org>]

On Tue, 21 Sept 2021 at 01:32, Frederic Weisbecker <frederic@kernel.org> wrote:
>
> From: Ard Biesheuvel <ardb@kernel.org>
>
> [fweisbec: rebased against 5.15-rc2. There has been quite some changes
>  on arm64 since then, especially with insn/patching, so some naming may
>  not be relevant anymore]
>

This patch does not include the static_call.c file references to which
are being added below.


> Implement arm64 support for the 'unoptimized' static call variety, which
> routes all calls through a single trampoline that is patched to perform a
> tail call to the selected function.
>
> Since static call targets may be located in modules loaded out of direct
> branching range, we need to use a ADRP/ADD pair to load the branch target
> into R16 and use a branch-to-register (BR) instruction to perform an
> indirect call. Unlike on x86, there is no pressing need on arm64 to avoid
> indirect calls at all cost, but hiding it from the compiler as is done
> here does have some benefits:
> - the literal is located in .rodata, which gives us the same robustness
>   advantage that code patching does;
> - no performance hit on CFI enabled Clang builds that decorate compiler
>   emitted indirect calls with branch target validity checks.
>
> Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Quentin Perret <qperret@google.com>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Catalin Marinas <catalin.marinas@arm.com>
> Cc: James Morse <james.morse@arm.com>
> Cc: Will Deacon <will@kernel.org>
> Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
> ---
>  arch/arm64/Kconfig                   |  1 +
>  arch/arm64/include/asm/insn.h        |  2 ++
>  arch/arm64/include/asm/static_call.h | 28 ++++++++++++++++++++++++++++
>  arch/arm64/kernel/Makefile           |  4 ++--
>  arch/arm64/kernel/patching.c         | 14 +++++++++++---
>  arch/arm64/kernel/vmlinux.lds.S      |  1 +
>  6 files changed, 45 insertions(+), 5 deletions(-)
>  create mode 100644 arch/arm64/include/asm/static_call.h
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 5c7ae4c3954b..5b51b359ccda 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -192,6 +192,7 @@ config ARM64
>         select HAVE_PERF_REGS
>         select HAVE_PERF_USER_STACK_DUMP
>         select HAVE_REGS_AND_STACK_ACCESS_API
> +       select HAVE_STATIC_CALL
>         select HAVE_FUNCTION_ARG_ACCESS_API
>         select HAVE_FUTEX_CMPXCHG if FUTEX
>         select MMU_GATHER_RCU_TABLE_FREE
> diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
> index 6b776c8667b2..681c08b170df 100644
> --- a/arch/arm64/include/asm/insn.h
> +++ b/arch/arm64/include/asm/insn.h
> @@ -547,6 +547,8 @@ u32 aarch64_set_branch_offset(u32 insn, s32 offset);
>  s32 aarch64_insn_adrp_get_offset(u32 insn);
>  u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset);
>
> +int aarch64_literal_write(void *addr, u64 literal);
> +
>  bool aarch32_insn_is_wide(u32 insn);
>
>  #define A32_RN_OFFSET  16
> diff --git a/arch/arm64/include/asm/static_call.h b/arch/arm64/include/asm/static_call.h
> new file mode 100644
> index 000000000000..665ec2a7cdb2
> --- /dev/null
> +++ b/arch/arm64/include/asm/static_call.h
> @@ -0,0 +1,28 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _ASM_STATIC_CALL_H
> +#define _ASM_STATIC_CALL_H
> +
> +#define __ARCH_DEFINE_STATIC_CALL_TRAMP(name, target)                      \
> +       asm("   .pushsection    .static_call.text, \"ax\"               \n" \
> +           "   .align          3                                       \n" \
> +           "   .globl          " STATIC_CALL_TRAMP_STR(name) "         \n" \
> +           STATIC_CALL_TRAMP_STR(name) ":                              \n" \
> +           "   hint    34      /* BTI C */                             \n" \
> +           "   adrp    x16, 1f                                         \n" \
> +           "   ldr     x16, [x16, :lo12:1f]                            \n" \
> +           "   cbz     x16, 0f                                         \n" \
> +           "   br      x16                                             \n" \
> +           "0: ret                                                     \n" \
> +           "   .popsection                                             \n" \
> +           "   .pushsection    .rodata, \"a\"                          \n" \
> +           "   .align          3                                       \n" \
> +           "1: .quad           " target "                              \n" \
> +           "   .popsection                                             \n")
> +
> +#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)                      \
> +       __ARCH_DEFINE_STATIC_CALL_TRAMP(name, #func)
> +
> +#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)                       \
> +       __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "0x0")
> +
> +#endif /* _ASM_STATIC_CALL_H */
> diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
> index 3f1490bfb938..83f03fc1e402 100644
> --- a/arch/arm64/kernel/Makefile
> +++ b/arch/arm64/kernel/Makefile
> @@ -28,8 +28,8 @@ obj-y                 := debug-monitors.o entry.o irq.o fpsimd.o              \
>                            return_address.o cpuinfo.o cpu_errata.o              \
>                            cpufeature.o alternative.o cacheinfo.o               \
>                            smp.o smp_spin_table.o topology.o smccc-call.o       \
> -                          syscall.o proton-pack.o idreg-override.o idle.o      \
> -                          patching.o
> +                          syscall.o proton-pack.o static_call.o                \
> +                          idreg-override.o idle.o patching.o
>
>  targets                        += efi-entry.o
>
> diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c
> index 771f543464e0..841c0499eca5 100644
> --- a/arch/arm64/kernel/patching.c
> +++ b/arch/arm64/kernel/patching.c
> @@ -66,7 +66,7 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
>         return ret;
>  }
>
> -static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
> +static int __kprobes __aarch64_insn_write(void *addr, void *insn, int size)
>  {
>         void *waddr = addr;
>         unsigned long flags = 0;
> @@ -75,7 +75,7 @@ static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
>         raw_spin_lock_irqsave(&patch_lock, flags);
>         waddr = patch_map(addr, FIX_TEXT_POKE0);
>
> -       ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
> +       ret = copy_to_kernel_nofault(waddr, insn, size);
>
>         patch_unmap(FIX_TEXT_POKE0);
>         raw_spin_unlock_irqrestore(&patch_lock, flags);
> @@ -85,7 +85,15 @@ static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
>
>  int __kprobes aarch64_insn_write(void *addr, u32 insn)
>  {
> -       return __aarch64_insn_write(addr, cpu_to_le32(insn));
> +       __le32 i = cpu_to_le32(insn);
> +
> +       return __aarch64_insn_write(addr, &i, AARCH64_INSN_SIZE);
> +}
> +
> +int aarch64_literal_write(void *addr, u64 literal)
> +{
> +       BUG_ON(!IS_ALIGNED((u64)addr, sizeof(u64)));
> +       return __aarch64_insn_write(addr, &literal, sizeof(u64));
>  }
>
>  int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
> diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
> index f6b1a88245db..ceb35c35192c 100644
> --- a/arch/arm64/kernel/vmlinux.lds.S
> +++ b/arch/arm64/kernel/vmlinux.lds.S
> @@ -161,6 +161,7 @@ SECTIONS
>                         IDMAP_TEXT
>                         HIBERNATE_TEXT
>                         TRAMP_TEXT
> +                       STATIC_CALL_TEXT
>                         *(.fixup)
>                         *(.gnu.warning)
>                 . = ALIGN(16);
> --
> 2.25.1
>