From: Alexei Starovoitov <alexei.starovoitov@gmail.com>
To: bpf@vger.kernel.org
Cc: daniel@iogearbox.net, andrii@kernel.org,
torvalds@linux-foundation.org, brho@google.com,
hannes@cmpxchg.org, akpm@linux-foundation.org, urezki@gmail.com,
hch@infradead.org, linux-mm@kvack.org, kernel-team@fb.com
Subject: [PATCH v3 bpf-next 04/14] bpf: Add x86-64 JIT support for bpf_addr_space_cast instruction.
Date: Thu, 7 Mar 2024 17:08:02 -0800 [thread overview]
Message-ID: <20240308010812.89848-5-alexei.starovoitov@gmail.com> (raw)
In-Reply-To: <20240308010812.89848-1-alexei.starovoitov@gmail.com>
From: Alexei Starovoitov <ast@kernel.org>
LLVM generates bpf_addr_space_cast instruction while translating
pointers between native (zero) address space and
__attribute__((address_space(N))).
The addr_space=1 is reserved as bpf_arena address space.
rY = addr_space_cast(rX, 0, 1) is processed by the verifier and
converted to normal 32-bit move: wX = wY
rY = addr_space_cast(rX, 1, 0) has to be converted by JIT:
aux_reg = upper_32_bits of arena->user_vm_start
aux_reg <<= 32
wX = wY // clear upper 32 bits of dst register
if (wX) // if not zero add upper bits of user_vm_start
wX |= aux_reg
JIT can do it more efficiently:
mov dst_reg32, src_reg32 // 32-bit move
shl dst_reg, 32
or dst_reg, user_vm_start
rol dst_reg, 32
xor r11, r11
test dst_reg32, dst_reg32 // check if lower 32-bit are zero
cmove r11, dst_reg // if so, set dst_reg to zero
// Intel swapped src/dst register encoding in CMOVcc
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
arch/x86/net/bpf_jit_comp.c | 42 ++++++++++++++++++++++++++++++++++++-
include/linux/filter.h | 1 +
kernel/bpf/core.c | 5 +++++
3 files changed, 47 insertions(+), 1 deletion(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 38705a1abe62..27058d7395f6 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1276,13 +1276,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
bool tail_call_seen = false;
bool seen_exit = false;
u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
- u64 arena_vm_start;
+ u64 arena_vm_start, user_vm_start;
int i, excnt = 0;
int ilen, proglen = 0;
u8 *prog = temp;
int err;
arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
+ user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
detect_reg_usage(insn, insn_cnt, callee_regs_used,
&tail_call_seen);
@@ -1350,6 +1351,40 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
break;
case BPF_ALU64 | BPF_MOV | BPF_X:
+ if (insn->off == BPF_ADDR_SPACE_CAST &&
+ insn->imm == 1U << 16) {
+ if (dst_reg != src_reg)
+ /* 32-bit mov */
+ emit_mov_reg(&prog, false, dst_reg, src_reg);
+ /* shl dst_reg, 32 */
+ maybe_emit_1mod(&prog, dst_reg, true);
+ EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32);
+
+ /* or dst_reg, user_vm_start */
+ maybe_emit_1mod(&prog, dst_reg, true);
+ if (is_axreg(dst_reg))
+ EMIT1_off32(0x0D, user_vm_start >> 32);
+ else
+ EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32);
+
+ /* rol dst_reg, 32 */
+ maybe_emit_1mod(&prog, dst_reg, true);
+ EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32);
+
+ /* xor r11, r11 */
+ EMIT3(0x4D, 0x31, 0xDB);
+
+ /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */
+ maybe_emit_mod(&prog, dst_reg, dst_reg, false);
+ EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg));
+
+ /* cmove r11, dst_reg; if so, set dst_reg to zero */
+ /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */
+ maybe_emit_mod(&prog, AUX_REG, dst_reg, true);
+ EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg));
+ break;
+ }
+ fallthrough;
case BPF_ALU | BPF_MOV | BPF_X:
if (insn->off == 0)
emit_mov_reg(&prog,
@@ -3432,6 +3467,11 @@ void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
}
}
+bool bpf_jit_supports_arena(void)
+{
+ return true;
+}
+
bool bpf_jit_supports_ptr_xchg(void)
{
return true;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index b119f04ecb0b..c99bc3df2d28 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -962,6 +962,7 @@ bool bpf_jit_supports_kfunc_call(void);
bool bpf_jit_supports_far_kfunc_call(void);
bool bpf_jit_supports_exceptions(void);
bool bpf_jit_supports_ptr_xchg(void);
+bool bpf_jit_supports_arena(void);
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
bool bpf_helper_changes_pkt_data(void *func);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index a8ecf69c7754..bdbdc75cdcd5 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2932,6 +2932,11 @@ bool __weak bpf_jit_supports_far_kfunc_call(void)
return false;
}
+bool __weak bpf_jit_supports_arena(void)
+{
+ return false;
+}
+
/* Return TRUE if the JIT backend satisfies the following two conditions:
* 1) JIT backend supports atomic_xchg() on pointer-sized words.
* 2) Under the specific arch, the implementation of xchg() is the same
--
2.43.0
next prev parent reply other threads:[~2024-03-08 1:08 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-03-08 1:07 [PATCH v3 bpf-next 00/14] bpf: Introduce BPF arena Alexei Starovoitov
2024-03-08 1:07 ` [PATCH v3 bpf-next 01/14] bpf: Introduce bpf_arena Alexei Starovoitov
2024-03-11 22:01 ` Andrii Nakryiko
2024-03-11 22:41 ` Alexei Starovoitov
2024-03-11 22:59 ` Andrii Nakryiko
2024-03-12 0:47 ` Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 02/14] bpf: Disasm support for addr_space_cast instruction Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 03/14] bpf: Add x86-64 JIT support for PROBE_MEM32 pseudo instructions Alexei Starovoitov
2024-03-11 22:05 ` Andrii Nakryiko
2024-03-11 22:44 ` Alexei Starovoitov
2024-03-08 1:08 ` Alexei Starovoitov [this message]
2024-03-08 1:08 ` [PATCH v3 bpf-next 05/14] bpf: Recognize addr_space_cast instruction in the verifier Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 06/14] bpf: Recognize btf_decl_tag("arg:arena") as PTR_TO_ARENA Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 07/14] libbpf: Add __arg_arena to bpf_helpers.h Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 08/14] libbpf: Add support for bpf_arena Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 09/14] bpftool: Recognize arena map type Alexei Starovoitov
2024-03-11 17:08 ` Quentin Monnet
2024-03-08 1:08 ` [PATCH v3 bpf-next 10/14] libbpf: Recognize __arena global varaibles Alexei Starovoitov
2024-03-11 17:09 ` Quentin Monnet
2024-03-08 1:08 ` [PATCH v3 bpf-next 11/14] bpf: Add helper macro bpf_addr_space_cast() Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 12/14] selftests/bpf: Add unit tests for bpf_arena_alloc/free_pages Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 13/14] selftests/bpf: Add bpf_arena_list test Alexei Starovoitov
2024-03-08 1:08 ` [PATCH v3 bpf-next 14/14] selftests/bpf: Add bpf_arena_htab test Alexei Starovoitov
2024-03-11 22:45 ` [PATCH v3 bpf-next 00/14] bpf: Introduce BPF arena Andrii Nakryiko
2024-03-11 23:02 ` Alexei Starovoitov
2024-03-11 22:50 ` patchwork-bot+netdevbpf
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240308010812.89848-5-alexei.starovoitov@gmail.com \
--to=alexei.starovoitov@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=andrii@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=brho@google.com \
--cc=daniel@iogearbox.net \
--cc=hannes@cmpxchg.org \
--cc=hch@infradead.org \
--cc=kernel-team@fb.com \
--cc=linux-mm@kvack.org \
--cc=torvalds@linux-foundation.org \
--cc=urezki@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).