* [PATCH v9 bpf-next 1/5] bpf: Allow "sk_buff" and "xdp_buff" as valid kfunc arg types
2023-01-27 19:16 [PATCH v9 bpf-next 0/5] Add skb + xdp dynptrs Joanne Koong
@ 2023-01-27 19:16 ` Joanne Koong
2023-01-27 19:17 ` [PATCH v9 bpf-next 2/5] bpf: Allow initializing dynptrs in kfuncs Joanne Koong
` (3 subsequent siblings)
4 siblings, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-27 19:16 UTC (permalink / raw)
To: bpf
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team,
Joanne Koong
The bpf mirror of the in-kernel sk_buff and xdp_buff data structures are
__sk_buff and xdp_md. Currently, when we pass in the program ctx to a
kfunc where the program ctx is a skb or xdp buffer, we reject the
program if the in-kernel definition is sk_buff/xdp_buff instead of
__sk_buff/xdp_md.
This change allows sk_buff and __sk_buff, and xdp_buff and xdp_md to be
valid matches. The user program may pass in their program ctx as a
__sk_buff or xdp_md, and the in-kernel definition of the kfunc may
define this arg as a sk_buff or xdp_buff. Please note that the
__sk_buff/xdp_md -> sk_buff/xdp_buff conversion happens in
convert_ctx_accesses() in the verifier.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
kernel/bpf/btf.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 47b8cb96f2c2..b4da17688c65 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -5606,6 +5606,10 @@ btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
* int socket_filter_bpf_prog(struct __sk_buff *skb)
* { // no fields of skb are ever used }
*/
+ if (strcmp(ctx_tname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0)
+ return ctx_type;
+ if (strcmp(ctx_tname, "xdp_md") == 0 && strcmp(tname, "xdp_buff") == 0)
+ return ctx_type;
if (strcmp(ctx_tname, tname))
return NULL;
return ctx_type;
--
2.30.2
^ permalink raw reply related [flat|nested] 44+ messages in thread* [PATCH v9 bpf-next 2/5] bpf: Allow initializing dynptrs in kfuncs
2023-01-27 19:16 [PATCH v9 bpf-next 0/5] Add skb + xdp dynptrs Joanne Koong
2023-01-27 19:16 ` [PATCH v9 bpf-next 1/5] bpf: Allow "sk_buff" and "xdp_buff" as valid kfunc arg types Joanne Koong
@ 2023-01-27 19:17 ` Joanne Koong
2023-01-27 19:17 ` [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs Joanne Koong
` (2 subsequent siblings)
4 siblings, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-27 19:17 UTC (permalink / raw)
To: bpf
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team,
Joanne Koong
This change allows kfuncs to take in an uninitialized dynptr as a
parameter. Before this change, only helper functions could successfully
use uninitialized dynptrs. This change moves the memory access check
(and stack state growing) into process_dynptr_func(), which both helpers
and kfuncs call.
This change also includes some light tidying up of existing code (eg
remove unused parameter, move spi checking logic, remove unneeded checks)
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
include/linux/bpf_verifier.h | 3 -
kernel/bpf/verifier.c | 135 +++++++++++++----------------------
2 files changed, 50 insertions(+), 88 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index aa83de1fe755..bee10101222d 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -618,9 +618,6 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env,
enum bpf_arg_type arg_type);
int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
u32 regno, u32 mem_size);
-struct bpf_call_arg_meta;
-int process_dynptr_func(struct bpf_verifier_env *env, int regno,
- enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta);
/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6bd097e0d45f..853ab671be0b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -264,7 +264,6 @@ struct bpf_call_arg_meta {
u32 ret_btf_id;
u32 subprogno;
struct btf_field *kptr_field;
- u8 uninit_dynptr_regno;
};
struct btf *btf_vmlinux;
@@ -946,41 +945,24 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
return 0;
}
-static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- int spi)
-{
- if (reg->type == CONST_PTR_TO_DYNPTR)
- return false;
-
- /* For -ERANGE (i.e. spi not falling into allocated stack slots), we
- * will do check_mem_access to check and update stack bounds later, so
- * return true for that case.
- */
- if (spi < 0)
- return spi == -ERANGE;
- /* We allow overwriting existing unreferenced STACK_DYNPTR slots, see
- * mark_stack_slots_dynptr which calls destroy_if_dynptr_stack_slot to
- * ensure dynptr objects at the slots we are touching are completely
- * destructed before we reinitialize them for a new one. For referenced
- * ones, destroy_if_dynptr_stack_slot returns an error early instead of
- * delaying it until the end where the user will get "Unreleased
- * reference" error.
- */
- return true;
-}
-
-static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- int spi)
+static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
struct bpf_func_state *state = func(env, reg);
- int i;
+ int i, spi;
- /* This already represents first slot of initialized bpf_dynptr */
+ /* This already represents first slot of initialized bpf_dynptr.
+ *
+ * Please also note that CONST_PTR_TO_DYNPTR already has fixed and
+ * var_off as 0 due to check_func_arg_reg_off's logic, so we don't
+ * need to check its offset and alignment.
+ */
if (reg->type == CONST_PTR_TO_DYNPTR)
return true;
+ spi = dynptr_get_spi(env, reg);
if (spi < 0)
return false;
+
if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
return false;
@@ -6165,11 +6147,11 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno,
* Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
* type, and declare it as 'const struct bpf_dynptr *' in their prototype.
*/
-int process_dynptr_func(struct bpf_verifier_env *env, int regno,
- enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta)
+int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
+ enum bpf_arg_type arg_type)
{
struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
- int spi = 0;
+ int err;
/* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
* ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
@@ -6178,15 +6160,6 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno,
verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n");
return -EFAULT;
}
- /* CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
- * check_func_arg_reg_off's logic. We only need to check offset
- * and its alignment for PTR_TO_STACK.
- */
- if (reg->type == PTR_TO_STACK) {
- spi = dynptr_get_spi(env, reg);
- if (spi < 0 && spi != -ERANGE)
- return spi;
- }
/* MEM_UNINIT - Points to memory that is an appropriate candidate for
* constructing a mutable bpf_dynptr object.
@@ -6204,32 +6177,47 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno,
* to.
*/
if (arg_type & MEM_UNINIT) {
- if (!is_dynptr_reg_valid_uninit(env, reg, spi)) {
+ int i, spi;
+
+ if (base_type(reg->type) == CONST_PTR_TO_DYNPTR) {
verbose(env, "Dynptr has to be an uninitialized dynptr\n");
return -EINVAL;
}
- /* We only support one dynptr being uninitialized at the moment,
- * which is sufficient for the helper functions we have right now.
+ /* For -ERANGE (i.e. spi not falling into allocated stack slots),
+ * check_mem_access will check and update stack bounds, so this
+ * is okay.
*/
- if (meta->uninit_dynptr_regno) {
- verbose(env, "verifier internal error: multiple uninitialized dynptr args\n");
- return -EFAULT;
+ spi = dynptr_get_spi(env, reg);
+ if (spi < 0 && spi != -ERANGE)
+ return spi;
+
+ /* we write BPF_DW bits (8 bytes) at a time */
+ for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
+ err = check_mem_access(env, insn_idx, regno,
+ i, BPF_DW, BPF_WRITE, -1, false);
+ if (err)
+ return err;
}
- meta->uninit_dynptr_regno = regno;
+ /* Please note that we allow overwriting existing unreferenced STACK_DYNPTR
+ * slots (mark_stack_slots_dynptr calls destroy_if_dynptr_stack_slot
+ * to ensure dynptr objects at the slots we are touching are completely
+ * destructed before we reinitialize them for a new one). For referenced
+ * ones, destroy_if_dynptr_stack_slot returns an error early instead of
+ * delaying it until the end where the user will get "Unreleased
+ * reference" error.
+ */
+ err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx);
} else /* MEM_RDONLY and None case from above */ {
- int err;
-
/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
return -EINVAL;
}
- if (!is_dynptr_reg_valid_init(env, reg, spi)) {
- verbose(env,
- "Expected an initialized dynptr as arg #%d\n",
+ if (!is_dynptr_reg_valid_init(env, reg)) {
+ verbose(env, "Expected an initialized dynptr as arg #%d\n",
regno);
return -EINVAL;
}
@@ -6256,10 +6244,8 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno,
}
err = mark_dynptr_read(env, reg);
- if (err)
- return err;
}
- return 0;
+ return err;
}
static bool arg_type_is_mem_size(enum bpf_arg_type type)
@@ -6623,7 +6609,8 @@ static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state
static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
struct bpf_call_arg_meta *meta,
- const struct bpf_func_proto *fn)
+ const struct bpf_func_proto *fn,
+ int insn_idx)
{
u32 regno = BPF_REG_1 + arg;
struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
@@ -6832,7 +6819,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
err = check_mem_size_reg(env, reg, regno, true, meta);
break;
case ARG_PTR_TO_DYNPTR:
- err = process_dynptr_func(env, regno, arg_type, meta);
+ err = process_dynptr_func(env, regno, insn_idx, arg_type);
if (err)
return err;
break;
@@ -8044,7 +8031,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
meta.func_id = func_id;
/* check args */
for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
- err = check_func_arg(env, i, &meta, fn);
+ err = check_func_arg(env, i, &meta, fn, insn_idx);
if (err)
return err;
}
@@ -8069,30 +8056,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
regs = cur_regs(env);
- /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
- * be reinitialized by any dynptr helper. Hence, mark_stack_slots_dynptr
- * is safe to do directly.
- */
- if (meta.uninit_dynptr_regno) {
- if (regs[meta.uninit_dynptr_regno].type == CONST_PTR_TO_DYNPTR) {
- verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be initialized\n");
- return -EFAULT;
- }
- /* we write BPF_DW bits (8 bytes) at a time */
- for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
- err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno,
- i, BPF_DW, BPF_WRITE, -1, false);
- if (err)
- return err;
- }
-
- err = mark_stack_slots_dynptr(env, ®s[meta.uninit_dynptr_regno],
- fn->arg_type[meta.uninit_dynptr_regno - BPF_REG_1],
- insn_idx);
- if (err)
- return err;
- }
-
if (meta.release_regno) {
err = -EINVAL;
/* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
@@ -9111,7 +9074,8 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
return ref_set_release_on_unlock(env, reg->ref_obj_id);
}
-static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta)
+static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
+ int insn_idx)
{
const char *func_name = meta->func_name, *ref_tname;
const struct btf *btf = meta->btf;
@@ -9305,7 +9269,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
return -EINVAL;
}
- ret = process_dynptr_func(env, regno, ARG_PTR_TO_DYNPTR | MEM_RDONLY, NULL);
+ ret = process_dynptr_func(env, regno, insn_idx,
+ ARG_PTR_TO_DYNPTR | MEM_RDONLY);
if (ret < 0)
return ret;
break;
@@ -9471,7 +9436,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
/* Check the arguments */
- err = check_kfunc_args(env, &meta);
+ err = check_kfunc_args(env, &meta, insn_idx);
if (err < 0)
return err;
/* In case of release function, we get register number of refcounted
--
2.30.2
^ permalink raw reply related [flat|nested] 44+ messages in thread* [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-27 19:16 [PATCH v9 bpf-next 0/5] Add skb + xdp dynptrs Joanne Koong
2023-01-27 19:16 ` [PATCH v9 bpf-next 1/5] bpf: Allow "sk_buff" and "xdp_buff" as valid kfunc arg types Joanne Koong
2023-01-27 19:17 ` [PATCH v9 bpf-next 2/5] bpf: Allow initializing dynptrs in kfuncs Joanne Koong
@ 2023-01-27 19:17 ` Joanne Koong
2023-01-29 23:39 ` Alexei Starovoitov
` (2 more replies)
2023-01-27 19:17 ` [PATCH v9 bpf-next 4/5] bpf: Add xdp dynptrs Joanne Koong
2023-01-27 19:17 ` [PATCH v9 bpf-next 5/5] selftests/bpf: tests for using dynptrs to parse skb and xdp buffers Joanne Koong
4 siblings, 3 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-27 19:17 UTC (permalink / raw)
To: bpf
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team,
Joanne Koong
Add skb dynptrs, which are dynptrs whose underlying pointer points
to a skb. The dynptr acts on skb data. skb dynptrs have two main
benefits. One is that they allow operations on sizes that are not
statically known at compile-time (eg variable-sized accesses).
Another is that parsing the packet data through dynptrs (instead of
through direct access of skb->data and skb->data_end) can be more
ergonomic and less brittle (eg does not need manual if checking for
being within bounds of data_end).
For bpf prog types that don't support writes on skb data, the dynptr is
read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
will return a data slice that is read-only where any writes to it will
be rejected by the verifier).
For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
interfaces, reading and writing from/to data in the head as well as from/to
non-linear paged buffers is supported. For data slices (through the
bpf_dynptr_data() interface), if the data is in a paged buffer, the user
must first call bpf_skb_pull_data() to pull the data into the linear
portion.
Any bpf_dynptr_write() automatically invalidates any prior data slices
to the skb dynptr. This is because a bpf_dynptr_write() may be writing
to data in a paged buffer, so it will need to pull the buffer first into
the head. The reason it needs to be pulled instead of writing directly to
the paged buffers is because they may be cloned (only the head of the skb
is by default uncloned). As such, any bpf_dynptr_write() will
automatically have its prior data slices invalidated, even if the write
is to data in the skb head (the verifier has no way of differentiating
whether the write is to the head or paged buffers during program load
time). Please note as well that any other helper calls that change the
underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
slices of the skb dynptr as well. The stack trace for this is
check_helper_call() -> clear_all_pkt_pointers() ->
__clear_all_pkt_pointers() -> mark_reg_unknown().
For examples of how skb dynptrs can be used, please see the attached
selftests.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
include/linux/bpf.h | 82 +++++++++------
include/linux/filter.h | 18 ++++
include/uapi/linux/bpf.h | 37 +++++--
kernel/bpf/btf.c | 18 ++++
kernel/bpf/helpers.c | 95 ++++++++++++++---
kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
net/core/filter.c | 60 ++++++++++-
tools/include/uapi/linux/bpf.h | 37 +++++--
8 files changed, 432 insertions(+), 100 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 14a0264fac57..1ac061b64582 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -575,11 +575,14 @@ enum bpf_type_flag {
/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
+ /* DYNPTR points to sk_buff */
+ DYNPTR_TYPE_SKB = BIT(14 + BPF_BASE_TYPE_BITS),
+
__BPF_TYPE_FLAG_MAX,
__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
};
-#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF)
+#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB)
/* Max number of base types. */
#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
@@ -1082,6 +1085,35 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
return bpf_func(ctx, insnsi);
}
+/* the implementation of the opaque uapi struct bpf_dynptr */
+struct bpf_dynptr_kern {
+ void *data;
+ /* Size represents the number of usable bytes of dynptr data.
+ * If for example the offset is at 4 for a local dynptr whose data is
+ * of type u64, the number of usable bytes is 4.
+ *
+ * The upper 8 bits are reserved. It is as follows:
+ * Bits 0 - 23 = size
+ * Bits 24 - 30 = dynptr type
+ * Bit 31 = whether dynptr is read-only
+ */
+ u32 size;
+ u32 offset;
+} __aligned(8);
+
+enum bpf_dynptr_type {
+ BPF_DYNPTR_TYPE_INVALID,
+ /* Points to memory that is local to the bpf program */
+ BPF_DYNPTR_TYPE_LOCAL,
+ /* Underlying data is a ringbuf record */
+ BPF_DYNPTR_TYPE_RINGBUF,
+ /* Underlying data is a sk_buff */
+ BPF_DYNPTR_TYPE_SKB,
+};
+
+int bpf_dynptr_check_size(u32 size);
+u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr);
+
#ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
@@ -2216,6 +2248,11 @@ static inline bool has_current_bpf_ctx(void)
}
void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
+
+void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
+ enum bpf_dynptr_type type, u32 offset, u32 size);
+void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
+void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
#else /* !CONFIG_BPF_SYSCALL */
static inline struct bpf_prog *bpf_prog_get(u32 ufd)
{
@@ -2445,6 +2482,19 @@ static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
}
+
+static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
+ enum bpf_dynptr_type type, u32 offset, u32 size)
+{
+}
+
+static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
+{
+}
+
+static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
+{
+}
#endif /* CONFIG_BPF_SYSCALL */
void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
@@ -2863,36 +2913,6 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
u32 num_args, struct bpf_bprintf_data *data);
void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
-/* the implementation of the opaque uapi struct bpf_dynptr */
-struct bpf_dynptr_kern {
- void *data;
- /* Size represents the number of usable bytes of dynptr data.
- * If for example the offset is at 4 for a local dynptr whose data is
- * of type u64, the number of usable bytes is 4.
- *
- * The upper 8 bits are reserved. It is as follows:
- * Bits 0 - 23 = size
- * Bits 24 - 30 = dynptr type
- * Bit 31 = whether dynptr is read-only
- */
- u32 size;
- u32 offset;
-} __aligned(8);
-
-enum bpf_dynptr_type {
- BPF_DYNPTR_TYPE_INVALID,
- /* Points to memory that is local to the bpf program */
- BPF_DYNPTR_TYPE_LOCAL,
- /* Underlying data is a kernel-produced ringbuf record */
- BPF_DYNPTR_TYPE_RINGBUF,
-};
-
-void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
- enum bpf_dynptr_type type, u32 offset, u32 size);
-void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
-int bpf_dynptr_check_size(u32 size);
-u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr);
-
#ifdef CONFIG_BPF_LSM
void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
void bpf_cgroup_atype_put(int cgroup_atype);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index ccc4a4a58c72..c87d13954d89 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1541,4 +1541,22 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u64 index
return XDP_REDIRECT;
}
+#ifdef CONFIG_NET
+int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
+int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
+ u32 len, u64 flags);
+#else /* CONFIG_NET */
+static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset,
+ void *to, u32 len)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset,
+ const void *from, u32 len, u64 flags)
+{
+ return -EOPNOTSUPP;
+}
+#endif /* CONFIG_NET */
+
#endif /* __LINUX_FILTER_H__ */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index ba0f0cfb5e42..f6910392d339 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5320,22 +5320,45 @@ union bpf_attr {
* Description
* Write *len* bytes from *src* into *dst*, starting from *offset*
* into *dst*.
- * *flags* is currently unused.
+ *
+ * *flags* must be 0 except for skb-type dynptrs.
+ *
+ * For skb-type dynptrs:
+ * * All data slices of the dynptr are automatically
+ * invalidated after **bpf_dynptr_write**\ (). If you wish to
+ * avoid this, please perform the write using direct data slices
+ * instead.
+ *
+ * * For *flags*, please see the flags accepted by
+ * **bpf_skb_store_bytes**\ ().
* Return
* 0 on success, -E2BIG if *offset* + *len* exceeds the length
* of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst*
- * is a read-only dynptr or if *flags* is not 0.
+ * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs,
+ * other errors correspond to errors returned by **bpf_skb_store_bytes**\ ().
*
* void *bpf_dynptr_data(const struct bpf_dynptr *ptr, u32 offset, u32 len)
* Description
* Get a pointer to the underlying dynptr data.
*
* *len* must be a statically known value. The returned data slice
- * is invalidated whenever the dynptr is invalidated.
- * Return
- * Pointer to the underlying dynptr data, NULL if the dynptr is
- * read-only, if the dynptr is invalid, or if the offset and length
- * is out of bounds.
+ * is invalidated whenever the dynptr is invalidated. Please note
+ * that if the dynptr is read-only, then the returned data slice will
+ * be read-only.
+ *
+ * For skb-type dynptrs:
+ * * If *offset* + *len* extends into the skb's paged buffers,
+ * the user should manually pull the skb with **bpf_skb_pull_data**\ ()
+ * and try again.
+ *
+ * * The data slice is automatically invalidated anytime
+ * **bpf_dynptr_write**\ () or a helper call that changes
+ * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
+ * is called.
+ * Return
+ * Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
+ * or if the offset and length is out of bounds or in a paged buffer for
+ * skb-type dynptrs.
*
* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
* Description
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index b4da17688c65..35d0780f2eb9 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -207,6 +207,11 @@ enum btf_kfunc_hook {
BTF_KFUNC_HOOK_TRACING,
BTF_KFUNC_HOOK_SYSCALL,
BTF_KFUNC_HOOK_FMODRET,
+ BTF_KFUNC_HOOK_CGROUP_SKB,
+ BTF_KFUNC_HOOK_SCHED_ACT,
+ BTF_KFUNC_HOOK_SK_SKB,
+ BTF_KFUNC_HOOK_SOCKET_FILTER,
+ BTF_KFUNC_HOOK_LWT,
BTF_KFUNC_HOOK_MAX,
};
@@ -7609,6 +7614,19 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type)
return BTF_KFUNC_HOOK_TRACING;
case BPF_PROG_TYPE_SYSCALL:
return BTF_KFUNC_HOOK_SYSCALL;
+ case BPF_PROG_TYPE_CGROUP_SKB:
+ return BTF_KFUNC_HOOK_CGROUP_SKB;
+ case BPF_PROG_TYPE_SCHED_ACT:
+ return BTF_KFUNC_HOOK_SCHED_ACT;
+ case BPF_PROG_TYPE_SK_SKB:
+ return BTF_KFUNC_HOOK_SK_SKB;
+ case BPF_PROG_TYPE_SOCKET_FILTER:
+ return BTF_KFUNC_HOOK_SOCKET_FILTER;
+ case BPF_PROG_TYPE_LWT_OUT:
+ case BPF_PROG_TYPE_LWT_IN:
+ case BPF_PROG_TYPE_LWT_XMIT:
+ case BPF_PROG_TYPE_LWT_SEG6LOCAL:
+ return BTF_KFUNC_HOOK_LWT;
default:
return BTF_KFUNC_HOOK_MAX;
}
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 458db2db2f81..a79d522b3a26 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1420,11 +1420,21 @@ static bool bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr)
return ptr->size & DYNPTR_RDONLY_BIT;
}
+void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
+{
+ ptr->size |= DYNPTR_RDONLY_BIT;
+}
+
static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_type type)
{
ptr->size |= type << DYNPTR_TYPE_SHIFT;
}
+static enum bpf_dynptr_type bpf_dynptr_get_type(const struct bpf_dynptr_kern *ptr)
+{
+ return (ptr->size & ~(DYNPTR_RDONLY_BIT)) >> DYNPTR_TYPE_SHIFT;
+}
+
u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr)
{
return ptr->size & DYNPTR_SIZE_MASK;
@@ -1497,6 +1507,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src,
u32, offset, u64, flags)
{
+ enum bpf_dynptr_type type;
int err;
if (!src->data || flags)
@@ -1506,13 +1517,23 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern
if (err)
return err;
- /* Source and destination may possibly overlap, hence use memmove to
- * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
- * pointing to overlapping PTR_TO_MAP_VALUE regions.
- */
- memmove(dst, src->data + src->offset + offset, len);
+ type = bpf_dynptr_get_type(src);
- return 0;
+ switch (type) {
+ case BPF_DYNPTR_TYPE_LOCAL:
+ case BPF_DYNPTR_TYPE_RINGBUF:
+ /* Source and destination may possibly overlap, hence use memmove to
+ * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
+ * pointing to overlapping PTR_TO_MAP_VALUE regions.
+ */
+ memmove(dst, src->data + src->offset + offset, len);
+ return 0;
+ case BPF_DYNPTR_TYPE_SKB:
+ return __bpf_skb_load_bytes(src->data, src->offset + offset, dst, len);
+ default:
+ WARN_ONCE(true, "bpf_dynptr_read: unknown dynptr type %d\n", type);
+ return -EFAULT;
+ }
}
static const struct bpf_func_proto bpf_dynptr_read_proto = {
@@ -1529,22 +1550,36 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src,
u32, len, u64, flags)
{
+ enum bpf_dynptr_type type;
int err;
- if (!dst->data || flags || bpf_dynptr_is_rdonly(dst))
+ if (!dst->data || bpf_dynptr_is_rdonly(dst))
return -EINVAL;
err = bpf_dynptr_check_off_len(dst, offset, len);
if (err)
return err;
- /* Source and destination may possibly overlap, hence use memmove to
- * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
- * pointing to overlapping PTR_TO_MAP_VALUE regions.
- */
- memmove(dst->data + dst->offset + offset, src, len);
+ type = bpf_dynptr_get_type(dst);
- return 0;
+ switch (type) {
+ case BPF_DYNPTR_TYPE_LOCAL:
+ case BPF_DYNPTR_TYPE_RINGBUF:
+ if (flags)
+ return -EINVAL;
+ /* Source and destination may possibly overlap, hence use memmove to
+ * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
+ * pointing to overlapping PTR_TO_MAP_VALUE regions.
+ */
+ memmove(dst->data + dst->offset + offset, src, len);
+ return 0;
+ case BPF_DYNPTR_TYPE_SKB:
+ return __bpf_skb_store_bytes(dst->data, dst->offset + offset, src, len,
+ flags);
+ default:
+ WARN_ONCE(true, "bpf_dynptr_write: unknown dynptr type %d\n", type);
+ return -EFAULT;
+ }
}
static const struct bpf_func_proto bpf_dynptr_write_proto = {
@@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
{
+ enum bpf_dynptr_type type;
+ void *data;
int err;
if (!ptr->data)
@@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
if (err)
return 0;
- if (bpf_dynptr_is_rdonly(ptr))
- return 0;
+ type = bpf_dynptr_get_type(ptr);
+
+ switch (type) {
+ case BPF_DYNPTR_TYPE_LOCAL:
+ case BPF_DYNPTR_TYPE_RINGBUF:
+ if (bpf_dynptr_is_rdonly(ptr))
+ return 0;
+
+ data = ptr->data;
+ break;
+ case BPF_DYNPTR_TYPE_SKB:
+ {
+ struct sk_buff *skb = ptr->data;
- return (unsigned long)(ptr->data + ptr->offset + offset);
+ /* if the data is paged, the caller needs to pull it first */
+ if (ptr->offset + offset + len > skb_headlen(skb))
+ return 0;
+
+ /* Depending on the prog type, the data slice will be either
+ * read-writable or read-only. The verifier will enforce that
+ * any writes to read-only data slices are rejected
+ */
+ data = skb->data;
+ break;
+ }
+ default:
+ WARN_ONCE(true, "bpf_dynptr_data: unknown dynptr type %d\n", type);
+ return 0;
+ }
+ return (unsigned long)(data + ptr->offset + offset);
}
static const struct bpf_func_proto bpf_dynptr_data_proto = {
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 853ab671be0b..3b022abc34e3 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -741,6 +741,8 @@ static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
return BPF_DYNPTR_TYPE_LOCAL;
case DYNPTR_TYPE_RINGBUF:
return BPF_DYNPTR_TYPE_RINGBUF;
+ case DYNPTR_TYPE_SKB:
+ return BPF_DYNPTR_TYPE_SKB;
default:
return BPF_DYNPTR_TYPE_INVALID;
}
@@ -1625,6 +1627,12 @@ static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
reg->type == PTR_TO_PACKET_END;
}
+static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
+{
+ return base_type(reg->type) == PTR_TO_MEM &&
+ reg->type & DYNPTR_TYPE_SKB;
+}
+
/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
enum bpf_reg_type which)
@@ -6148,7 +6156,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno,
* type, and declare it as 'const struct bpf_dynptr *' in their prototype.
*/
int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
- enum bpf_arg_type arg_type)
+ enum bpf_arg_type arg_type, int func_id)
{
struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
int err;
@@ -6233,6 +6241,9 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
case DYNPTR_TYPE_RINGBUF:
err_extra = "ringbuf";
break;
+ case DYNPTR_TYPE_SKB:
+ err_extra = "skb ";
+ break;
default:
err_extra = "<unknown>";
break;
@@ -6581,6 +6592,28 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env,
}
}
+static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
+ const struct bpf_func_proto *fn,
+ struct bpf_reg_state *regs)
+{
+ struct bpf_reg_state *state = NULL;
+ int i;
+
+ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
+ if (arg_type_is_dynptr(fn->arg_type[i])) {
+ if (state) {
+ verbose(env, "verifier internal error: multiple dynptr args\n");
+ return NULL;
+ }
+ state = ®s[BPF_REG_1 + i];
+ }
+
+ if (!state)
+ verbose(env, "verifier internal error: no dynptr arg found\n");
+
+ return state;
+}
+
static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
struct bpf_func_state *state = func(env, reg);
@@ -6607,6 +6640,24 @@ static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state
return state->stack[spi].spilled_ptr.ref_obj_id;
}
+static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
+ struct bpf_reg_state *reg)
+{
+ struct bpf_func_state *state = func(env, reg);
+ int spi;
+
+ if (reg->type == CONST_PTR_TO_DYNPTR)
+ return reg->dynptr.type;
+
+ spi = __get_spi(reg->off);
+ if (spi < 0) {
+ verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
+ return BPF_DYNPTR_TYPE_INVALID;
+ }
+
+ return state->stack[spi].spilled_ptr.dynptr.type;
+}
+
static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
struct bpf_call_arg_meta *meta,
const struct bpf_func_proto *fn,
@@ -6819,7 +6870,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
err = check_mem_size_reg(env, reg, regno, true, meta);
break;
case ARG_PTR_TO_DYNPTR:
- err = process_dynptr_func(env, regno, insn_idx, arg_type);
+ err = process_dynptr_func(env, regno, insn_idx, arg_type, meta->func_id);
if (err)
return err;
break;
@@ -7267,6 +7318,9 @@ static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
* are now invalid, so turn them into unknown SCALAR_VALUE.
+ *
+ * This also applies to dynptr slices belonging to skb dynptrs,
+ * since these slices point to packet data.
*/
static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
{
@@ -7274,7 +7328,7 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
struct bpf_reg_state *reg;
bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
- if (reg_is_pkt_pointer_any(reg))
+ if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
__mark_reg_unknown(env, reg);
}));
}
@@ -7958,6 +8012,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
int *insn_idx_p)
{
enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
+ enum bpf_dynptr_type dynptr_type = BPF_DYNPTR_TYPE_INVALID;
const struct bpf_func_proto *fn = NULL;
enum bpf_return_type ret_type;
enum bpf_type_flag ret_flag;
@@ -8140,43 +8195,61 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
}
break;
case BPF_FUNC_dynptr_data:
- for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
- if (arg_type_is_dynptr(fn->arg_type[i])) {
- struct bpf_reg_state *reg = ®s[BPF_REG_1 + i];
- int id, ref_obj_id;
-
- if (meta.dynptr_id) {
- verbose(env, "verifier internal error: meta.dynptr_id already set\n");
- return -EFAULT;
- }
+ {
+ struct bpf_reg_state *reg;
+ int id, ref_obj_id;
- if (meta.ref_obj_id) {
- verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
- return -EFAULT;
- }
+ reg = get_dynptr_arg_reg(env, fn, regs);
+ if (!reg)
+ return -EFAULT;
- id = dynptr_id(env, reg);
- if (id < 0) {
- verbose(env, "verifier internal error: failed to obtain dynptr id\n");
- return id;
- }
+ if (meta.dynptr_id) {
+ verbose(env, "verifier internal error: meta.dynptr_id already set\n");
+ return -EFAULT;
+ }
+ if (meta.ref_obj_id) {
+ verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
+ return -EFAULT;
+ }
- ref_obj_id = dynptr_ref_obj_id(env, reg);
- if (ref_obj_id < 0) {
- verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
- return ref_obj_id;
- }
+ id = dynptr_id(env, reg);
+ if (id < 0) {
+ verbose(env, "verifier internal error: failed to obtain dynptr id\n");
+ return id;
+ }
- meta.dynptr_id = id;
- meta.ref_obj_id = ref_obj_id;
- break;
- }
+ ref_obj_id = dynptr_ref_obj_id(env, reg);
+ if (ref_obj_id < 0) {
+ verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
+ return ref_obj_id;
}
- if (i == MAX_BPF_FUNC_REG_ARGS) {
- verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n");
+
+ meta.dynptr_id = id;
+ meta.ref_obj_id = ref_obj_id;
+
+ dynptr_type = dynptr_get_type(env, reg);
+ if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
return -EFAULT;
- }
+
break;
+ }
+ case BPF_FUNC_dynptr_write:
+ {
+ struct bpf_reg_state *reg;
+
+ reg = get_dynptr_arg_reg(env, fn, regs);
+ if (!reg)
+ return -EFAULT;
+
+ dynptr_type = dynptr_get_type(env, reg);
+ if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
+ return -EFAULT;
+
+ if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
+ changes_data = true;
+
+ break;
+ }
case BPF_FUNC_user_ringbuf_drain:
err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
set_user_ringbuf_callback_state);
@@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
regs[BPF_REG_0].mem_size = meta.mem_size;
+ if (func_id == BPF_FUNC_dynptr_data &&
+ dynptr_type == BPF_DYNPTR_TYPE_SKB) {
+ bool seen_direct_write = env->seen_direct_write;
+
+ regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
+ if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
+ regs[BPF_REG_0].type |= MEM_RDONLY;
+ else
+ /*
+ * Calling may_access_direct_pkt_data() will set
+ * env->seen_direct_write to true if the skb is
+ * writable. As an optimization, we can ignore
+ * setting env->seen_direct_write.
+ *
+ * env->seen_direct_write is used by skb
+ * programs to determine whether the skb's page
+ * buffers should be cloned. Since data slice
+ * writes would only be to the head, we can skip
+ * this.
+ */
+ env->seen_direct_write = seen_direct_write;
+ }
break;
case RET_PTR_TO_MEM_OR_BTF_ID:
{
@@ -8649,6 +8744,7 @@ enum special_kfunc_type {
KF_bpf_list_pop_back,
KF_bpf_cast_to_kern_ctx,
KF_bpf_rdonly_cast,
+ KF_bpf_dynptr_from_skb,
KF_bpf_rcu_read_lock,
KF_bpf_rcu_read_unlock,
};
@@ -8662,6 +8758,7 @@ BTF_ID(func, bpf_list_pop_front)
BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
+BTF_ID(func, bpf_dynptr_from_skb)
BTF_SET_END(special_kfunc_set)
BTF_ID_LIST(special_kfunc_list)
@@ -8673,6 +8770,7 @@ BTF_ID(func, bpf_list_pop_front)
BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
+BTF_ID(func, bpf_dynptr_from_skb)
BTF_ID(func, bpf_rcu_read_lock)
BTF_ID(func, bpf_rcu_read_unlock)
@@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
return ret;
break;
case KF_ARG_PTR_TO_DYNPTR:
+ {
+ enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
+
if (reg->type != PTR_TO_STACK &&
reg->type != CONST_PTR_TO_DYNPTR) {
verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
return -EINVAL;
}
- ret = process_dynptr_func(env, regno, insn_idx,
- ARG_PTR_TO_DYNPTR | MEM_RDONLY);
+ if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
+ dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
+ else
+ dynptr_arg_type |= MEM_RDONLY;
+
+ ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
+ meta->func_id);
if (ret < 0)
return ret;
break;
+ }
case KF_ARG_PTR_TO_LIST_HEAD:
if (reg->type != PTR_TO_MAP_VALUE &&
reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
@@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
*cnt = 1;
+ } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
+ bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
+ struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_4, is_rdonly) };
+
+ insn_buf[0] = addr[0];
+ insn_buf[1] = addr[1];
+ insn_buf[2] = *insn;
+ *cnt = 3;
}
return 0;
}
diff --git a/net/core/filter.c b/net/core/filter.c
index 6da78b3d381e..ddb47126071a 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
}
-BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
- const void *, from, u32, len, u64, flags)
+int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
+ u32 len, u64 flags)
{
void *ptr;
@@ -1710,6 +1710,12 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
return 0;
}
+BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
+ const void *, from, u32, len, u64, flags)
+{
+ return __bpf_skb_store_bytes(skb, offset, from, len, flags);
+}
+
static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
.func = bpf_skb_store_bytes,
.gpl_only = false,
@@ -1721,8 +1727,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
.arg5_type = ARG_ANYTHING,
};
-BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
- void *, to, u32, len)
+int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len)
{
void *ptr;
@@ -1741,6 +1746,12 @@ BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
return -EFAULT;
}
+BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
+ void *, to, u32, len)
+{
+ return __bpf_skb_load_bytes(skb, offset, to, len);
+}
+
static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
.func = bpf_skb_load_bytes,
.gpl_only = false,
@@ -1852,6 +1863,22 @@ static const struct bpf_func_proto bpf_skb_pull_data_proto = {
.arg2_type = ARG_ANYTHING,
};
+int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
+ struct bpf_dynptr_kern *ptr, int is_rdonly)
+{
+ if (flags) {
+ bpf_dynptr_set_null(ptr);
+ return -EINVAL;
+ }
+
+ bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
+
+ if (is_rdonly)
+ bpf_dynptr_set_rdonly(ptr);
+
+ return 0;
+}
+
BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
{
return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
@@ -11607,3 +11634,28 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
return func;
}
+
+BTF_SET8_START(bpf_kfunc_check_set_skb)
+BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
+BTF_SET8_END(bpf_kfunc_check_set_skb)
+
+static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
+ .owner = THIS_MODULE,
+ .set = &bpf_kfunc_check_set_skb,
+};
+
+static int __init bpf_kfunc_init(void)
+{
+ int ret;
+
+ ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
+ return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
+}
+late_initcall(bpf_kfunc_init);
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 7f024ac22edd..6b58e5a75fc5 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5320,22 +5320,45 @@ union bpf_attr {
* Description
* Write *len* bytes from *src* into *dst*, starting from *offset*
* into *dst*.
- * *flags* is currently unused.
+ *
+ * *flags* must be 0 except for skb-type dynptrs.
+ *
+ * For skb-type dynptrs:
+ * * All data slices of the dynptr are automatically
+ * invalidated after **bpf_dynptr_write**\ (). If you wish to
+ * avoid this, please perform the write using direct data slices
+ * instead.
+ *
+ * * For *flags*, please see the flags accepted by
+ * **bpf_skb_store_bytes**\ ().
* Return
* 0 on success, -E2BIG if *offset* + *len* exceeds the length
* of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst*
- * is a read-only dynptr or if *flags* is not 0.
+ * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs,
+ * other errors correspond to errors returned by **bpf_skb_store_bytes**\ ().
*
* void *bpf_dynptr_data(const struct bpf_dynptr *ptr, u32 offset, u32 len)
* Description
* Get a pointer to the underlying dynptr data.
*
* *len* must be a statically known value. The returned data slice
- * is invalidated whenever the dynptr is invalidated.
- * Return
- * Pointer to the underlying dynptr data, NULL if the dynptr is
- * read-only, if the dynptr is invalid, or if the offset and length
- * is out of bounds.
+ * is invalidated whenever the dynptr is invalidated. Please note
+ * that if the dynptr is read-only, then the returned data slice will
+ * be read-only.
+ *
+ * For skb-type dynptrs:
+ * * If *offset* + *len* extends into the skb's paged buffers,
+ * the user should manually pull the skb with **bpf_skb_pull_data**\ ()
+ * and try again.
+ *
+ * * The data slice is automatically invalidated anytime
+ * **bpf_dynptr_write**\ () or a helper call that changes
+ * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
+ * is called.
+ * Return
+ * Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
+ * or if the offset and length is out of bounds or in a paged buffer for
+ * skb-type dynptrs.
*
* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
* Description
--
2.30.2
^ permalink raw reply related [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-27 19:17 ` [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs Joanne Koong
@ 2023-01-29 23:39 ` Alexei Starovoitov
2023-01-31 0:44 ` Joanne Koong
2023-01-30 22:04 ` Martin KaFai Lau
2023-01-31 0:48 ` Andrii Nakryiko
2 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-29 23:39 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> Add skb dynptrs, which are dynptrs whose underlying pointer points
> to a skb. The dynptr acts on skb data. skb dynptrs have two main
> benefits. One is that they allow operations on sizes that are not
> statically known at compile-time (eg variable-sized accesses).
> Another is that parsing the packet data through dynptrs (instead of
> through direct access of skb->data and skb->data_end) can be more
> ergonomic and less brittle (eg does not need manual if checking for
> being within bounds of data_end).
>
> For bpf prog types that don't support writes on skb data, the dynptr is
> read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> will return a data slice that is read-only where any writes to it will
> be rejected by the verifier).
>
> For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> interfaces, reading and writing from/to data in the head as well as from/to
> non-linear paged buffers is supported. For data slices (through the
> bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> must first call bpf_skb_pull_data() to pull the data into the linear
> portion.
Looks like there is an assumption in parts of this patch that
linear part of skb is always writeable. That's not the case.
See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
For TC progs it calls bpf_unclone_prologue() which adds hidden
bpf_skb_pull_data() in the beginning of the prog to make it writeable.
> Any bpf_dynptr_write() automatically invalidates any prior data slices
> to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> to data in a paged buffer, so it will need to pull the buffer first into
> the head. The reason it needs to be pulled instead of writing directly to
> the paged buffers is because they may be cloned (only the head of the skb
> is by default uncloned). As such, any bpf_dynptr_write() will
> automatically have its prior data slices invalidated, even if the write
> is to data in the skb head (the verifier has no way of differentiating
> whether the write is to the head or paged buffers during program load
> time).
Could you explain the workflow how bpf_dynptr_write() invalidates other
pkt pointers ?
I expected bpf_dynptr_write() to be in bpf_helper_changes_pkt_data().
Looks like bpf_dynptr_write() calls bpf_skb_store_bytes() underneath,
but that doesn't help the verifier.
> Please note as well that any other helper calls that change the
> underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> slices of the skb dynptr as well. The stack trace for this is
> check_helper_call() -> clear_all_pkt_pointers() ->
> __clear_all_pkt_pointers() -> mark_reg_unknown().
__clear_all_pkt_pointers isn't present in the tree. Typo ?
>
> For examples of how skb dynptrs can be used, please see the attached
> selftests.
>
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
> include/linux/bpf.h | 82 +++++++++------
> include/linux/filter.h | 18 ++++
> include/uapi/linux/bpf.h | 37 +++++--
> kernel/bpf/btf.c | 18 ++++
> kernel/bpf/helpers.c | 95 ++++++++++++++---
> kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> net/core/filter.c | 60 ++++++++++-
> tools/include/uapi/linux/bpf.h | 37 +++++--
> 8 files changed, 432 insertions(+), 100 deletions(-)
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 14a0264fac57..1ac061b64582 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -575,11 +575,14 @@ enum bpf_type_flag {
> /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
> MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
>
> + /* DYNPTR points to sk_buff */
> + DYNPTR_TYPE_SKB = BIT(14 + BPF_BASE_TYPE_BITS),
> +
> __BPF_TYPE_FLAG_MAX,
> __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
> };
>
> -#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF)
> +#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB)
>
> /* Max number of base types. */
> #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
> @@ -1082,6 +1085,35 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
> return bpf_func(ctx, insnsi);
> }
>
> +/* the implementation of the opaque uapi struct bpf_dynptr */
> +struct bpf_dynptr_kern {
> + void *data;
> + /* Size represents the number of usable bytes of dynptr data.
> + * If for example the offset is at 4 for a local dynptr whose data is
> + * of type u64, the number of usable bytes is 4.
> + *
> + * The upper 8 bits are reserved. It is as follows:
> + * Bits 0 - 23 = size
> + * Bits 24 - 30 = dynptr type
> + * Bit 31 = whether dynptr is read-only
> + */
> + u32 size;
> + u32 offset;
> +} __aligned(8);
> +
> +enum bpf_dynptr_type {
> + BPF_DYNPTR_TYPE_INVALID,
> + /* Points to memory that is local to the bpf program */
> + BPF_DYNPTR_TYPE_LOCAL,
> + /* Underlying data is a ringbuf record */
> + BPF_DYNPTR_TYPE_RINGBUF,
> + /* Underlying data is a sk_buff */
> + BPF_DYNPTR_TYPE_SKB,
> +};
> +
> +int bpf_dynptr_check_size(u32 size);
> +u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr);
> +
> #ifdef CONFIG_BPF_JIT
> int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
> int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
> @@ -2216,6 +2248,11 @@ static inline bool has_current_bpf_ctx(void)
> }
>
> void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
> +
> +void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
> + enum bpf_dynptr_type type, u32 offset, u32 size);
> +void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
> +void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
> #else /* !CONFIG_BPF_SYSCALL */
> static inline struct bpf_prog *bpf_prog_get(u32 ufd)
> {
> @@ -2445,6 +2482,19 @@ static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
> static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
> {
> }
> +
> +static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
> + enum bpf_dynptr_type type, u32 offset, u32 size)
> +{
> +}
> +
> +static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
> +{
> +}
> +
> +static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
> +{
> +}
> #endif /* CONFIG_BPF_SYSCALL */
>
> void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
> @@ -2863,36 +2913,6 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
> u32 num_args, struct bpf_bprintf_data *data);
> void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
>
> -/* the implementation of the opaque uapi struct bpf_dynptr */
> -struct bpf_dynptr_kern {
> - void *data;
> - /* Size represents the number of usable bytes of dynptr data.
> - * If for example the offset is at 4 for a local dynptr whose data is
> - * of type u64, the number of usable bytes is 4.
> - *
> - * The upper 8 bits are reserved. It is as follows:
> - * Bits 0 - 23 = size
> - * Bits 24 - 30 = dynptr type
> - * Bit 31 = whether dynptr is read-only
> - */
> - u32 size;
> - u32 offset;
> -} __aligned(8);
> -
> -enum bpf_dynptr_type {
> - BPF_DYNPTR_TYPE_INVALID,
> - /* Points to memory that is local to the bpf program */
> - BPF_DYNPTR_TYPE_LOCAL,
> - /* Underlying data is a kernel-produced ringbuf record */
> - BPF_DYNPTR_TYPE_RINGBUF,
> -};
> -
> -void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
> - enum bpf_dynptr_type type, u32 offset, u32 size);
> -void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
> -int bpf_dynptr_check_size(u32 size);
> -u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr);
> -
> #ifdef CONFIG_BPF_LSM
> void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
> void bpf_cgroup_atype_put(int cgroup_atype);
> diff --git a/include/linux/filter.h b/include/linux/filter.h
> index ccc4a4a58c72..c87d13954d89 100644
> --- a/include/linux/filter.h
> +++ b/include/linux/filter.h
> @@ -1541,4 +1541,22 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u64 index
> return XDP_REDIRECT;
> }
>
> +#ifdef CONFIG_NET
> +int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
> +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> + u32 len, u64 flags);
> +#else /* CONFIG_NET */
> +static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset,
> + void *to, u32 len)
> +{
> + return -EOPNOTSUPP;
> +}
> +
> +static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset,
> + const void *from, u32 len, u64 flags)
> +{
> + return -EOPNOTSUPP;
> +}
> +#endif /* CONFIG_NET */
> +
> #endif /* __LINUX_FILTER_H__ */
> diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
> index ba0f0cfb5e42..f6910392d339 100644
> --- a/include/uapi/linux/bpf.h
> +++ b/include/uapi/linux/bpf.h
> @@ -5320,22 +5320,45 @@ union bpf_attr {
> * Description
> * Write *len* bytes from *src* into *dst*, starting from *offset*
> * into *dst*.
> - * *flags* is currently unused.
> + *
> + * *flags* must be 0 except for skb-type dynptrs.
> + *
> + * For skb-type dynptrs:
> + * * All data slices of the dynptr are automatically
> + * invalidated after **bpf_dynptr_write**\ (). If you wish to
> + * avoid this, please perform the write using direct data slices
> + * instead.
> + *
> + * * For *flags*, please see the flags accepted by
> + * **bpf_skb_store_bytes**\ ().
> * Return
> * 0 on success, -E2BIG if *offset* + *len* exceeds the length
> * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst*
> - * is a read-only dynptr or if *flags* is not 0.
> + * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs,
> + * other errors correspond to errors returned by **bpf_skb_store_bytes**\ ().
> *
> * void *bpf_dynptr_data(const struct bpf_dynptr *ptr, u32 offset, u32 len)
> * Description
> * Get a pointer to the underlying dynptr data.
> *
> * *len* must be a statically known value. The returned data slice
> - * is invalidated whenever the dynptr is invalidated.
> - * Return
> - * Pointer to the underlying dynptr data, NULL if the dynptr is
> - * read-only, if the dynptr is invalid, or if the offset and length
> - * is out of bounds.
> + * is invalidated whenever the dynptr is invalidated. Please note
> + * that if the dynptr is read-only, then the returned data slice will
> + * be read-only.
> + *
> + * For skb-type dynptrs:
> + * * If *offset* + *len* extends into the skb's paged buffers,
> + * the user should manually pull the skb with **bpf_skb_pull_data**\ ()
> + * and try again.
> + *
> + * * The data slice is automatically invalidated anytime
> + * **bpf_dynptr_write**\ () or a helper call that changes
> + * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
> + * is called.
> + * Return
> + * Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
> + * or if the offset and length is out of bounds or in a paged buffer for
> + * skb-type dynptrs.
> *
> * s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
> * Description
> diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
> index b4da17688c65..35d0780f2eb9 100644
> --- a/kernel/bpf/btf.c
> +++ b/kernel/bpf/btf.c
> @@ -207,6 +207,11 @@ enum btf_kfunc_hook {
> BTF_KFUNC_HOOK_TRACING,
> BTF_KFUNC_HOOK_SYSCALL,
> BTF_KFUNC_HOOK_FMODRET,
> + BTF_KFUNC_HOOK_CGROUP_SKB,
> + BTF_KFUNC_HOOK_SCHED_ACT,
> + BTF_KFUNC_HOOK_SK_SKB,
> + BTF_KFUNC_HOOK_SOCKET_FILTER,
> + BTF_KFUNC_HOOK_LWT,
> BTF_KFUNC_HOOK_MAX,
> };
>
> @@ -7609,6 +7614,19 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type)
> return BTF_KFUNC_HOOK_TRACING;
> case BPF_PROG_TYPE_SYSCALL:
> return BTF_KFUNC_HOOK_SYSCALL;
> + case BPF_PROG_TYPE_CGROUP_SKB:
> + return BTF_KFUNC_HOOK_CGROUP_SKB;
> + case BPF_PROG_TYPE_SCHED_ACT:
> + return BTF_KFUNC_HOOK_SCHED_ACT;
> + case BPF_PROG_TYPE_SK_SKB:
> + return BTF_KFUNC_HOOK_SK_SKB;
> + case BPF_PROG_TYPE_SOCKET_FILTER:
> + return BTF_KFUNC_HOOK_SOCKET_FILTER;
> + case BPF_PROG_TYPE_LWT_OUT:
> + case BPF_PROG_TYPE_LWT_IN:
> + case BPF_PROG_TYPE_LWT_XMIT:
> + case BPF_PROG_TYPE_LWT_SEG6LOCAL:
> + return BTF_KFUNC_HOOK_LWT;
> default:
> return BTF_KFUNC_HOOK_MAX;
> }
> diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
> index 458db2db2f81..a79d522b3a26 100644
> --- a/kernel/bpf/helpers.c
> +++ b/kernel/bpf/helpers.c
> @@ -1420,11 +1420,21 @@ static bool bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr)
> return ptr->size & DYNPTR_RDONLY_BIT;
> }
>
> +void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
> +{
> + ptr->size |= DYNPTR_RDONLY_BIT;
> +}
> +
> static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_type type)
> {
> ptr->size |= type << DYNPTR_TYPE_SHIFT;
> }
>
> +static enum bpf_dynptr_type bpf_dynptr_get_type(const struct bpf_dynptr_kern *ptr)
> +{
> + return (ptr->size & ~(DYNPTR_RDONLY_BIT)) >> DYNPTR_TYPE_SHIFT;
> +}
> +
> u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr)
> {
> return ptr->size & DYNPTR_SIZE_MASK;
> @@ -1497,6 +1507,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
> BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern *, src,
> u32, offset, u64, flags)
> {
> + enum bpf_dynptr_type type;
> int err;
>
> if (!src->data || flags)
> @@ -1506,13 +1517,23 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern
> if (err)
> return err;
>
> - /* Source and destination may possibly overlap, hence use memmove to
> - * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
> - * pointing to overlapping PTR_TO_MAP_VALUE regions.
> - */
> - memmove(dst, src->data + src->offset + offset, len);
> + type = bpf_dynptr_get_type(src);
>
> - return 0;
> + switch (type) {
> + case BPF_DYNPTR_TYPE_LOCAL:
> + case BPF_DYNPTR_TYPE_RINGBUF:
> + /* Source and destination may possibly overlap, hence use memmove to
> + * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
> + * pointing to overlapping PTR_TO_MAP_VALUE regions.
> + */
> + memmove(dst, src->data + src->offset + offset, len);
> + return 0;
> + case BPF_DYNPTR_TYPE_SKB:
> + return __bpf_skb_load_bytes(src->data, src->offset + offset, dst, len);
> + default:
> + WARN_ONCE(true, "bpf_dynptr_read: unknown dynptr type %d\n", type);
> + return -EFAULT;
> + }
> }
>
> static const struct bpf_func_proto bpf_dynptr_read_proto = {
> @@ -1529,22 +1550,36 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
> BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, void *, src,
> u32, len, u64, flags)
> {
> + enum bpf_dynptr_type type;
> int err;
>
> - if (!dst->data || flags || bpf_dynptr_is_rdonly(dst))
> + if (!dst->data || bpf_dynptr_is_rdonly(dst))
> return -EINVAL;
>
> err = bpf_dynptr_check_off_len(dst, offset, len);
> if (err)
> return err;
>
> - /* Source and destination may possibly overlap, hence use memmove to
> - * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
> - * pointing to overlapping PTR_TO_MAP_VALUE regions.
> - */
> - memmove(dst->data + dst->offset + offset, src, len);
> + type = bpf_dynptr_get_type(dst);
>
> - return 0;
> + switch (type) {
> + case BPF_DYNPTR_TYPE_LOCAL:
> + case BPF_DYNPTR_TYPE_RINGBUF:
> + if (flags)
> + return -EINVAL;
> + /* Source and destination may possibly overlap, hence use memmove to
> + * copy the data. E.g. bpf_dynptr_from_mem may create two dynptr
> + * pointing to overlapping PTR_TO_MAP_VALUE regions.
> + */
> + memmove(dst->data + dst->offset + offset, src, len);
> + return 0;
> + case BPF_DYNPTR_TYPE_SKB:
> + return __bpf_skb_store_bytes(dst->data, dst->offset + offset, src, len,
> + flags);
> + default:
> + WARN_ONCE(true, "bpf_dynptr_write: unknown dynptr type %d\n", type);
> + return -EFAULT;
> + }
> }
>
> static const struct bpf_func_proto bpf_dynptr_write_proto = {
> @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
>
> BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> {
> + enum bpf_dynptr_type type;
> + void *data;
> int err;
>
> if (!ptr->data)
> @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> if (err)
> return 0;
>
> - if (bpf_dynptr_is_rdonly(ptr))
> - return 0;
> + type = bpf_dynptr_get_type(ptr);
> +
> + switch (type) {
> + case BPF_DYNPTR_TYPE_LOCAL:
> + case BPF_DYNPTR_TYPE_RINGBUF:
> + if (bpf_dynptr_is_rdonly(ptr))
> + return 0;
> +
> + data = ptr->data;
> + break;
> + case BPF_DYNPTR_TYPE_SKB:
> + {
> + struct sk_buff *skb = ptr->data;
>
> - return (unsigned long)(ptr->data + ptr->offset + offset);
> + /* if the data is paged, the caller needs to pull it first */
> + if (ptr->offset + offset + len > skb_headlen(skb))
> + return 0;
> +
> + /* Depending on the prog type, the data slice will be either
> + * read-writable or read-only. The verifier will enforce that
> + * any writes to read-only data slices are rejected
> + */
> + data = skb->data;
> + break;
> + }
> + default:
> + WARN_ONCE(true, "bpf_dynptr_data: unknown dynptr type %d\n", type);
> + return 0;
> + }
> + return (unsigned long)(data + ptr->offset + offset);
> }
>
> static const struct bpf_func_proto bpf_dynptr_data_proto = {
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 853ab671be0b..3b022abc34e3 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -741,6 +741,8 @@ static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
> return BPF_DYNPTR_TYPE_LOCAL;
> case DYNPTR_TYPE_RINGBUF:
> return BPF_DYNPTR_TYPE_RINGBUF;
> + case DYNPTR_TYPE_SKB:
> + return BPF_DYNPTR_TYPE_SKB;
> default:
> return BPF_DYNPTR_TYPE_INVALID;
> }
> @@ -1625,6 +1627,12 @@ static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
> reg->type == PTR_TO_PACKET_END;
> }
>
> +static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
> +{
> + return base_type(reg->type) == PTR_TO_MEM &&
> + reg->type & DYNPTR_TYPE_SKB;
> +}
> +
> /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
> static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
> enum bpf_reg_type which)
> @@ -6148,7 +6156,7 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno,
> * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
> */
> int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
> - enum bpf_arg_type arg_type)
> + enum bpf_arg_type arg_type, int func_id)
> {
> struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
> int err;
> @@ -6233,6 +6241,9 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
> case DYNPTR_TYPE_RINGBUF:
> err_extra = "ringbuf";
> break;
> + case DYNPTR_TYPE_SKB:
> + err_extra = "skb ";
> + break;
> default:
> err_extra = "<unknown>";
> break;
> @@ -6581,6 +6592,28 @@ int check_func_arg_reg_off(struct bpf_verifier_env *env,
> }
> }
>
> +static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
> + const struct bpf_func_proto *fn,
> + struct bpf_reg_state *regs)
> +{
> + struct bpf_reg_state *state = NULL;
> + int i;
> +
> + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
> + if (arg_type_is_dynptr(fn->arg_type[i])) {
> + if (state) {
> + verbose(env, "verifier internal error: multiple dynptr args\n");
> + return NULL;
> + }
> + state = ®s[BPF_REG_1 + i];
> + }
> +
> + if (!state)
> + verbose(env, "verifier internal error: no dynptr arg found\n");
> +
> + return state;
> +}
Looks like refactoring is mixed with new features.
Moving struct bpf_dynptr_kern to a different place and factoring out get_dynptr_arg_reg()
could have been a separate patch to make it easier to review.
> +
> static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
> {
> struct bpf_func_state *state = func(env, reg);
> @@ -6607,6 +6640,24 @@ static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state
> return state->stack[spi].spilled_ptr.ref_obj_id;
> }
>
> +static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
> + struct bpf_reg_state *reg)
> +{
> + struct bpf_func_state *state = func(env, reg);
> + int spi;
> +
> + if (reg->type == CONST_PTR_TO_DYNPTR)
> + return reg->dynptr.type;
> +
> + spi = __get_spi(reg->off);
> + if (spi < 0) {
> + verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
> + return BPF_DYNPTR_TYPE_INVALID;
> + }
> +
> + return state->stack[spi].spilled_ptr.dynptr.type;
> +}
> +
> static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
> struct bpf_call_arg_meta *meta,
> const struct bpf_func_proto *fn,
> @@ -6819,7 +6870,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
> err = check_mem_size_reg(env, reg, regno, true, meta);
> break;
> case ARG_PTR_TO_DYNPTR:
> - err = process_dynptr_func(env, regno, insn_idx, arg_type);
> + err = process_dynptr_func(env, regno, insn_idx, arg_type, meta->func_id);
> if (err)
> return err;
> break;
> @@ -7267,6 +7318,9 @@ static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
>
> /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
> * are now invalid, so turn them into unknown SCALAR_VALUE.
> + *
> + * This also applies to dynptr slices belonging to skb dynptrs,
> + * since these slices point to packet data.
> */
> static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
> {
> @@ -7274,7 +7328,7 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
> struct bpf_reg_state *reg;
>
> bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
> - if (reg_is_pkt_pointer_any(reg))
> + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
> __mark_reg_unknown(env, reg);
> }));
> }
> @@ -7958,6 +8012,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> int *insn_idx_p)
> {
> enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
> + enum bpf_dynptr_type dynptr_type = BPF_DYNPTR_TYPE_INVALID;
> const struct bpf_func_proto *fn = NULL;
> enum bpf_return_type ret_type;
> enum bpf_type_flag ret_flag;
> @@ -8140,43 +8195,61 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> }
> break;
> case BPF_FUNC_dynptr_data:
> - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
> - if (arg_type_is_dynptr(fn->arg_type[i])) {
> - struct bpf_reg_state *reg = ®s[BPF_REG_1 + i];
> - int id, ref_obj_id;
> -
> - if (meta.dynptr_id) {
> - verbose(env, "verifier internal error: meta.dynptr_id already set\n");
> - return -EFAULT;
> - }
> + {
> + struct bpf_reg_state *reg;
> + int id, ref_obj_id;
>
> - if (meta.ref_obj_id) {
> - verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
> - return -EFAULT;
> - }
> + reg = get_dynptr_arg_reg(env, fn, regs);
> + if (!reg)
> + return -EFAULT;
>
> - id = dynptr_id(env, reg);
> - if (id < 0) {
> - verbose(env, "verifier internal error: failed to obtain dynptr id\n");
> - return id;
> - }
> + if (meta.dynptr_id) {
> + verbose(env, "verifier internal error: meta.dynptr_id already set\n");
> + return -EFAULT;
> + }
> + if (meta.ref_obj_id) {
> + verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
> + return -EFAULT;
> + }
>
> - ref_obj_id = dynptr_ref_obj_id(env, reg);
> - if (ref_obj_id < 0) {
> - verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
> - return ref_obj_id;
> - }
> + id = dynptr_id(env, reg);
> + if (id < 0) {
> + verbose(env, "verifier internal error: failed to obtain dynptr id\n");
> + return id;
> + }
>
> - meta.dynptr_id = id;
> - meta.ref_obj_id = ref_obj_id;
> - break;
> - }
> + ref_obj_id = dynptr_ref_obj_id(env, reg);
> + if (ref_obj_id < 0) {
> + verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
> + return ref_obj_id;
> }
> - if (i == MAX_BPF_FUNC_REG_ARGS) {
> - verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n");
> +
> + meta.dynptr_id = id;
> + meta.ref_obj_id = ref_obj_id;
> +
> + dynptr_type = dynptr_get_type(env, reg);
> + if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
> return -EFAULT;
> - }
> +
> break;
> + }
> + case BPF_FUNC_dynptr_write:
> + {
> + struct bpf_reg_state *reg;
> +
> + reg = get_dynptr_arg_reg(env, fn, regs);
> + if (!reg)
> + return -EFAULT;
> +
> + dynptr_type = dynptr_get_type(env, reg);
> + if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
> + return -EFAULT;
> +
> + if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
> + changes_data = true;
> +
> + break;
> + }
> case BPF_FUNC_user_ringbuf_drain:
> err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
> set_user_ringbuf_callback_state);
> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> mark_reg_known_zero(env, regs, BPF_REG_0);
> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> regs[BPF_REG_0].mem_size = meta.mem_size;
> + if (func_id == BPF_FUNC_dynptr_data &&
> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> + bool seen_direct_write = env->seen_direct_write;
> +
> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> + regs[BPF_REG_0].type |= MEM_RDONLY;
> + else
> + /*
> + * Calling may_access_direct_pkt_data() will set
> + * env->seen_direct_write to true if the skb is
> + * writable. As an optimization, we can ignore
> + * setting env->seen_direct_write.
> + *
> + * env->seen_direct_write is used by skb
> + * programs to determine whether the skb's page
> + * buffers should be cloned. Since data slice
> + * writes would only be to the head, we can skip
> + * this.
> + */
> + env->seen_direct_write = seen_direct_write;
This looks incorrect. skb head might not be writeable.
> + }
> break;
> case RET_PTR_TO_MEM_OR_BTF_ID:
> {
> @@ -8649,6 +8744,7 @@ enum special_kfunc_type {
> KF_bpf_list_pop_back,
> KF_bpf_cast_to_kern_ctx,
> KF_bpf_rdonly_cast,
> + KF_bpf_dynptr_from_skb,
> KF_bpf_rcu_read_lock,
> KF_bpf_rcu_read_unlock,
> };
> @@ -8662,6 +8758,7 @@ BTF_ID(func, bpf_list_pop_front)
> BTF_ID(func, bpf_list_pop_back)
> BTF_ID(func, bpf_cast_to_kern_ctx)
> BTF_ID(func, bpf_rdonly_cast)
> +BTF_ID(func, bpf_dynptr_from_skb)
> BTF_SET_END(special_kfunc_set)
>
> BTF_ID_LIST(special_kfunc_list)
> @@ -8673,6 +8770,7 @@ BTF_ID(func, bpf_list_pop_front)
> BTF_ID(func, bpf_list_pop_back)
> BTF_ID(func, bpf_cast_to_kern_ctx)
> BTF_ID(func, bpf_rdonly_cast)
> +BTF_ID(func, bpf_dynptr_from_skb)
> BTF_ID(func, bpf_rcu_read_lock)
> BTF_ID(func, bpf_rcu_read_unlock)
>
> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> return ret;
> break;
> case KF_ARG_PTR_TO_DYNPTR:
> + {
> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> +
> if (reg->type != PTR_TO_STACK &&
> reg->type != CONST_PTR_TO_DYNPTR) {
> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> return -EINVAL;
> }
>
> - ret = process_dynptr_func(env, regno, insn_idx,
> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> + else
> + dynptr_arg_type |= MEM_RDONLY;
> +
> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> + meta->func_id);
> if (ret < 0)
> return ret;
> break;
> + }
> case KF_ARG_PTR_TO_LIST_HEAD:
> if (reg->type != PTR_TO_MAP_VALUE &&
> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> *cnt = 1;
> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> + struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_4, is_rdonly) };
Why use 16-byte insn to pass boolean in R4 ?
Single 8-byte MOV would do.
> +
> + insn_buf[0] = addr[0];
> + insn_buf[1] = addr[1];
> + insn_buf[2] = *insn;
> + *cnt = 3;
> }
> return 0;
> }
> diff --git a/net/core/filter.c b/net/core/filter.c
> index 6da78b3d381e..ddb47126071a 100644
> --- a/net/core/filter.c
> +++ b/net/core/filter.c
> @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> }
>
> -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> - const void *, from, u32, len, u64, flags)
> +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> + u32 len, u64 flags)
This change is just to be able to call __bpf_skb_store_bytes() ?
If so, it's unnecessary.
See:
BPF_CALL_4(sk_reuseport_load_bytes,
const struct sk_reuseport_kern *, reuse_kern, u32, offset,
void *, to, u32, len)
{
return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
}
> {
> void *ptr;
>
> @@ -1710,6 +1710,12 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> return 0;
> }
>
> +BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> + const void *, from, u32, len, u64, flags)
> +{
> + return __bpf_skb_store_bytes(skb, offset, from, len, flags);
> +}
> +
> static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
> .func = bpf_skb_store_bytes,
> .gpl_only = false,
> @@ -1721,8 +1727,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
> .arg5_type = ARG_ANYTHING,
> };
>
> -BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
> - void *, to, u32, len)
> +int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len)
> {
> void *ptr;
>
> @@ -1741,6 +1746,12 @@ BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
> return -EFAULT;
> }
>
> +BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
> + void *, to, u32, len)
> +{
> + return __bpf_skb_load_bytes(skb, offset, to, len);
> +}
> +
> static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
> .func = bpf_skb_load_bytes,
> .gpl_only = false,
> @@ -1852,6 +1863,22 @@ static const struct bpf_func_proto bpf_skb_pull_data_proto = {
> .arg2_type = ARG_ANYTHING,
> };
>
> +int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
> + struct bpf_dynptr_kern *ptr, int is_rdonly)
It probably needs
__diag_ignore_all("-Wmissing-prototypes",
like other kfuncs to suppress build warn.
> +{
> + if (flags) {
> + bpf_dynptr_set_null(ptr);
> + return -EINVAL;
> + }
> +
> + bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
> +
> + if (is_rdonly)
> + bpf_dynptr_set_rdonly(ptr);
> +
> + return 0;
> +}
> +
> BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
> {
> return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
> @@ -11607,3 +11634,28 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
>
> return func;
> }
> +
> +BTF_SET8_START(bpf_kfunc_check_set_skb)
> +BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
> +BTF_SET8_END(bpf_kfunc_check_set_skb)
> +
> +static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
> + .owner = THIS_MODULE,
> + .set = &bpf_kfunc_check_set_skb,
> +};
> +
> +static int __init bpf_kfunc_init(void)
> +{
> + int ret;
> +
> + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
> + return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
> +}
> +late_initcall(bpf_kfunc_init);
> diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
> index 7f024ac22edd..6b58e5a75fc5 100644
> --- a/tools/include/uapi/linux/bpf.h
> +++ b/tools/include/uapi/linux/bpf.h
> @@ -5320,22 +5320,45 @@ union bpf_attr {
> * Description
> * Write *len* bytes from *src* into *dst*, starting from *offset*
> * into *dst*.
> - * *flags* is currently unused.
> + *
> + * *flags* must be 0 except for skb-type dynptrs.
> + *
> + * For skb-type dynptrs:
> + * * All data slices of the dynptr are automatically
> + * invalidated after **bpf_dynptr_write**\ (). If you wish to
> + * avoid this, please perform the write using direct data slices
> + * instead.
> + *
> + * * For *flags*, please see the flags accepted by
> + * **bpf_skb_store_bytes**\ ().
> * Return
> * 0 on success, -E2BIG if *offset* + *len* exceeds the length
> * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst*
> - * is a read-only dynptr or if *flags* is not 0.
> + * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs,
> + * other errors correspond to errors returned by **bpf_skb_store_bytes**\ ().
> *
> * void *bpf_dynptr_data(const struct bpf_dynptr *ptr, u32 offset, u32 len)
> * Description
> * Get a pointer to the underlying dynptr data.
> *
> * *len* must be a statically known value. The returned data slice
> - * is invalidated whenever the dynptr is invalidated.
> - * Return
> - * Pointer to the underlying dynptr data, NULL if the dynptr is
> - * read-only, if the dynptr is invalid, or if the offset and length
> - * is out of bounds.
> + * is invalidated whenever the dynptr is invalidated. Please note
> + * that if the dynptr is read-only, then the returned data slice will
> + * be read-only.
> + *
> + * For skb-type dynptrs:
> + * * If *offset* + *len* extends into the skb's paged buffers,
> + * the user should manually pull the skb with **bpf_skb_pull_data**\ ()
> + * and try again.
> + *
> + * * The data slice is automatically invalidated anytime
> + * **bpf_dynptr_write**\ () or a helper call that changes
> + * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
> + * is called.
> + * Return
> + * Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
> + * or if the offset and length is out of bounds or in a paged buffer for
> + * skb-type dynptrs.
> *
> * s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
> * Description
> --
> 2.30.2
>
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-29 23:39 ` Alexei Starovoitov
@ 2023-01-31 0:44 ` Joanne Koong
2023-01-31 5:36 ` Alexei Starovoitov
0 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 0:44 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > benefits. One is that they allow operations on sizes that are not
> > statically known at compile-time (eg variable-sized accesses).
> > Another is that parsing the packet data through dynptrs (instead of
> > through direct access of skb->data and skb->data_end) can be more
> > ergonomic and less brittle (eg does not need manual if checking for
> > being within bounds of data_end).
> >
> > For bpf prog types that don't support writes on skb data, the dynptr is
> > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > will return a data slice that is read-only where any writes to it will
> > be rejected by the verifier).
> >
> > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > interfaces, reading and writing from/to data in the head as well as from/to
> > non-linear paged buffers is supported. For data slices (through the
> > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > must first call bpf_skb_pull_data() to pull the data into the linear
> > portion.
>
> Looks like there is an assumption in parts of this patch that
> linear part of skb is always writeable. That's not the case.
> See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
> For TC progs it calls bpf_unclone_prologue() which adds hidden
> bpf_skb_pull_data() in the beginning of the prog to make it writeable.
I think we can make this assumption? For writable progs (referenced in
the may_access_direct_pkt_data() function), all of them have a
gen_prologue that unclones the buffer (eg tc_cls_act, lwt_xmit, sk_skb
progs) or their linear portion is okay to write into by default (eg
xdp, sk_msg, cg_sockopt progs).
>
> > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > to data in a paged buffer, so it will need to pull the buffer first into
> > the head. The reason it needs to be pulled instead of writing directly to
> > the paged buffers is because they may be cloned (only the head of the skb
> > is by default uncloned). As such, any bpf_dynptr_write() will
> > automatically have its prior data slices invalidated, even if the write
> > is to data in the skb head (the verifier has no way of differentiating
> > whether the write is to the head or paged buffers during program load
> > time).
>
> Could you explain the workflow how bpf_dynptr_write() invalidates other
> pkt pointers ?
> I expected bpf_dynptr_write() to be in bpf_helper_changes_pkt_data().
> Looks like bpf_dynptr_write() calls bpf_skb_store_bytes() underneath,
> but that doesn't help the verifier.
In the verifier in check_helper_call(), for the BPF_FUNC_dynptr_write
case (line 8236) the "changes_data" variable gets set to true if the
dynptr is an skb type. At the end of check_helper_call() on line 8474,
since "changes_data" is true, clear_all_pkt_pointer() gets called,
which invalidates the other packet pointers.
>
> > Please note as well that any other helper calls that change the
> > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > slices of the skb dynptr as well. The stack trace for this is
> > check_helper_call() -> clear_all_pkt_pointers() ->
> > __clear_all_pkt_pointers() -> mark_reg_unknown().
>
> __clear_all_pkt_pointers isn't present in the tree. Typo ?
I'll update this message, clear_all_pkt_pointers() and
__clear_all_pkt_pointers() were combined in a previous commit.
>
> >
> > For examples of how skb dynptrs can be used, please see the attached
> > selftests.
> >
> > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > ---
> > include/linux/bpf.h | 82 +++++++++------
> > include/linux/filter.h | 18 ++++
> > include/uapi/linux/bpf.h | 37 +++++--
> > kernel/bpf/btf.c | 18 ++++
> > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > net/core/filter.c | 60 ++++++++++-
> > tools/include/uapi/linux/bpf.h | 37 +++++--
> > 8 files changed, 432 insertions(+), 100 deletions(-)
> >
> > diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> > index 14a0264fac57..1ac061b64582 100644
[...]
> > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > mark_reg_known_zero(env, regs, BPF_REG_0);
> > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > regs[BPF_REG_0].mem_size = meta.mem_size;
> > + if (func_id == BPF_FUNC_dynptr_data &&
> > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > + bool seen_direct_write = env->seen_direct_write;
> > +
> > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > + else
> > + /*
> > + * Calling may_access_direct_pkt_data() will set
> > + * env->seen_direct_write to true if the skb is
> > + * writable. As an optimization, we can ignore
> > + * setting env->seen_direct_write.
> > + *
> > + * env->seen_direct_write is used by skb
> > + * programs to determine whether the skb's page
> > + * buffers should be cloned. Since data slice
> > + * writes would only be to the head, we can skip
> > + * this.
> > + */
> > + env->seen_direct_write = seen_direct_write;
>
> This looks incorrect. skb head might not be writeable.
>
> > + }
> > break;
> > case RET_PTR_TO_MEM_OR_BTF_ID:
> > {
> > @@ -8649,6 +8744,7 @@ enum special_kfunc_type {
> > KF_bpf_list_pop_back,
> > KF_bpf_cast_to_kern_ctx,
> > KF_bpf_rdonly_cast,
> > + KF_bpf_dynptr_from_skb,
> > KF_bpf_rcu_read_lock,
> > KF_bpf_rcu_read_unlock,
> > };
> > @@ -8662,6 +8758,7 @@ BTF_ID(func, bpf_list_pop_front)
> > BTF_ID(func, bpf_list_pop_back)
> > BTF_ID(func, bpf_cast_to_kern_ctx)
> > BTF_ID(func, bpf_rdonly_cast)
> > +BTF_ID(func, bpf_dynptr_from_skb)
> > BTF_SET_END(special_kfunc_set)
> >
> > BTF_ID_LIST(special_kfunc_list)
> > @@ -8673,6 +8770,7 @@ BTF_ID(func, bpf_list_pop_front)
> > BTF_ID(func, bpf_list_pop_back)
> > BTF_ID(func, bpf_cast_to_kern_ctx)
> > BTF_ID(func, bpf_rdonly_cast)
> > +BTF_ID(func, bpf_dynptr_from_skb)
> > BTF_ID(func, bpf_rcu_read_lock)
> > BTF_ID(func, bpf_rcu_read_unlock)
> >
> > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > return ret;
> > break;
> > case KF_ARG_PTR_TO_DYNPTR:
> > + {
> > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > +
> > if (reg->type != PTR_TO_STACK &&
> > reg->type != CONST_PTR_TO_DYNPTR) {
> > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > return -EINVAL;
> > }
> >
> > - ret = process_dynptr_func(env, regno, insn_idx,
> > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > + else
> > + dynptr_arg_type |= MEM_RDONLY;
> > +
> > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > + meta->func_id);
> > if (ret < 0)
> > return ret;
> > break;
> > + }
> > case KF_ARG_PTR_TO_LIST_HEAD:
> > if (reg->type != PTR_TO_MAP_VALUE &&
> > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > *cnt = 1;
> > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > + struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_4, is_rdonly) };
>
> Why use 16-byte insn to pass boolean in R4 ?
> Single 8-byte MOV would do.
Great, I'll change it to a 8-byte MOV
>
> > +
> > + insn_buf[0] = addr[0];
> > + insn_buf[1] = addr[1];
> > + insn_buf[2] = *insn;
> > + *cnt = 3;
> > }
> > return 0;
> > }
> > diff --git a/net/core/filter.c b/net/core/filter.c
> > index 6da78b3d381e..ddb47126071a 100644
> > --- a/net/core/filter.c
> > +++ b/net/core/filter.c
> > @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> > skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> > }
> >
> > -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > - const void *, from, u32, len, u64, flags)
> > +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> > + u32 len, u64 flags)
>
> This change is just to be able to call __bpf_skb_store_bytes() ?
> If so, it's unnecessary.
> See:
> BPF_CALL_4(sk_reuseport_load_bytes,
> const struct sk_reuseport_kern *, reuse_kern, u32, offset,
> void *, to, u32, len)
> {
> return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
> }
>
There was prior feedback [0] that using four underscores to call a
helper function is confusing and makes it ungreppable
[0] https://lore.kernel.org/bpf/CAEf4Bzaz4=tEvESd_twhx1bdepdOP3L4SmUiaKqGFJtX=CJruQ@mail.gmail.com/
> > {
> > void *ptr;
> >
> > @@ -1710,6 +1710,12 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > return 0;
> > }
> >
> > +BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > + const void *, from, u32, len, u64, flags)
> > +{
> > + return __bpf_skb_store_bytes(skb, offset, from, len, flags);
> > +}
> > +
[...]
> > @@ -1852,6 +1863,22 @@ static const struct bpf_func_proto bpf_skb_pull_data_proto = {
> > .arg2_type = ARG_ANYTHING,
> > };
> >
> > +int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
> > + struct bpf_dynptr_kern *ptr, int is_rdonly)
>
> It probably needs
> __diag_ignore_all("-Wmissing-prototypes",
> like other kfuncs to suppress build warn.
>
Awesome, thanks. I'll add this in.
> > +{
> > + if (flags) {
> > + bpf_dynptr_set_null(ptr);
> > + return -EINVAL;
> > + }
> > +
> > + bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
> > +
> > + if (is_rdonly)
> > + bpf_dynptr_set_rdonly(ptr);
> > +
> > + return 0;
> > +}
> > +
> > BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
[...]
> > --
> > 2.30.2
> >
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 0:44 ` Joanne Koong
@ 2023-01-31 5:36 ` Alexei Starovoitov
2023-01-31 17:54 ` Joanne Koong
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-31 5:36 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > benefits. One is that they allow operations on sizes that are not
> > > statically known at compile-time (eg variable-sized accesses).
> > > Another is that parsing the packet data through dynptrs (instead of
> > > through direct access of skb->data and skb->data_end) can be more
> > > ergonomic and less brittle (eg does not need manual if checking for
> > > being within bounds of data_end).
> > >
> > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > will return a data slice that is read-only where any writes to it will
> > > be rejected by the verifier).
> > >
> > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > interfaces, reading and writing from/to data in the head as well as from/to
> > > non-linear paged buffers is supported. For data slices (through the
> > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > portion.
> >
> > Looks like there is an assumption in parts of this patch that
> > linear part of skb is always writeable. That's not the case.
> > See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
> > For TC progs it calls bpf_unclone_prologue() which adds hidden
> > bpf_skb_pull_data() in the beginning of the prog to make it writeable.
>
> I think we can make this assumption? For writable progs (referenced in
> the may_access_direct_pkt_data() function), all of them have a
> gen_prologue that unclones the buffer (eg tc_cls_act, lwt_xmit, sk_skb
> progs) or their linear portion is okay to write into by default (eg
> xdp, sk_msg, cg_sockopt progs).
but the patch was preserving seen_direct_write in some cases.
I'm still confused.
> >
> > > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > > to data in a paged buffer, so it will need to pull the buffer first into
> > > the head. The reason it needs to be pulled instead of writing directly to
> > > the paged buffers is because they may be cloned (only the head of the skb
> > > is by default uncloned). As such, any bpf_dynptr_write() will
> > > automatically have its prior data slices invalidated, even if the write
> > > is to data in the skb head (the verifier has no way of differentiating
> > > whether the write is to the head or paged buffers during program load
> > > time).
> >
> > Could you explain the workflow how bpf_dynptr_write() invalidates other
> > pkt pointers ?
> > I expected bpf_dynptr_write() to be in bpf_helper_changes_pkt_data().
> > Looks like bpf_dynptr_write() calls bpf_skb_store_bytes() underneath,
> > but that doesn't help the verifier.
>
> In the verifier in check_helper_call(), for the BPF_FUNC_dynptr_write
> case (line 8236) the "changes_data" variable gets set to true if the
> dynptr is an skb type. At the end of check_helper_call() on line 8474,
> since "changes_data" is true, clear_all_pkt_pointer() gets called,
> which invalidates the other packet pointers.
Ahh. I see. Thanks for explaining.
> >
> > > Please note as well that any other helper calls that change the
> > > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > > slices of the skb dynptr as well. The stack trace for this is
> > > check_helper_call() -> clear_all_pkt_pointers() ->
> > > __clear_all_pkt_pointers() -> mark_reg_unknown().
> >
> > __clear_all_pkt_pointers isn't present in the tree. Typo ?
>
> I'll update this message, clear_all_pkt_pointers() and
> __clear_all_pkt_pointers() were combined in a previous commit.
>
> >
> > >
> > > For examples of how skb dynptrs can be used, please see the attached
> > > selftests.
> > >
> > > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > > ---
> > > include/linux/bpf.h | 82 +++++++++------
> > > include/linux/filter.h | 18 ++++
> > > include/uapi/linux/bpf.h | 37 +++++--
> > > kernel/bpf/btf.c | 18 ++++
> > > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > > net/core/filter.c | 60 ++++++++++-
> > > tools/include/uapi/linux/bpf.h | 37 +++++--
> > > 8 files changed, 432 insertions(+), 100 deletions(-)
> > >
> > > diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> > > index 14a0264fac57..1ac061b64582 100644
> [...]
> > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > + bool seen_direct_write = env->seen_direct_write;
> > > +
> > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > + else
> > > + /*
> > > + * Calling may_access_direct_pkt_data() will set
> > > + * env->seen_direct_write to true if the skb is
> > > + * writable. As an optimization, we can ignore
> > > + * setting env->seen_direct_write.
> > > + *
> > > + * env->seen_direct_write is used by skb
> > > + * programs to determine whether the skb's page
> > > + * buffers should be cloned. Since data slice
> > > + * writes would only be to the head, we can skip
> > > + * this.
I was talking about above comment. It reads as 'write to the head are allowed'.
But they're not. seen_direct_write is needed to do hidden pull.
> > > + */
> > > + env->seen_direct_write = seen_direct_write;
> >
> > This looks incorrect. skb head might not be writeable.
> >
> > > + }
> > > break;
> > > case RET_PTR_TO_MEM_OR_BTF_ID:
> > > {
> > > @@ -8649,6 +8744,7 @@ enum special_kfunc_type {
> > > KF_bpf_list_pop_back,
> > > KF_bpf_cast_to_kern_ctx,
> > > KF_bpf_rdonly_cast,
> > > + KF_bpf_dynptr_from_skb,
> > > KF_bpf_rcu_read_lock,
> > > KF_bpf_rcu_read_unlock,
> > > };
> > > @@ -8662,6 +8758,7 @@ BTF_ID(func, bpf_list_pop_front)
> > > BTF_ID(func, bpf_list_pop_back)
> > > BTF_ID(func, bpf_cast_to_kern_ctx)
> > > BTF_ID(func, bpf_rdonly_cast)
> > > +BTF_ID(func, bpf_dynptr_from_skb)
> > > BTF_SET_END(special_kfunc_set)
> > >
> > > BTF_ID_LIST(special_kfunc_list)
> > > @@ -8673,6 +8770,7 @@ BTF_ID(func, bpf_list_pop_front)
> > > BTF_ID(func, bpf_list_pop_back)
> > > BTF_ID(func, bpf_cast_to_kern_ctx)
> > > BTF_ID(func, bpf_rdonly_cast)
> > > +BTF_ID(func, bpf_dynptr_from_skb)
> > > BTF_ID(func, bpf_rcu_read_lock)
> > > BTF_ID(func, bpf_rcu_read_unlock)
> > >
> > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > return ret;
> > > break;
> > > case KF_ARG_PTR_TO_DYNPTR:
> > > + {
> > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > +
> > > if (reg->type != PTR_TO_STACK &&
> > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > return -EINVAL;
> > > }
> > >
> > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > + else
> > > + dynptr_arg_type |= MEM_RDONLY;
> > > +
> > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > + meta->func_id);
> > > if (ret < 0)
> > > return ret;
> > > break;
> > > + }
> > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > *cnt = 1;
> > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > + struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_4, is_rdonly) };
> >
> > Why use 16-byte insn to pass boolean in R4 ?
> > Single 8-byte MOV would do.
>
> Great, I'll change it to a 8-byte MOV
>
> >
> > > +
> > > + insn_buf[0] = addr[0];
> > > + insn_buf[1] = addr[1];
> > > + insn_buf[2] = *insn;
> > > + *cnt = 3;
> > > }
> > > return 0;
> > > }
> > > diff --git a/net/core/filter.c b/net/core/filter.c
> > > index 6da78b3d381e..ddb47126071a 100644
> > > --- a/net/core/filter.c
> > > +++ b/net/core/filter.c
> > > @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> > > skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> > > }
> > >
> > > -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > > - const void *, from, u32, len, u64, flags)
> > > +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> > > + u32 len, u64 flags)
> >
> > This change is just to be able to call __bpf_skb_store_bytes() ?
> > If so, it's unnecessary.
> > See:
> > BPF_CALL_4(sk_reuseport_load_bytes,
> > const struct sk_reuseport_kern *, reuse_kern, u32, offset,
> > void *, to, u32, len)
> > {
> > return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
> > }
> >
>
> There was prior feedback [0] that using four underscores to call a
> helper function is confusing and makes it ungreppable
There are plenty of ungreppable funcs in the kernel.
Try finding where folio_test_dirty() is defined.
mm subsystem is full of such 'features'.
Not friendly for casual kernel code reader, but useful.
Since quadruple underscore is already used in the code base
I see no reason to sacrifice bpf_skb_load_bytes performance with extra call.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 5:36 ` Alexei Starovoitov
@ 2023-01-31 17:54 ` Joanne Koong
2023-01-31 19:50 ` Alexei Starovoitov
2023-02-08 21:46 ` Joanne Koong
0 siblings, 2 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 17:54 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 9:36 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> > On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> > > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > > benefits. One is that they allow operations on sizes that are not
> > > > statically known at compile-time (eg variable-sized accesses).
> > > > Another is that parsing the packet data through dynptrs (instead of
> > > > through direct access of skb->data and skb->data_end) can be more
> > > > ergonomic and less brittle (eg does not need manual if checking for
> > > > being within bounds of data_end).
> > > >
> > > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > > will return a data slice that is read-only where any writes to it will
> > > > be rejected by the verifier).
> > > >
> > > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > > interfaces, reading and writing from/to data in the head as well as from/to
> > > > non-linear paged buffers is supported. For data slices (through the
> > > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > > portion.
> > >
> > > Looks like there is an assumption in parts of this patch that
> > > linear part of skb is always writeable. That's not the case.
> > > See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
> > > For TC progs it calls bpf_unclone_prologue() which adds hidden
> > > bpf_skb_pull_data() in the beginning of the prog to make it writeable.
> >
> > I think we can make this assumption? For writable progs (referenced in
> > the may_access_direct_pkt_data() function), all of them have a
> > gen_prologue that unclones the buffer (eg tc_cls_act, lwt_xmit, sk_skb
> > progs) or their linear portion is okay to write into by default (eg
> > xdp, sk_msg, cg_sockopt progs).
>
> but the patch was preserving seen_direct_write in some cases.
> I'm still confused.
seen_direct_write is used to determine whether to actually unclone or
not in the program's prologue function (eg tc_cls_act_prologue() ->
bpf_unclone_prologue() where in bpf_unclone_prologue(), if
direct_write was not true, then it can skip doing the actual
uncloning).
I think the part of the patch you're talking about regarding
seen_direct_write is this in check_helper_call():
+ if (func_id == BPF_FUNC_dynptr_data &&
+ dynptr_type == BPF_DYNPTR_TYPE_SKB) {
+ bool seen_direct_write = env->seen_direct_write;
+
+ regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
+ if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
+ regs[BPF_REG_0].type |= MEM_RDONLY;
+ else
+ /*
+ * Calling may_access_direct_pkt_data() will set
+ * env->seen_direct_write to true if the skb is
+ * writable. As an optimization, we can ignore
+ * setting env->seen_direct_write.
+ *
+ * env->seen_direct_write is used by skb
+ * programs to determine whether the skb's page
+ * buffers should be cloned. Since data slice
+ * writes would only be to the head, we can skip
+ * this.
+ */
+ env->seen_direct_write = seen_direct_write;
+ }
If the data slice for a skb dynptr is writable, then seen_direct_write
gets set to true (done internally in may_access_direct_pkt_data()) so
that the skb is actually uncloned, whereas if it's read-only, then
env->seen_direct_write gets reset to its original value (since the
may_access_direct_pkt_data() call will have set env->seen_direct_write
to true)
>
> > >
> > > > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > > > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > > > to data in a paged buffer, so it will need to pull the buffer first into
> > > > the head. The reason it needs to be pulled instead of writing directly to
> > > > the paged buffers is because they may be cloned (only the head of the skb
> > > > is by default uncloned). As such, any bpf_dynptr_write() will
> > > > automatically have its prior data slices invalidated, even if the write
> > > > is to data in the skb head (the verifier has no way of differentiating
> > > > whether the write is to the head or paged buffers during program load
> > > > time).
> > >
> > > Could you explain the workflow how bpf_dynptr_write() invalidates other
> > > pkt pointers ?
> > > I expected bpf_dynptr_write() to be in bpf_helper_changes_pkt_data().
> > > Looks like bpf_dynptr_write() calls bpf_skb_store_bytes() underneath,
> > > but that doesn't help the verifier.
> >
> > In the verifier in check_helper_call(), for the BPF_FUNC_dynptr_write
> > case (line 8236) the "changes_data" variable gets set to true if the
> > dynptr is an skb type. At the end of check_helper_call() on line 8474,
> > since "changes_data" is true, clear_all_pkt_pointer() gets called,
> > which invalidates the other packet pointers.
>
> Ahh. I see. Thanks for explaining.
>
> > >
> > > > Please note as well that any other helper calls that change the
> > > > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > > > slices of the skb dynptr as well. The stack trace for this is
> > > > check_helper_call() -> clear_all_pkt_pointers() ->
> > > > __clear_all_pkt_pointers() -> mark_reg_unknown().
> > >
> > > __clear_all_pkt_pointers isn't present in the tree. Typo ?
> >
> > I'll update this message, clear_all_pkt_pointers() and
> > __clear_all_pkt_pointers() were combined in a previous commit.
> >
> > >
> > > >
> > > > For examples of how skb dynptrs can be used, please see the attached
> > > > selftests.
> > > >
> > > > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > > > ---
> > > > include/linux/bpf.h | 82 +++++++++------
> > > > include/linux/filter.h | 18 ++++
> > > > include/uapi/linux/bpf.h | 37 +++++--
> > > > kernel/bpf/btf.c | 18 ++++
> > > > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > > > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > > > net/core/filter.c | 60 ++++++++++-
> > > > tools/include/uapi/linux/bpf.h | 37 +++++--
> > > > 8 files changed, 432 insertions(+), 100 deletions(-)
> > > >
> > > > diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> > > > index 14a0264fac57..1ac061b64582 100644
> > [...]
> > > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > > + bool seen_direct_write = env->seen_direct_write;
> > > > +
> > > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > > + else
> > > > + /*
> > > > + * Calling may_access_direct_pkt_data() will set
> > > > + * env->seen_direct_write to true if the skb is
> > > > + * writable. As an optimization, we can ignore
> > > > + * setting env->seen_direct_write.
> > > > + *
> > > > + * env->seen_direct_write is used by skb
> > > > + * programs to determine whether the skb's page
> > > > + * buffers should be cloned. Since data slice
> > > > + * writes would only be to the head, we can skip
> > > > + * this.
>
> I was talking about above comment. It reads as 'write to the head are allowed'.
> But they're not. seen_direct_write is needed to do hidden pull.
>
I will remove this line, I agree that it is confusing.
> > > > + */
> > > > + env->seen_direct_write = seen_direct_write;
> > >
> > > This looks incorrect. skb head might not be writeable.
> > >
> > > > + }
> > > > break;
> > > > case RET_PTR_TO_MEM_OR_BTF_ID:
> > > > {
> > > > @@ -8649,6 +8744,7 @@ enum special_kfunc_type {
> > > > KF_bpf_list_pop_back,
> > > > KF_bpf_cast_to_kern_ctx,
> > > > KF_bpf_rdonly_cast,
> > > > + KF_bpf_dynptr_from_skb,
> > > > KF_bpf_rcu_read_lock,
> > > > KF_bpf_rcu_read_unlock,
> > > > };
> > > > @@ -8662,6 +8758,7 @@ BTF_ID(func, bpf_list_pop_front)
> > > > BTF_ID(func, bpf_list_pop_back)
> > > > BTF_ID(func, bpf_cast_to_kern_ctx)
> > > > BTF_ID(func, bpf_rdonly_cast)
> > > > +BTF_ID(func, bpf_dynptr_from_skb)
> > > > BTF_SET_END(special_kfunc_set)
> > > >
> > > > BTF_ID_LIST(special_kfunc_list)
> > > > @@ -8673,6 +8770,7 @@ BTF_ID(func, bpf_list_pop_front)
> > > > BTF_ID(func, bpf_list_pop_back)
> > > > BTF_ID(func, bpf_cast_to_kern_ctx)
> > > > BTF_ID(func, bpf_rdonly_cast)
> > > > +BTF_ID(func, bpf_dynptr_from_skb)
> > > > BTF_ID(func, bpf_rcu_read_lock)
> > > > BTF_ID(func, bpf_rcu_read_unlock)
> > > >
> > > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > > return ret;
> > > > break;
> > > > case KF_ARG_PTR_TO_DYNPTR:
> > > > + {
> > > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > > +
> > > > if (reg->type != PTR_TO_STACK &&
> > > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > > return -EINVAL;
> > > > }
> > > >
> > > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > > + else
> > > > + dynptr_arg_type |= MEM_RDONLY;
> > > > +
> > > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > > + meta->func_id);
> > > > if (ret < 0)
> > > > return ret;
> > > > break;
> > > > + }
> > > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > > *cnt = 1;
> > > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > > + struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_4, is_rdonly) };
> > >
> > > Why use 16-byte insn to pass boolean in R4 ?
> > > Single 8-byte MOV would do.
> >
> > Great, I'll change it to a 8-byte MOV
> >
> > >
> > > > +
> > > > + insn_buf[0] = addr[0];
> > > > + insn_buf[1] = addr[1];
> > > > + insn_buf[2] = *insn;
> > > > + *cnt = 3;
> > > > }
> > > > return 0;
> > > > }
> > > > diff --git a/net/core/filter.c b/net/core/filter.c
> > > > index 6da78b3d381e..ddb47126071a 100644
> > > > --- a/net/core/filter.c
> > > > +++ b/net/core/filter.c
> > > > @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> > > > skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> > > > }
> > > >
> > > > -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > > > - const void *, from, u32, len, u64, flags)
> > > > +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> > > > + u32 len, u64 flags)
> > >
> > > This change is just to be able to call __bpf_skb_store_bytes() ?
> > > If so, it's unnecessary.
> > > See:
> > > BPF_CALL_4(sk_reuseport_load_bytes,
> > > const struct sk_reuseport_kern *, reuse_kern, u32, offset,
> > > void *, to, u32, len)
> > > {
> > > return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
> > > }
> > >
> >
> > There was prior feedback [0] that using four underscores to call a
> > helper function is confusing and makes it ungreppable
>
> There are plenty of ungreppable funcs in the kernel.
> Try finding where folio_test_dirty() is defined.
> mm subsystem is full of such 'features'.
> Not friendly for casual kernel code reader, but useful.
>
> Since quadruple underscore is already used in the code base
> I see no reason to sacrifice bpf_skb_load_bytes performance with extra call.
I don't have a preference either way, I'll change it to use the
quadruple underscore in the next version
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 17:54 ` Joanne Koong
@ 2023-01-31 19:50 ` Alexei Starovoitov
2023-01-31 21:29 ` Joanne Koong
2023-02-08 21:46 ` Joanne Koong
1 sibling, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-31 19:50 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, Daniel Borkmann, Andrii Nakryiko, Martin KaFai Lau,
Alexei Starovoitov, Network Development, Kumar Kartikeya Dwivedi,
Kernel Team
On Tue, Jan 31, 2023 at 9:55 AM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 9:36 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> > > On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > > >
> > > > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> > > > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > > > benefits. One is that they allow operations on sizes that are not
> > > > > statically known at compile-time (eg variable-sized accesses).
> > > > > Another is that parsing the packet data through dynptrs (instead of
> > > > > through direct access of skb->data and skb->data_end) can be more
> > > > > ergonomic and less brittle (eg does not need manual if checking for
> > > > > being within bounds of data_end).
> > > > >
> > > > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > > > will return a data slice that is read-only where any writes to it will
> > > > > be rejected by the verifier).
> > > > >
> > > > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > > > interfaces, reading and writing from/to data in the head as well as from/to
> > > > > non-linear paged buffers is supported. For data slices (through the
> > > > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > > > portion.
> > > >
> > > > Looks like there is an assumption in parts of this patch that
> > > > linear part of skb is always writeable. That's not the case.
> > > > See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
> > > > For TC progs it calls bpf_unclone_prologue() which adds hidden
> > > > bpf_skb_pull_data() in the beginning of the prog to make it writeable.
> > >
> > > I think we can make this assumption? For writable progs (referenced in
> > > the may_access_direct_pkt_data() function), all of them have a
> > > gen_prologue that unclones the buffer (eg tc_cls_act, lwt_xmit, sk_skb
> > > progs) or their linear portion is okay to write into by default (eg
> > > xdp, sk_msg, cg_sockopt progs).
> >
> > but the patch was preserving seen_direct_write in some cases.
> > I'm still confused.
>
> seen_direct_write is used to determine whether to actually unclone or
> not in the program's prologue function (eg tc_cls_act_prologue() ->
> bpf_unclone_prologue() where in bpf_unclone_prologue(), if
> direct_write was not true, then it can skip doing the actual
> uncloning).
>
> I think the part of the patch you're talking about regarding
> seen_direct_write is this in check_helper_call():
>
> + if (func_id == BPF_FUNC_dynptr_data &&
> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> + bool seen_direct_write = env->seen_direct_write;
> +
> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> + regs[BPF_REG_0].type |= MEM_RDONLY;
> + else
> + /*
> + * Calling may_access_direct_pkt_data() will set
> + * env->seen_direct_write to true if the skb is
> + * writable. As an optimization, we can ignore
> + * setting env->seen_direct_write.
> + *
> + * env->seen_direct_write is used by skb
> + * programs to determine whether the skb's page
> + * buffers should be cloned. Since data slice
> + * writes would only be to the head, we can skip
> + * this.
> + */
> + env->seen_direct_write = seen_direct_write;
> + }
>
> If the data slice for a skb dynptr is writable, then seen_direct_write
> gets set to true (done internally in may_access_direct_pkt_data()) so
> that the skb is actually uncloned, whereas if it's read-only, then
> env->seen_direct_write gets reset to its original value (since the
> may_access_direct_pkt_data() call will have set env->seen_direct_write
> to true)
I'm still confused.
When may_access_direct_pkt_data() returns false
it doesn't change seen_direct_write.
When it returns true it also sets seen_direct_write=true.
But the code above restores it to whatever value it had before.
How is this correct?
Are you saying that another may_access_direct_pkt_data() gets
called somewhere in the verifier that sets seen_direct_write=true?
But what's the harm in doing it twice or N times in all cases?
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 19:50 ` Alexei Starovoitov
@ 2023-01-31 21:29 ` Joanne Koong
0 siblings, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 21:29 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: bpf, Daniel Borkmann, Andrii Nakryiko, Martin KaFai Lau,
Alexei Starovoitov, Network Development, Kumar Kartikeya Dwivedi,
Kernel Team
On Tue, Jan 31, 2023 at 11:50 AM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 9:55 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 9:36 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> > > > On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> > > > <alexei.starovoitov@gmail.com> wrote:
> > > > >
> > > > > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> > > > > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > > > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > > > > benefits. One is that they allow operations on sizes that are not
> > > > > > statically known at compile-time (eg variable-sized accesses).
> > > > > > Another is that parsing the packet data through dynptrs (instead of
> > > > > > through direct access of skb->data and skb->data_end) can be more
> > > > > > ergonomic and less brittle (eg does not need manual if checking for
> > > > > > being within bounds of data_end).
> > > > > >
> > > > > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > > > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > > > > will return a data slice that is read-only where any writes to it will
> > > > > > be rejected by the verifier).
> > > > > >
> > > > > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > > > > interfaces, reading and writing from/to data in the head as well as from/to
> > > > > > non-linear paged buffers is supported. For data slices (through the
> > > > > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > > > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > > > > portion.
> > > > >
> > > > > Looks like there is an assumption in parts of this patch that
> > > > > linear part of skb is always writeable. That's not the case.
> > > > > See if (ops->gen_prologue || env->seen_direct_write) in convert_ctx_accesses().
> > > > > For TC progs it calls bpf_unclone_prologue() which adds hidden
> > > > > bpf_skb_pull_data() in the beginning of the prog to make it writeable.
> > > >
> > > > I think we can make this assumption? For writable progs (referenced in
> > > > the may_access_direct_pkt_data() function), all of them have a
> > > > gen_prologue that unclones the buffer (eg tc_cls_act, lwt_xmit, sk_skb
> > > > progs) or their linear portion is okay to write into by default (eg
> > > > xdp, sk_msg, cg_sockopt progs).
> > >
> > > but the patch was preserving seen_direct_write in some cases.
> > > I'm still confused.
> >
> > seen_direct_write is used to determine whether to actually unclone or
> > not in the program's prologue function (eg tc_cls_act_prologue() ->
> > bpf_unclone_prologue() where in bpf_unclone_prologue(), if
> > direct_write was not true, then it can skip doing the actual
> > uncloning).
> >
> > I think the part of the patch you're talking about regarding
> > seen_direct_write is this in check_helper_call():
> >
> > + if (func_id == BPF_FUNC_dynptr_data &&
> > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > + bool seen_direct_write = env->seen_direct_write;
> > +
> > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > + else
> > + /*
> > + * Calling may_access_direct_pkt_data() will set
> > + * env->seen_direct_write to true if the skb is
> > + * writable. As an optimization, we can ignore
> > + * setting env->seen_direct_write.
> > + *
> > + * env->seen_direct_write is used by skb
> > + * programs to determine whether the skb's page
> > + * buffers should be cloned. Since data slice
> > + * writes would only be to the head, we can skip
> > + * this.
> > + */
> > + env->seen_direct_write = seen_direct_write;
> > + }
> >
> > If the data slice for a skb dynptr is writable, then seen_direct_write
> > gets set to true (done internally in may_access_direct_pkt_data()) so
> > that the skb is actually uncloned, whereas if it's read-only, then
> > env->seen_direct_write gets reset to its original value (since the
> > may_access_direct_pkt_data() call will have set env->seen_direct_write
> > to true)
>
> I'm still confused.
> When may_access_direct_pkt_data() returns false
> it doesn't change seen_direct_write.
> When it returns true it also sets seen_direct_write=true.
> But the code above restores it to whatever value it had before.
> How is this correct?
> Are you saying that another may_access_direct_pkt_data() gets
> called somewhere in the verifier that sets seen_direct_write=true?
> But what's the harm in doing it twice or N times in all cases?
I'm confused now too. I added this in v7, judging from the comment
block, I think I added this because I thought uncloning an skb only
needs to happen if the skb's page buffers get written to (aka only if
the skb needs to be pulled), not if it's linear portion gets written
to. This is incorrect - writing to the linear part also needs to
unclone the skb. I will fix this section when I resubmit
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 17:54 ` Joanne Koong
2023-01-31 19:50 ` Alexei Starovoitov
@ 2023-02-08 21:46 ` Joanne Koong
2023-02-08 23:22 ` Alexei Starovoitov
1 sibling, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-02-08 21:46 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Tue, Jan 31, 2023 at 9:54 AM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 9:36 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> > > On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > > >
> > > > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
[...]
> > > > > diff --git a/net/core/filter.c b/net/core/filter.c
> > > > > index 6da78b3d381e..ddb47126071a 100644
> > > > > --- a/net/core/filter.c
> > > > > +++ b/net/core/filter.c
> > > > > @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> > > > > skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> > > > > }
> > > > >
> > > > > -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > > > > - const void *, from, u32, len, u64, flags)
> > > > > +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> > > > > + u32 len, u64 flags)
> > > >
> > > > This change is just to be able to call __bpf_skb_store_bytes() ?
> > > > If so, it's unnecessary.
> > > > See:
> > > > BPF_CALL_4(sk_reuseport_load_bytes,
> > > > const struct sk_reuseport_kern *, reuse_kern, u32, offset,
> > > > void *, to, u32, len)
> > > > {
> > > > return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
> > > > }
> > > >
> > >
> > > There was prior feedback [0] that using four underscores to call a
> > > helper function is confusing and makes it ungreppable
> >
> > There are plenty of ungreppable funcs in the kernel.
> > Try finding where folio_test_dirty() is defined.
> > mm subsystem is full of such 'features'.
> > Not friendly for casual kernel code reader, but useful.
> >
> > Since quadruple underscore is already used in the code base
> > I see no reason to sacrifice bpf_skb_load_bytes performance with extra call.
>
> I don't have a preference either way, I'll change it to use the
> quadruple underscore in the next version
I think we still need these extra __bpf_skb_store/load_bytes()
functions, because BPF_CALL_x static inlines the
bpf_skb_store/load_bytes helpers in net/core/filter.c, and we need to
call these bpf_skb_store/load_bytes helpers from another file
(kernel/bpf/helpers.c). I think the only other alternative is moving
the BPF_CALL_x declaration of bpf_skb_store/load bytes to
include/linux/filter.h, but I think having the extra
__bpf_skb_store/load_bytes() is cleaner.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-08 21:46 ` Joanne Koong
@ 2023-02-08 23:22 ` Alexei Starovoitov
0 siblings, 0 replies; 44+ messages in thread
From: Alexei Starovoitov @ 2023-02-08 23:22 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, Daniel Borkmann, Andrii Nakryiko, Martin KaFai Lau,
Alexei Starovoitov, Network Development, Kumar Kartikeya Dwivedi,
Kernel Team
On Wed, Feb 8, 2023 at 1:47 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 9:54 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 9:36 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Mon, Jan 30, 2023 at 04:44:12PM -0800, Joanne Koong wrote:
> > > > On Sun, Jan 29, 2023 at 3:39 PM Alexei Starovoitov
> > > > <alexei.starovoitov@gmail.com> wrote:
> > > > >
> > > > > On Fri, Jan 27, 2023 at 11:17:01AM -0800, Joanne Koong wrote:
> [...]
> > > > > > diff --git a/net/core/filter.c b/net/core/filter.c
> > > > > > index 6da78b3d381e..ddb47126071a 100644
> > > > > > --- a/net/core/filter.c
> > > > > > +++ b/net/core/filter.c
> > > > > > @@ -1684,8 +1684,8 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
> > > > > > skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
> > > > > > }
> > > > > >
> > > > > > -BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
> > > > > > - const void *, from, u32, len, u64, flags)
> > > > > > +int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
> > > > > > + u32 len, u64 flags)
> > > > >
> > > > > This change is just to be able to call __bpf_skb_store_bytes() ?
> > > > > If so, it's unnecessary.
> > > > > See:
> > > > > BPF_CALL_4(sk_reuseport_load_bytes,
> > > > > const struct sk_reuseport_kern *, reuse_kern, u32, offset,
> > > > > void *, to, u32, len)
> > > > > {
> > > > > return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
> > > > > }
> > > > >
> > > >
> > > > There was prior feedback [0] that using four underscores to call a
> > > > helper function is confusing and makes it ungreppable
> > >
> > > There are plenty of ungreppable funcs in the kernel.
> > > Try finding where folio_test_dirty() is defined.
> > > mm subsystem is full of such 'features'.
> > > Not friendly for casual kernel code reader, but useful.
> > >
> > > Since quadruple underscore is already used in the code base
> > > I see no reason to sacrifice bpf_skb_load_bytes performance with extra call.
> >
> > I don't have a preference either way, I'll change it to use the
> > quadruple underscore in the next version
>
> I think we still need these extra __bpf_skb_store/load_bytes()
> functions, because BPF_CALL_x static inlines the
> bpf_skb_store/load_bytes helpers in net/core/filter.c, and we need to
> call these bpf_skb_store/load_bytes helpers from another file
> (kernel/bpf/helpers.c). I think the only other alternative is moving
> the BPF_CALL_x declaration of bpf_skb_store/load bytes to
> include/linux/filter.h, but I think having the extra
> __bpf_skb_store/load_bytes() is cleaner.
bpf_skb_load_bytes() is a performance critical function.
I'm worried about the cost of the extra call.
Will compiler be smart enough to inline __bpf_skb_load_bytes()
in both cases? Probably not if they're in different .c files.
Not sure how to solve it. Make it a static inline in skbuff.h ?
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-27 19:17 ` [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs Joanne Koong
2023-01-29 23:39 ` Alexei Starovoitov
@ 2023-01-30 22:04 ` Martin KaFai Lau
2023-01-30 22:31 ` Alexei Starovoitov
2023-01-31 18:18 ` Joanne Koong
2023-01-31 0:48 ` Andrii Nakryiko
2 siblings, 2 replies; 44+ messages in thread
From: Martin KaFai Lau @ 2023-01-30 22:04 UTC (permalink / raw)
To: Joanne Koong
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team, bpf
On 1/27/23 11:17 AM, Joanne Koong wrote:
> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> mark_reg_known_zero(env, regs, BPF_REG_0);
> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> regs[BPF_REG_0].mem_size = meta.mem_size;
> + if (func_id == BPF_FUNC_dynptr_data &&
> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> + bool seen_direct_write = env->seen_direct_write;
> +
> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> + regs[BPF_REG_0].type |= MEM_RDONLY;
> + else
> + /*
> + * Calling may_access_direct_pkt_data() will set
> + * env->seen_direct_write to true if the skb is
> + * writable. As an optimization, we can ignore
> + * setting env->seen_direct_write.
> + *
> + * env->seen_direct_write is used by skb
> + * programs to determine whether the skb's page
> + * buffers should be cloned. Since data slice
> + * writes would only be to the head, we can skip
> + * this.
> + */
> + env->seen_direct_write = seen_direct_write;
> + }
[ ... ]
> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> return ret;
> break;
> case KF_ARG_PTR_TO_DYNPTR:
> + {
> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> +
> if (reg->type != PTR_TO_STACK &&
> reg->type != CONST_PTR_TO_DYNPTR) {
> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> return -EINVAL;
> }
>
> - ret = process_dynptr_func(env, regno, insn_idx,
> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> + else
> + dynptr_arg_type |= MEM_RDONLY;
> +
> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> + meta->func_id);
> if (ret < 0)
> return ret;
> break;
> + }
> case KF_ARG_PTR_TO_LIST_HEAD:
> if (reg->type != PTR_TO_MAP_VALUE &&
> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> *cnt = 1;
> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
Does it need to restore the env->seen_direct_write here also?
It seems this 'seen_direct_write' saving/restoring is needed now because
'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is actually
writing the packet. Some refactoring can help to avoid issue like this.
While at 'seen_direct_write', Alexei has also pointed out that the verifier
needs to track whether the (packet) 'slice' returned by bpf_dynptr_data() has
been written. It should be tracked in 'seen_direct_write'. Take a look at how
reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere in
v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
PTR_TO_MEM instead of PTR_TO_PACKET.
[ ... ]
> +int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
> + struct bpf_dynptr_kern *ptr, int is_rdonly)
hmm... this exposed kfunc takes "int is_rdonly".
What if the bpf prog calls it like bpf_dynptr_from_skb(..., false) in some hook
that is not writable to packet?
> +{
> + if (flags) {
> + bpf_dynptr_set_null(ptr);
> + return -EINVAL;
> + }
> +
> + bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
> +
> + if (is_rdonly)
> + bpf_dynptr_set_rdonly(ptr);
> +
> + return 0;
> +}
> +
> BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
> {
> return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
> @@ -11607,3 +11634,28 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
>
> return func;
> }
> +
> +BTF_SET8_START(bpf_kfunc_check_set_skb)
> +BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
> +BTF_SET8_END(bpf_kfunc_check_set_skb)
> +
> +static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
> + .owner = THIS_MODULE,
> + .set = &bpf_kfunc_check_set_skb,
> +};
> +
> +static int __init bpf_kfunc_init(void)
> +{
> + int ret;
> +
> + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
> + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
> + return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
> +}
> +late_initcall(bpf_kfunc_init);
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-30 22:04 ` Martin KaFai Lau
@ 2023-01-30 22:31 ` Alexei Starovoitov
2023-01-30 23:11 ` Martin KaFai Lau
2023-01-31 1:04 ` Andrii Nakryiko
2023-01-31 18:18 ` Joanne Koong
1 sibling, 2 replies; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-30 22:31 UTC (permalink / raw)
To: Martin KaFai Lau
Cc: Joanne Koong, daniel, andrii, martin.lau, ast, netdev, memxor,
kernel-team, bpf
On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> On 1/27/23 11:17 AM, Joanne Koong wrote:
> > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > mark_reg_known_zero(env, regs, BPF_REG_0);
> > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > regs[BPF_REG_0].mem_size = meta.mem_size;
> > + if (func_id == BPF_FUNC_dynptr_data &&
> > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > + bool seen_direct_write = env->seen_direct_write;
> > +
> > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > + else
> > + /*
> > + * Calling may_access_direct_pkt_data() will set
> > + * env->seen_direct_write to true if the skb is
> > + * writable. As an optimization, we can ignore
> > + * setting env->seen_direct_write.
> > + *
> > + * env->seen_direct_write is used by skb
> > + * programs to determine whether the skb's page
> > + * buffers should be cloned. Since data slice
> > + * writes would only be to the head, we can skip
> > + * this.
> > + */
> > + env->seen_direct_write = seen_direct_write;
> > + }
>
> [ ... ]
>
> > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > return ret;
> > break;
> > case KF_ARG_PTR_TO_DYNPTR:
> > + {
> > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > +
> > if (reg->type != PTR_TO_STACK &&
> > reg->type != CONST_PTR_TO_DYNPTR) {
> > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > return -EINVAL;
> > }
> > - ret = process_dynptr_func(env, regno, insn_idx,
> > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > + else
> > + dynptr_arg_type |= MEM_RDONLY;
> > +
> > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > + meta->func_id);
> > if (ret < 0)
> > return ret;
> > break;
> > + }
> > case KF_ARG_PTR_TO_LIST_HEAD:
> > if (reg->type != PTR_TO_MAP_VALUE &&
> > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > *cnt = 1;
> > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
>
> Does it need to restore the env->seen_direct_write here also?
>
> It seems this 'seen_direct_write' saving/restoring is needed now because
> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> actually writing the packet. Some refactoring can help to avoid issue like
> this.
>
> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> has been written. It should be tracked in 'seen_direct_write'. Take a look
> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> PTR_TO_MEM instead of PTR_TO_PACKET.
btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
(nothing is being called by the bpf prog).
In this case we don't need to repeat this approach. If so we don't need to
set seen_direct_write.
Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
And technically we don't need to limit it to skb head. It can handle any off/len.
It will work for skb, but there is no equivalent for xdp_pull_data().
I don't think we can implement xdp_pull_data in all drivers.
That's massive amount of work, but we need to be consistent if we want
dynptr to wrap both skb and xdp.
We can say dynptr_data is for head only, but we've seen bugs where people
had to switch from data/data_end to load_bytes.
Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
the packet calling that in bpf_dynptr_data is a heavy hammer.
It feels that we need to go back to skb_header_pointer-like discussion.
Something like:
bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
Whether buffer is a part of dynptr or program provided is tbd.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-30 22:31 ` Alexei Starovoitov
@ 2023-01-30 23:11 ` Martin KaFai Lau
2023-01-31 1:04 ` Andrii Nakryiko
1 sibling, 0 replies; 44+ messages in thread
From: Martin KaFai Lau @ 2023-01-30 23:11 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Joanne Koong, daniel, andrii, martin.lau, ast, netdev, memxor,
kernel-team, bpf
On 1/30/23 2:31 PM, Alexei Starovoitov wrote:
> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
>> On 1/27/23 11:17 AM, Joanne Koong wrote:
>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
>>> mark_reg_known_zero(env, regs, BPF_REG_0);
>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
>>> regs[BPF_REG_0].mem_size = meta.mem_size;
>>> + if (func_id == BPF_FUNC_dynptr_data &&
>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
>>> + bool seen_direct_write = env->seen_direct_write;
>>> +
>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
>>> + else
>>> + /*
>>> + * Calling may_access_direct_pkt_data() will set
>>> + * env->seen_direct_write to true if the skb is
>>> + * writable. As an optimization, we can ignore
>>> + * setting env->seen_direct_write.
>>> + *
>>> + * env->seen_direct_write is used by skb
>>> + * programs to determine whether the skb's page
>>> + * buffers should be cloned. Since data slice
>>> + * writes would only be to the head, we can skip
>>> + * this.
>>> + */
>>> + env->seen_direct_write = seen_direct_write;
>>> + }
>>
>> [ ... ]
>>
>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
>>> return ret;
>>> break;
>>> case KF_ARG_PTR_TO_DYNPTR:
>>> + {
>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
>>> +
>>> if (reg->type != PTR_TO_STACK &&
>>> reg->type != CONST_PTR_TO_DYNPTR) {
>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
>>> return -EINVAL;
>>> }
>>> - ret = process_dynptr_func(env, regno, insn_idx,
>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
>>> + else
>>> + dynptr_arg_type |= MEM_RDONLY;
>>> +
>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
>>> + meta->func_id);
>>> if (ret < 0)
>>> return ret;
>>> break;
>>> + }
>>> case KF_ARG_PTR_TO_LIST_HEAD:
>>> if (reg->type != PTR_TO_MAP_VALUE &&
>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
>>> *cnt = 1;
>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
>>
>> Does it need to restore the env->seen_direct_write here also?
>>
>> It seems this 'seen_direct_write' saving/restoring is needed now because
>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
>> actually writing the packet. Some refactoring can help to avoid issue like
>> this.
>>
>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
>> has been written. It should be tracked in 'seen_direct_write'. Take a look
>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
>> PTR_TO_MEM instead of PTR_TO_PACKET.
>
> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> (nothing is being called by the bpf prog).
> In this case we don't need to repeat this approach. If so we don't need to
> set seen_direct_write.
> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> And technically we don't need to limit it to skb head. It can handle any off/len.
> It will work for skb, but there is no equivalent for xdp_pull_data().
> I don't think we can implement xdp_pull_data in all drivers.
> That's massive amount of work, but we need to be consistent if we want
> dynptr to wrap both skb and xdp.
> We can say dynptr_data is for head only, but we've seen bugs where people
> had to switch from data/data_end to load_bytes.
>
> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> the packet calling that in bpf_dynptr_data is a heavy hammer.
>
> It feels that we need to go back to skb_header_pointer-like discussion.
> Something like:
> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> Whether buffer is a part of dynptr or program provided is tbd.
It would be nice if it can get something similar to skb_header_pointer(). I
think Jakub has mentioned that also.
No pull in the common case. Copy to 'buffer' when it is not in head (for skb) or
across frags (for xdp). In the copy-to-'buffer' case and the bpf prog did change
the 'buffer', the bpf prog will be responsible to call bpf_dynptr_write() to
write back to the skb/xdp if needed?
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-30 22:31 ` Alexei Starovoitov
2023-01-30 23:11 ` Martin KaFai Lau
@ 2023-01-31 1:04 ` Andrii Nakryiko
2023-01-31 1:49 ` Martin KaFai Lau
2023-01-31 18:30 ` Joanne Koong
1 sibling, 2 replies; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 1:04 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Martin KaFai Lau, Joanne Koong, daniel, andrii, martin.lau, ast,
netdev, memxor, kernel-team, bpf
On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > + bool seen_direct_write = env->seen_direct_write;
> > > +
> > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > + else
> > > + /*
> > > + * Calling may_access_direct_pkt_data() will set
> > > + * env->seen_direct_write to true if the skb is
> > > + * writable. As an optimization, we can ignore
> > > + * setting env->seen_direct_write.
> > > + *
> > > + * env->seen_direct_write is used by skb
> > > + * programs to determine whether the skb's page
> > > + * buffers should be cloned. Since data slice
> > > + * writes would only be to the head, we can skip
> > > + * this.
> > > + */
> > > + env->seen_direct_write = seen_direct_write;
> > > + }
> >
> > [ ... ]
> >
> > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > return ret;
> > > break;
> > > case KF_ARG_PTR_TO_DYNPTR:
> > > + {
> > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > +
> > > if (reg->type != PTR_TO_STACK &&
> > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > return -EINVAL;
> > > }
> > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > + else
> > > + dynptr_arg_type |= MEM_RDONLY;
> > > +
> > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > + meta->func_id);
> > > if (ret < 0)
> > > return ret;
> > > break;
> > > + }
> > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > *cnt = 1;
> > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> >
> > Does it need to restore the env->seen_direct_write here also?
> >
> > It seems this 'seen_direct_write' saving/restoring is needed now because
> > 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > actually writing the packet. Some refactoring can help to avoid issue like
> > this.
> >
> > While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > has been written. It should be tracked in 'seen_direct_write'. Take a look
> > at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > PTR_TO_MEM instead of PTR_TO_PACKET.
>
> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> (nothing is being called by the bpf prog).
> In this case we don't need to repeat this approach. If so we don't need to
> set seen_direct_write.
> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> And technically we don't need to limit it to skb head. It can handle any off/len.
> It will work for skb, but there is no equivalent for xdp_pull_data().
> I don't think we can implement xdp_pull_data in all drivers.
> That's massive amount of work, but we need to be consistent if we want
> dynptr to wrap both skb and xdp.
> We can say dynptr_data is for head only, but we've seen bugs where people
> had to switch from data/data_end to load_bytes.
>
> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> the packet calling that in bpf_dynptr_data is a heavy hammer.
>
> It feels that we need to go back to skb_header_pointer-like discussion.
> Something like:
> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> Whether buffer is a part of dynptr or program provided is tbd.
making it hidden within dynptr would make this approach unreliable
(memory allocations, which can fail, etc). But if we ask users to pass
it directly, then it should be relatively easy to use in practice with
some pre-allocated per-CPU buffer:
struct {
__int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__int(max_entries, 1);
__type(key, int);
__type(value, char[4096]);
} scratch SEC(".maps");
...
struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
void *p, *buf;
int zero = 0;
buf = bpf_map_lookup_elem(&scratch, &zero);
if (!buf) return 0; /* can't happen */
p = bpf_dynptr_slice(dp, off, 16, buf);
if (p == NULL) {
/* out of range */
} else {
/* work with p directly */
}
/* if we wrote something to p and it was copied to buffer, write it back */
if (p == buf) {
bpf_dynptr_write(dp, buf, 16);
}
We'll just need to teach verifier to make sure that buf is at least 16
byte long.
But I wonder if for simple cases when users are mostly sure that they
are going to access only header data directly we can have an option
for bpf_dynptr_from_skb() to specify what should be the behavior for
bpf_dynptr_slice():
- either return NULL for anything that crosses into frags (no
surprising perf penalty, but surprising NULLs);
- do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
beyond header (potential perf penalty, but on NULLs, if off+len is
within packet).
And then bpf_dynptr_from_skb() can accept a flag specifying this
behavior and store it somewhere in struct bpf_dynptr.
Thoughts?
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 1:04 ` Andrii Nakryiko
@ 2023-01-31 1:49 ` Martin KaFai Lau
2023-01-31 4:43 ` Andrii Nakryiko
2023-01-31 18:30 ` Joanne Koong
1 sibling, 1 reply; 44+ messages in thread
From: Martin KaFai Lau @ 2023-01-31 1:49 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Joanne Koong, daniel, andrii, martin.lau, ast, netdev, memxor,
kernel-team, bpf, Alexei Starovoitov
On 1/30/23 5:04 PM, Andrii Nakryiko wrote:
> On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
>>
>> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
>>> On 1/27/23 11:17 AM, Joanne Koong wrote:
>>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
>>>> mark_reg_known_zero(env, regs, BPF_REG_0);
>>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
>>>> regs[BPF_REG_0].mem_size = meta.mem_size;
>>>> + if (func_id == BPF_FUNC_dynptr_data &&
>>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
>>>> + bool seen_direct_write = env->seen_direct_write;
>>>> +
>>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
>>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
>>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
>>>> + else
>>>> + /*
>>>> + * Calling may_access_direct_pkt_data() will set
>>>> + * env->seen_direct_write to true if the skb is
>>>> + * writable. As an optimization, we can ignore
>>>> + * setting env->seen_direct_write.
>>>> + *
>>>> + * env->seen_direct_write is used by skb
>>>> + * programs to determine whether the skb's page
>>>> + * buffers should be cloned. Since data slice
>>>> + * writes would only be to the head, we can skip
>>>> + * this.
>>>> + */
>>>> + env->seen_direct_write = seen_direct_write;
>>>> + }
>>>
>>> [ ... ]
>>>
>>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
>>>> return ret;
>>>> break;
>>>> case KF_ARG_PTR_TO_DYNPTR:
>>>> + {
>>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
>>>> +
>>>> if (reg->type != PTR_TO_STACK &&
>>>> reg->type != CONST_PTR_TO_DYNPTR) {
>>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
>>>> return -EINVAL;
>>>> }
>>>> - ret = process_dynptr_func(env, regno, insn_idx,
>>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
>>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
>>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
>>>> + else
>>>> + dynptr_arg_type |= MEM_RDONLY;
>>>> +
>>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
>>>> + meta->func_id);
>>>> if (ret < 0)
>>>> return ret;
>>>> break;
>>>> + }
>>>> case KF_ARG_PTR_TO_LIST_HEAD:
>>>> if (reg->type != PTR_TO_MAP_VALUE &&
>>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
>>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
>>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
>>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
>>>> *cnt = 1;
>>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
>>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
>>>
>>> Does it need to restore the env->seen_direct_write here also?
>>>
>>> It seems this 'seen_direct_write' saving/restoring is needed now because
>>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
>>> actually writing the packet. Some refactoring can help to avoid issue like
>>> this.
>>>
>>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
>>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
>>> has been written. It should be tracked in 'seen_direct_write'. Take a look
>>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
>>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
>>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
>>> PTR_TO_MEM instead of PTR_TO_PACKET.
>>
>> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
>> (nothing is being called by the bpf prog).
>> In this case we don't need to repeat this approach. If so we don't need to
>> set seen_direct_write.
>> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
>> And technically we don't need to limit it to skb head. It can handle any off/len.
>> It will work for skb, but there is no equivalent for xdp_pull_data().
>> I don't think we can implement xdp_pull_data in all drivers.
>> That's massive amount of work, but we need to be consistent if we want
>> dynptr to wrap both skb and xdp.
>> We can say dynptr_data is for head only, but we've seen bugs where people
>> had to switch from data/data_end to load_bytes.
>>
>> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
>> the packet calling that in bpf_dynptr_data is a heavy hammer.
>>
>> It feels that we need to go back to skb_header_pointer-like discussion.
>> Something like:
>> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
>> Whether buffer is a part of dynptr or program provided is tbd.
>
> making it hidden within dynptr would make this approach unreliable
> (memory allocations, which can fail, etc). But if we ask users to pass
> it directly, then it should be relatively easy to use in practice with
> some pre-allocated per-CPU buffer:
>
>
> struct {
> __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> __int(max_entries, 1);
> __type(key, int);
> __type(value, char[4096]);
> } scratch SEC(".maps");
>
>
> ...
>
>
> struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> void *p, *buf;
> int zero = 0;
>
> buf = bpf_map_lookup_elem(&scratch, &zero);
> if (!buf) return 0; /* can't happen */
>
> p = bpf_dynptr_slice(dp, off, 16, buf);
> if (p == NULL) {
> /* out of range */
> } else {
> /* work with p directly */
> }
>
> /* if we wrote something to p and it was copied to buffer, write it back */
> if (p == buf) {
> bpf_dynptr_write(dp, buf, 16);
> }
>
>
> We'll just need to teach verifier to make sure that buf is at least 16
> byte long.
A fifth __sz arg may do:
bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
*buffer, u32 buffer__sz);
The bpf prog usually has buffer in the stack for the common small header parsing.
One side note is the bpf_dynptr_slice() still needs to check if the skb is
cloned or not even the off/len is within the head range.
> But I wonder if for simple cases when users are mostly sure that they
> are going to access only header data directly we can have an option
> for bpf_dynptr_from_skb() to specify what should be the behavior for
> bpf_dynptr_slice():
>
> - either return NULL for anything that crosses into frags (no
> surprising perf penalty, but surprising NULLs);
> - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> beyond header (potential perf penalty, but on NULLs, if off+len is
> within packet).
>
> And then bpf_dynptr_from_skb() can accept a flag specifying this
> behavior and store it somewhere in struct bpf_dynptr.
xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
need the write back handling.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 1:49 ` Martin KaFai Lau
@ 2023-01-31 4:43 ` Andrii Nakryiko
2023-01-31 5:30 ` Alexei Starovoitov
0 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 4:43 UTC (permalink / raw)
To: Martin KaFai Lau
Cc: Joanne Koong, daniel, andrii, martin.lau, ast, netdev, memxor,
kernel-team, bpf, Alexei Starovoitov
On Mon, Jan 30, 2023 at 5:49 PM Martin KaFai Lau <martin.lau@linux.dev> wrote:
>
> On 1/30/23 5:04 PM, Andrii Nakryiko wrote:
> > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> >>
> >> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> >>> On 1/27/23 11:17 AM, Joanne Koong wrote:
> >>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> >>>> mark_reg_known_zero(env, regs, BPF_REG_0);
> >>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> >>>> regs[BPF_REG_0].mem_size = meta.mem_size;
> >>>> + if (func_id == BPF_FUNC_dynptr_data &&
> >>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> >>>> + bool seen_direct_write = env->seen_direct_write;
> >>>> +
> >>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> >>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> >>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
> >>>> + else
> >>>> + /*
> >>>> + * Calling may_access_direct_pkt_data() will set
> >>>> + * env->seen_direct_write to true if the skb is
> >>>> + * writable. As an optimization, we can ignore
> >>>> + * setting env->seen_direct_write.
> >>>> + *
> >>>> + * env->seen_direct_write is used by skb
> >>>> + * programs to determine whether the skb's page
> >>>> + * buffers should be cloned. Since data slice
> >>>> + * writes would only be to the head, we can skip
> >>>> + * this.
> >>>> + */
> >>>> + env->seen_direct_write = seen_direct_write;
> >>>> + }
> >>>
> >>> [ ... ]
> >>>
> >>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> >>>> return ret;
> >>>> break;
> >>>> case KF_ARG_PTR_TO_DYNPTR:
> >>>> + {
> >>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> >>>> +
> >>>> if (reg->type != PTR_TO_STACK &&
> >>>> reg->type != CONST_PTR_TO_DYNPTR) {
> >>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> >>>> return -EINVAL;
> >>>> }
> >>>> - ret = process_dynptr_func(env, regno, insn_idx,
> >>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> >>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> >>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> >>>> + else
> >>>> + dynptr_arg_type |= MEM_RDONLY;
> >>>> +
> >>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> >>>> + meta->func_id);
> >>>> if (ret < 0)
> >>>> return ret;
> >>>> break;
> >>>> + }
> >>>> case KF_ARG_PTR_TO_LIST_HEAD:
> >>>> if (reg->type != PTR_TO_MAP_VALUE &&
> >>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> >>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> >>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> >>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> >>>> *cnt = 1;
> >>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> >>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> >>>
> >>> Does it need to restore the env->seen_direct_write here also?
> >>>
> >>> It seems this 'seen_direct_write' saving/restoring is needed now because
> >>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> >>> actually writing the packet. Some refactoring can help to avoid issue like
> >>> this.
> >>>
> >>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> >>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> >>> has been written. It should be tracked in 'seen_direct_write'. Take a look
> >>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> >>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> >>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> >>> PTR_TO_MEM instead of PTR_TO_PACKET.
> >>
> >> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> >> (nothing is being called by the bpf prog).
> >> In this case we don't need to repeat this approach. If so we don't need to
> >> set seen_direct_write.
> >> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> >> And technically we don't need to limit it to skb head. It can handle any off/len.
> >> It will work for skb, but there is no equivalent for xdp_pull_data().
> >> I don't think we can implement xdp_pull_data in all drivers.
> >> That's massive amount of work, but we need to be consistent if we want
> >> dynptr to wrap both skb and xdp.
> >> We can say dynptr_data is for head only, but we've seen bugs where people
> >> had to switch from data/data_end to load_bytes.
> >>
> >> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> >> the packet calling that in bpf_dynptr_data is a heavy hammer.
> >>
> >> It feels that we need to go back to skb_header_pointer-like discussion.
> >> Something like:
> >> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> >> Whether buffer is a part of dynptr or program provided is tbd.
> >
> > making it hidden within dynptr would make this approach unreliable
> > (memory allocations, which can fail, etc). But if we ask users to pass
> > it directly, then it should be relatively easy to use in practice with
> > some pre-allocated per-CPU buffer:
> >
> >
> > struct {
> > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > __int(max_entries, 1);
> > __type(key, int);
> > __type(value, char[4096]);
> > } scratch SEC(".maps");
> >
> >
> > ...
> >
> >
> > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > void *p, *buf;
> > int zero = 0;
> >
> > buf = bpf_map_lookup_elem(&scratch, &zero);
> > if (!buf) return 0; /* can't happen */
> >
> > p = bpf_dynptr_slice(dp, off, 16, buf);
> > if (p == NULL) {
> > /* out of range */
> > } else {
> > /* work with p directly */
> > }
> >
> > /* if we wrote something to p and it was copied to buffer, write it back */
> > if (p == buf) {
> > bpf_dynptr_write(dp, buf, 16);
> > }
> >
> >
> > We'll just need to teach verifier to make sure that buf is at least 16
> > byte long.
>
> A fifth __sz arg may do:
> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
> *buffer, u32 buffer__sz);
We'll need to make sure that buffer__sz is >= len (or preferably not
require extra size at all). We can check that at runtime, of course,
but rejecting too small buffer at verification time would be a better
experience.
>
> The bpf prog usually has buffer in the stack for the common small header parsing.
sure, that would work for small chunks
>
> One side note is the bpf_dynptr_slice() still needs to check if the skb is
> cloned or not even the off/len is within the head range.
yep, and the above snippet will still do the right thing with
bpf_dynptr_write(), right? bpf_dynptr_write() will have to pull
anyways, if I understand correctly?
>
> > But I wonder if for simple cases when users are mostly sure that they
> > are going to access only header data directly we can have an option
> > for bpf_dynptr_from_skb() to specify what should be the behavior for
> > bpf_dynptr_slice():
> >
> > - either return NULL for anything that crosses into frags (no
> > surprising perf penalty, but surprising NULLs);
> > - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> > beyond header (potential perf penalty, but on NULLs, if off+len is
> > within packet).
> >
> > And then bpf_dynptr_from_skb() can accept a flag specifying this
> > behavior and store it somewhere in struct bpf_dynptr.
>
> xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
> need the write back handling.
>
Sure, unfortunately, can't have everything. I'm just thinking how to
make bpf_dynptr_data() generically usable. Think about some common BPF
routine that calculates hash for all bytes pointed to by dynptr,
regardless of underlying dynptr type; it can iterate in small chunks,
get memory slice, if possible, but fallback to generic
bpf_dynptr_read() if doesn't. This will work for skb, xdp, LOCAL,
RINGBUF, any other dynptr type.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 4:43 ` Andrii Nakryiko
@ 2023-01-31 5:30 ` Alexei Starovoitov
2023-01-31 22:07 ` Martin KaFai Lau
` (2 more replies)
0 siblings, 3 replies; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-31 5:30 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Martin KaFai Lau, Joanne Koong, daniel, andrii, martin.lau, ast,
netdev, memxor, kernel-team, bpf
On Mon, Jan 30, 2023 at 08:43:47PM -0800, Andrii Nakryiko wrote:
> On Mon, Jan 30, 2023 at 5:49 PM Martin KaFai Lau <martin.lau@linux.dev> wrote:
> >
> > On 1/30/23 5:04 PM, Andrii Nakryiko wrote:
> > > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > >>
> > >> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > >>> On 1/27/23 11:17 AM, Joanne Koong wrote:
> > >>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > >>>> mark_reg_known_zero(env, regs, BPF_REG_0);
> > >>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > >>>> regs[BPF_REG_0].mem_size = meta.mem_size;
> > >>>> + if (func_id == BPF_FUNC_dynptr_data &&
> > >>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > >>>> + bool seen_direct_write = env->seen_direct_write;
> > >>>> +
> > >>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > >>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > >>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
> > >>>> + else
> > >>>> + /*
> > >>>> + * Calling may_access_direct_pkt_data() will set
> > >>>> + * env->seen_direct_write to true if the skb is
> > >>>> + * writable. As an optimization, we can ignore
> > >>>> + * setting env->seen_direct_write.
> > >>>> + *
> > >>>> + * env->seen_direct_write is used by skb
> > >>>> + * programs to determine whether the skb's page
> > >>>> + * buffers should be cloned. Since data slice
> > >>>> + * writes would only be to the head, we can skip
> > >>>> + * this.
> > >>>> + */
> > >>>> + env->seen_direct_write = seen_direct_write;
> > >>>> + }
> > >>>
> > >>> [ ... ]
> > >>>
> > >>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > >>>> return ret;
> > >>>> break;
> > >>>> case KF_ARG_PTR_TO_DYNPTR:
> > >>>> + {
> > >>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > >>>> +
> > >>>> if (reg->type != PTR_TO_STACK &&
> > >>>> reg->type != CONST_PTR_TO_DYNPTR) {
> > >>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > >>>> return -EINVAL;
> > >>>> }
> > >>>> - ret = process_dynptr_func(env, regno, insn_idx,
> > >>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > >>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > >>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > >>>> + else
> > >>>> + dynptr_arg_type |= MEM_RDONLY;
> > >>>> +
> > >>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > >>>> + meta->func_id);
> > >>>> if (ret < 0)
> > >>>> return ret;
> > >>>> break;
> > >>>> + }
> > >>>> case KF_ARG_PTR_TO_LIST_HEAD:
> > >>>> if (reg->type != PTR_TO_MAP_VALUE &&
> > >>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > >>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > >>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > >>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > >>>> *cnt = 1;
> > >>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > >>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > >>>
> > >>> Does it need to restore the env->seen_direct_write here also?
> > >>>
> > >>> It seems this 'seen_direct_write' saving/restoring is needed now because
> > >>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > >>> actually writing the packet. Some refactoring can help to avoid issue like
> > >>> this.
> > >>>
> > >>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > >>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > >>> has been written. It should be tracked in 'seen_direct_write'. Take a look
> > >>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > >>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > >>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > >>> PTR_TO_MEM instead of PTR_TO_PACKET.
> > >>
> > >> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > >> (nothing is being called by the bpf prog).
> > >> In this case we don't need to repeat this approach. If so we don't need to
> > >> set seen_direct_write.
> > >> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > >> And technically we don't need to limit it to skb head. It can handle any off/len.
> > >> It will work for skb, but there is no equivalent for xdp_pull_data().
> > >> I don't think we can implement xdp_pull_data in all drivers.
> > >> That's massive amount of work, but we need to be consistent if we want
> > >> dynptr to wrap both skb and xdp.
> > >> We can say dynptr_data is for head only, but we've seen bugs where people
> > >> had to switch from data/data_end to load_bytes.
> > >>
> > >> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > >> the packet calling that in bpf_dynptr_data is a heavy hammer.
> > >>
> > >> It feels that we need to go back to skb_header_pointer-like discussion.
> > >> Something like:
> > >> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > >> Whether buffer is a part of dynptr or program provided is tbd.
> > >
> > > making it hidden within dynptr would make this approach unreliable
> > > (memory allocations, which can fail, etc). But if we ask users to pass
> > > it directly, then it should be relatively easy to use in practice with
> > > some pre-allocated per-CPU buffer:
bpf_skb_pull_data() is even more unreliable, since it's a bigger allocation.
I like preallocated approach more, so we're in agreement here.
> > >
> > >
> > > struct {
> > > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > > __int(max_entries, 1);
> > > __type(key, int);
> > > __type(value, char[4096]);
> > > } scratch SEC(".maps");
> > >
> > >
> > > ...
> > >
> > >
> > > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > > void *p, *buf;
> > > int zero = 0;
> > >
> > > buf = bpf_map_lookup_elem(&scratch, &zero);
> > > if (!buf) return 0; /* can't happen */
> > >
> > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > if (p == NULL) {
> > > /* out of range */
> > > } else {
> > > /* work with p directly */
> > > }
> > >
> > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > if (p == buf) {
> > > bpf_dynptr_write(dp, buf, 16);
> > > }
> > >
> > >
> > > We'll just need to teach verifier to make sure that buf is at least 16
> > > byte long.
> >
> > A fifth __sz arg may do:
> > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
> > *buffer, u32 buffer__sz);
>
> We'll need to make sure that buffer__sz is >= len (or preferably not
> require extra size at all). We can check that at runtime, of course,
> but rejecting too small buffer at verification time would be a better
> experience.
I don't follow. Why two equivalent 'len' args ?
Just to allow 'len' to be a variable instead of constant ?
It's unusual for the verifier to have 'len' before 'buffer',
but this is fixable.
How about adding 'rd_only vs rdwr' flag ?
Then MEM_RDONLY for ret value of bpf_dynptr_slice can be set by the verifier
and in run-time bpf_dynptr_slice() wouldn't need to check for skb->cloned.
if (rd_only) return skb_header_pointer()
if (rdwr) bpf_try_make_writable(); return skb->data + off;
and final bpf_dynptr_write() is not needed.
But that doesn't work for xdp, since there is no pull.
It's not clear how to deal with BPF_F_RECOMPUTE_CSUM though.
Expose __skb_postpull_rcsum/__skb_postpush_rcsum as kfuncs?
But that defeats Andrii's goal to use dynptr as a generic wrapper.
skb is quite special.
Maybe something like:
void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
void *buffer, u32 buffer__sz)
{
if (skb_cloned()) {
skb_copy_bits(skb, offset, buffer, len);
return buffer;
}
return skb_header_pointer(...);
}
When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
No need for rdonly flag, but extra copy is there in case of cloned which
could have been avoided with extra rd_only flag.
In case of xdp it will be:
void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
void *buffer, u32 buffer__sz)
{
ptr = bpf_xdp_pointer(xdp, offset, len);
if (ptr)
return ptr;
bpf_xdp_copy_buf(xdp, offset, buffer, len, false); /* copy into buf */
return buffer;
}
bpf_dynptr_write will use bpf_xdp_copy_buf(,true); /* copy into xdp */
> >
> > The bpf prog usually has buffer in the stack for the common small header parsing.
>
> sure, that would work for small chunks
>
> >
> > One side note is the bpf_dynptr_slice() still needs to check if the skb is
> > cloned or not even the off/len is within the head range.
>
> yep, and the above snippet will still do the right thing with
> bpf_dynptr_write(), right? bpf_dynptr_write() will have to pull
> anyways, if I understand correctly?
Yes and No. bpf_skb_store_bytes is doing pull followed by memcpy,
while xdp_store_bytes does scatter gather copy into frags.
We should probably add similar copy to skb case to avoid allocations and pull.
Then in case of:
if (p == buf) {
bpf_dynptr_write(dp, buf, 16);
}
the write will guarantee to succeed for both xdp and skb and the user
doesn't need to add error checking for alloc failures in case of skb.
> >
> > > But I wonder if for simple cases when users are mostly sure that they
> > > are going to access only header data directly we can have an option
> > > for bpf_dynptr_from_skb() to specify what should be the behavior for
> > > bpf_dynptr_slice():
> > >
> > > - either return NULL for anything that crosses into frags (no
> > > surprising perf penalty, but surprising NULLs);
> > > - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> > > beyond header (potential perf penalty, but on NULLs, if off+len is
> > > within packet).
> > >
> > > And then bpf_dynptr_from_skb() can accept a flag specifying this
> > > behavior and store it somewhere in struct bpf_dynptr.
> >
> > xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
> > need the write back handling.
> >
>
> Sure, unfortunately, can't have everything. I'm just thinking how to
> make bpf_dynptr_data() generically usable. Think about some common BPF
> routine that calculates hash for all bytes pointed to by dynptr,
> regardless of underlying dynptr type; it can iterate in small chunks,
> get memory slice, if possible, but fallback to generic
> bpf_dynptr_read() if doesn't. This will work for skb, xdp, LOCAL,
> RINGBUF, any other dynptr type.
It looks to me that dynptr on top of skb, xdp, local can work as generic reader,
but dynptr as a generic writer doesn't look possible.
BPF_F_RECOMPUTE_CSUM and BPF_F_INVALIDATE_HASH are special to skb.
There is also bpf_skb_change_proto and crazy complex bpf_skb_adjust_room.
I don't think writing into skb vs xdp vs ringbuf are generalizable.
The prog needs to do a ton more work to write into skb correctly.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 5:30 ` Alexei Starovoitov
@ 2023-01-31 22:07 ` Martin KaFai Lau
2023-01-31 23:17 ` Joanne Koong
2023-02-01 0:11 ` Andrii Nakryiko
2 siblings, 0 replies; 44+ messages in thread
From: Martin KaFai Lau @ 2023-01-31 22:07 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Joanne Koong, daniel, andrii, martin.lau, ast, netdev, memxor,
kernel-team, bpf, Andrii Nakryiko
On 1/30/23 9:30 PM, Alexei Starovoitov wrote:
>>>>> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
>>>>> the packet calling that in bpf_dynptr_data is a heavy hammer.
>>>>>
>>>>> It feels that we need to go back to skb_header_pointer-like discussion.
>>>>> Something like:
>>>>> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
>>>>> Whether buffer is a part of dynptr or program provided is tbd.
>>>>
>>>> making it hidden within dynptr would make this approach unreliable
>>>> (memory allocations, which can fail, etc). But if we ask users to pass
>>>> it directly, then it should be relatively easy to use in practice with
>>>> some pre-allocated per-CPU buffer:
>
> bpf_skb_pull_data() is even more unreliable, since it's a bigger allocation.
> I like preallocated approach more, so we're in agreement here.
>
>>>>
>>>>
>>>> struct {
>>>> __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
>>>> __int(max_entries, 1);
>>>> __type(key, int);
>>>> __type(value, char[4096]);
>>>> } scratch SEC(".maps");
>>>>
>>>>
>>>> ...
>>>>
>>>>
>>>> struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
>>>> void *p, *buf;
>>>> int zero = 0;
>>>>
>>>> buf = bpf_map_lookup_elem(&scratch, &zero);
>>>> if (!buf) return 0; /* can't happen */
>>>>
>>>> p = bpf_dynptr_slice(dp, off, 16, buf);
>>>> if (p == NULL) {
>>>> /* out of range */
>>>> } else {
>>>> /* work with p directly */
>>>> }
>>>>
>>>> /* if we wrote something to p and it was copied to buffer, write it back */
>>>> if (p == buf) {
>>>> bpf_dynptr_write(dp, buf, 16);
>>>> }
>>>>
>>>>
>>>> We'll just need to teach verifier to make sure that buf is at least 16
>>>> byte long.
>>>
>>> A fifth __sz arg may do:
>>> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
>>> *buffer, u32 buffer__sz);
>>
>> We'll need to make sure that buffer__sz is >= len (or preferably not
>> require extra size at all). We can check that at runtime, of course,
>> but rejecting too small buffer at verification time would be a better
>> experience.
>
> I don't follow. Why two equivalent 'len' args ?
> Just to allow 'len' to be a variable instead of constant ?
> It's unusual for the verifier to have 'len' before 'buffer',
> but this is fixable.
Agree. One const scalar 'len' should be enough. Buffer should have the same size
as the requesting slice.
>
> How about adding 'rd_only vs rdwr' flag ?
> Then MEM_RDONLY for ret value of bpf_dynptr_slice can be set by the verifier
> and in run-time bpf_dynptr_slice() wouldn't need to check for skb->cloned.
> if (rd_only) return skb_header_pointer()
> if (rdwr) bpf_try_make_writable(); return skb->data + off;
> and final bpf_dynptr_write() is not needed.
>
> But that doesn't work for xdp, since there is no pull.
>
> It's not clear how to deal with BPF_F_RECOMPUTE_CSUM though.
> Expose __skb_postpull_rcsum/__skb_postpush_rcsum as kfuncs?
> But that defeats Andrii's goal to use dynptr as a generic wrapper.
> skb is quite special.
>
> Maybe something like:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> if (skb_cloned()) {
> skb_copy_bits(skb, offset, buffer, len);
> return buffer;
> }
> return skb_header_pointer(...);
> }
>
> When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> No need for rdonly flag, but extra copy is there in case of cloned which
> could have been avoided with extra rd_only flag.
>
> In case of xdp it will be:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> ptr = bpf_xdp_pointer(xdp, offset, len);
> if (ptr)
> return ptr;
> bpf_xdp_copy_buf(xdp, offset, buffer, len, false); /* copy into buf */
> return buffer;
> }
>
> bpf_dynptr_write will use bpf_xdp_copy_buf(,true); /* copy into xdp */
My preference would be making bpf_dynptr_slice() work similarly for skb and xdp,
so above bpf_dynptr_slice() skb and xdp logic looks good.
Regarding, MEM_RDONLY, it probably is not relevant to xdp. For skb, not sure how
often is the 'skb_cloned() && !skb_clone_writable()'. May be it can be left for
later optimization?
Regarding BPF_F_RECOMPUTE_CSUM, I wonder if bpf_csum_diff() is enough to come up
the csum. Then the missing kfunc is to update the skb->csum. Not sure how the
csum logic will look like in xdp, probably getting csum from the xdp-hint,
calculate csum_diff and then set it to the to-be-created skb. All this is likely
a kfunc also, eg. a kfunc to directly allocate skb during the XDP_PASS case. The
bpf prog will have to be written differently if it needs to deal with the csum
but the header parsing part could at least be shared.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 5:30 ` Alexei Starovoitov
2023-01-31 22:07 ` Martin KaFai Lau
@ 2023-01-31 23:17 ` Joanne Koong
2023-02-01 0:46 ` Alexei Starovoitov
2023-02-01 0:11 ` Andrii Nakryiko
2 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 23:17 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Andrii Nakryiko, Martin KaFai Lau, daniel, andrii, martin.lau,
ast, netdev, memxor, kernel-team, bpf
oOn Mon, Jan 30, 2023 at 9:30 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 08:43:47PM -0800, Andrii Nakryiko wrote:
> > On Mon, Jan 30, 2023 at 5:49 PM Martin KaFai Lau <martin.lau@linux.dev> wrote:
> > >
> > > On 1/30/23 5:04 PM, Andrii Nakryiko wrote:
> > > > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > > > <alexei.starovoitov@gmail.com> wrote:
> > > >>
> > > >> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > > >>> On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > >>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > >>>> mark_reg_known_zero(env, regs, BPF_REG_0);
> > > >>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > >>>> regs[BPF_REG_0].mem_size = meta.mem_size;
> > > >>>> + if (func_id == BPF_FUNC_dynptr_data &&
> > > >>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > >>>> + bool seen_direct_write = env->seen_direct_write;
> > > >>>> +
> > > >>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > >>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > >>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > >>>> + else
> > > >>>> + /*
> > > >>>> + * Calling may_access_direct_pkt_data() will set
> > > >>>> + * env->seen_direct_write to true if the skb is
> > > >>>> + * writable. As an optimization, we can ignore
> > > >>>> + * setting env->seen_direct_write.
> > > >>>> + *
> > > >>>> + * env->seen_direct_write is used by skb
> > > >>>> + * programs to determine whether the skb's page
> > > >>>> + * buffers should be cloned. Since data slice
> > > >>>> + * writes would only be to the head, we can skip
> > > >>>> + * this.
> > > >>>> + */
> > > >>>> + env->seen_direct_write = seen_direct_write;
> > > >>>> + }
> > > >>>
> > > >>> [ ... ]
> > > >>>
> > > >>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > >>>> return ret;
> > > >>>> break;
> > > >>>> case KF_ARG_PTR_TO_DYNPTR:
> > > >>>> + {
> > > >>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > >>>> +
> > > >>>> if (reg->type != PTR_TO_STACK &&
> > > >>>> reg->type != CONST_PTR_TO_DYNPTR) {
> > > >>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > >>>> return -EINVAL;
> > > >>>> }
> > > >>>> - ret = process_dynptr_func(env, regno, insn_idx,
> > > >>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > >>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > >>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > >>>> + else
> > > >>>> + dynptr_arg_type |= MEM_RDONLY;
> > > >>>> +
> > > >>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > >>>> + meta->func_id);
> > > >>>> if (ret < 0)
> > > >>>> return ret;
> > > >>>> break;
> > > >>>> + }
> > > >>>> case KF_ARG_PTR_TO_LIST_HEAD:
> > > >>>> if (reg->type != PTR_TO_MAP_VALUE &&
> > > >>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > >>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > >>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > >>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > >>>> *cnt = 1;
> > > >>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > >>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > >>>
> > > >>> Does it need to restore the env->seen_direct_write here also?
> > > >>>
> > > >>> It seems this 'seen_direct_write' saving/restoring is needed now because
> > > >>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > > >>> actually writing the packet. Some refactoring can help to avoid issue like
> > > >>> this.
> > > >>>
> > > >>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > > >>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > > >>> has been written. It should be tracked in 'seen_direct_write'. Take a look
> > > >>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > > >>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > > >>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > > >>> PTR_TO_MEM instead of PTR_TO_PACKET.
> > > >>
> > > >> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > > >> (nothing is being called by the bpf prog).
> > > >> In this case we don't need to repeat this approach. If so we don't need to
> > > >> set seen_direct_write.
> > > >> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > > >> And technically we don't need to limit it to skb head. It can handle any off/len.
> > > >> It will work for skb, but there is no equivalent for xdp_pull_data().
> > > >> I don't think we can implement xdp_pull_data in all drivers.
> > > >> That's massive amount of work, but we need to be consistent if we want
> > > >> dynptr to wrap both skb and xdp.
> > > >> We can say dynptr_data is for head only, but we've seen bugs where people
> > > >> had to switch from data/data_end to load_bytes.
> > > >>
> > > >> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > > >> the packet calling that in bpf_dynptr_data is a heavy hammer.
> > > >>
> > > >> It feels that we need to go back to skb_header_pointer-like discussion.
> > > >> Something like:
> > > >> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > > >> Whether buffer is a part of dynptr or program provided is tbd.
> > > >
> > > > making it hidden within dynptr would make this approach unreliable
> > > > (memory allocations, which can fail, etc). But if we ask users to pass
> > > > it directly, then it should be relatively easy to use in practice with
> > > > some pre-allocated per-CPU buffer:
>
> bpf_skb_pull_data() is even more unreliable, since it's a bigger allocation.
> I like preallocated approach more, so we're in agreement here.
>
> > > >
> > > >
> > > > struct {
> > > > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > > > __int(max_entries, 1);
> > > > __type(key, int);
> > > > __type(value, char[4096]);
> > > > } scratch SEC(".maps");
> > > >
> > > >
> > > > ...
> > > >
> > > >
> > > > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > > > void *p, *buf;
> > > > int zero = 0;
> > > >
> > > > buf = bpf_map_lookup_elem(&scratch, &zero);
> > > > if (!buf) return 0; /* can't happen */
> > > >
> > > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > > if (p == NULL) {
> > > > /* out of range */
> > > > } else {
> > > > /* work with p directly */
> > > > }
> > > >
> > > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > > if (p == buf) {
> > > > bpf_dynptr_write(dp, buf, 16);
> > > > }
> > > >
> > > >
> > > > We'll just need to teach verifier to make sure that buf is at least 16
> > > > byte long.
> > >
> > > A fifth __sz arg may do:
> > > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
> > > *buffer, u32 buffer__sz);
> >
> > We'll need to make sure that buffer__sz is >= len (or preferably not
> > require extra size at all). We can check that at runtime, of course,
> > but rejecting too small buffer at verification time would be a better
> > experience.
>
> I don't follow. Why two equivalent 'len' args ?
> Just to allow 'len' to be a variable instead of constant ?
> It's unusual for the verifier to have 'len' before 'buffer',
> but this is fixable.
>
> How about adding 'rd_only vs rdwr' flag ?
> Then MEM_RDONLY for ret value of bpf_dynptr_slice can be set by the verifier
> and in run-time bpf_dynptr_slice() wouldn't need to check for skb->cloned.
> if (rd_only) return skb_header_pointer()
> if (rdwr) bpf_try_make_writable(); return skb->data + off;
> and final bpf_dynptr_write() is not needed.
>
> But that doesn't work for xdp, since there is no pull.
>
> It's not clear how to deal with BPF_F_RECOMPUTE_CSUM though.
> Expose __skb_postpull_rcsum/__skb_postpush_rcsum as kfuncs?
> But that defeats Andrii's goal to use dynptr as a generic wrapper.
> skb is quite special.
If it's the common case that skbs use the same flag across writes in
their bpf prog, then we can have bpf_dynptr_from_skb take in
BPF_F_RECOMPUTE_CSUM/BPF_F_INVALIDATE_HASH in its flags arg and then
always apply this when the skb does a write to packet data.
>
> Maybe something like:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> if (skb_cloned()) {
> skb_copy_bits(skb, offset, buffer, len);
> return buffer;
> }
> return skb_header_pointer(...);
> }
>
> When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> No need for rdonly flag, but extra copy is there in case of cloned which
> could have been avoided with extra rd_only flag.
We're able to track in the verifier whether the slice gets written to
or not, so if it does get written to in the skb case, can't we just
add in a call to bpf_try_make_writable() as a post-processing fixup
that gets called before bpf_dynptr_slice? Then bpf_dynptr_slice() can
just return a directly writable ptr and avoid the extra memcpy
>
> In case of xdp it will be:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> ptr = bpf_xdp_pointer(xdp, offset, len);
> if (ptr)
> return ptr;
> bpf_xdp_copy_buf(xdp, offset, buffer, len, false); /* copy into buf */
> return buffer;
> }
>
> bpf_dynptr_write will use bpf_xdp_copy_buf(,true); /* copy into xdp */
>
> > >
> > > The bpf prog usually has buffer in the stack for the common small header parsing.
> >
> > sure, that would work for small chunks
> >
> > >
> > > One side note is the bpf_dynptr_slice() still needs to check if the skb is
> > > cloned or not even the off/len is within the head range.
[...]
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 23:17 ` Joanne Koong
@ 2023-02-01 0:46 ` Alexei Starovoitov
0 siblings, 0 replies; 44+ messages in thread
From: Alexei Starovoitov @ 2023-02-01 0:46 UTC (permalink / raw)
To: Joanne Koong
Cc: Andrii Nakryiko, Martin KaFai Lau, daniel, andrii, martin.lau,
ast, netdev, memxor, kernel-team, bpf
On Tue, Jan 31, 2023 at 03:17:08PM -0800, Joanne Koong wrote:
> >
> > It's not clear how to deal with BPF_F_RECOMPUTE_CSUM though.
> > Expose __skb_postpull_rcsum/__skb_postpush_rcsum as kfuncs?
> > But that defeats Andrii's goal to use dynptr as a generic wrapper.
> > skb is quite special.
>
> If it's the common case that skbs use the same flag across writes in
> their bpf prog, then we can have bpf_dynptr_from_skb take in
> BPF_F_RECOMPUTE_CSUM/BPF_F_INVALIDATE_HASH in its flags arg and then
> always apply this when the skb does a write to packet data.
Remembering these flags at creation of dynptr is an interesting idea,
but it doesn't help with direct write into ptr returned from bpf_dynptr_slice.
The __skb_postpull_rcsum needs to be done before the write and
__skb_postpush_rcsum after the write.
> >
> > Maybe something like:
> > void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> > void *buffer, u32 buffer__sz)
> > {
> > if (skb_cloned()) {
> > skb_copy_bits(skb, offset, buffer, len);
> > return buffer;
> > }
> > return skb_header_pointer(...);
> > }
> >
> > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > No need for rdonly flag, but extra copy is there in case of cloned which
> > could have been avoided with extra rd_only flag.
>
> We're able to track in the verifier whether the slice gets written to
> or not, so if it does get written to in the skb case, can't we just
> add in a call to bpf_try_make_writable() as a post-processing fixup
> that gets called before bpf_dynptr_slice? Then bpf_dynptr_slice() can
> just return a directly writable ptr and avoid the extra memcpy
It's doable, but bpf_try_make_writable can fail and it's much slower than memcpy.
I'm not sure what you're optimizing here.
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 5:30 ` Alexei Starovoitov
2023-01-31 22:07 ` Martin KaFai Lau
2023-01-31 23:17 ` Joanne Koong
@ 2023-02-01 0:11 ` Andrii Nakryiko
2023-02-01 0:40 ` Alexei Starovoitov
2 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-02-01 0:11 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Martin KaFai Lau, Joanne Koong, daniel, andrii, martin.lau, ast,
netdev, memxor, kernel-team, bpf
On Mon, Jan 30, 2023 at 9:30 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 08:43:47PM -0800, Andrii Nakryiko wrote:
> > On Mon, Jan 30, 2023 at 5:49 PM Martin KaFai Lau <martin.lau@linux.dev> wrote:
> > >
> > > On 1/30/23 5:04 PM, Andrii Nakryiko wrote:
> > > > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > > > <alexei.starovoitov@gmail.com> wrote:
> > > >>
> > > >> On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > > >>> On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > >>>> @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > >>>> mark_reg_known_zero(env, regs, BPF_REG_0);
> > > >>>> regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > >>>> regs[BPF_REG_0].mem_size = meta.mem_size;
> > > >>>> + if (func_id == BPF_FUNC_dynptr_data &&
> > > >>>> + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > >>>> + bool seen_direct_write = env->seen_direct_write;
> > > >>>> +
> > > >>>> + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > >>>> + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > >>>> + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > >>>> + else
> > > >>>> + /*
> > > >>>> + * Calling may_access_direct_pkt_data() will set
> > > >>>> + * env->seen_direct_write to true if the skb is
> > > >>>> + * writable. As an optimization, we can ignore
> > > >>>> + * setting env->seen_direct_write.
> > > >>>> + *
> > > >>>> + * env->seen_direct_write is used by skb
> > > >>>> + * programs to determine whether the skb's page
> > > >>>> + * buffers should be cloned. Since data slice
> > > >>>> + * writes would only be to the head, we can skip
> > > >>>> + * this.
> > > >>>> + */
> > > >>>> + env->seen_direct_write = seen_direct_write;
> > > >>>> + }
> > > >>>
> > > >>> [ ... ]
> > > >>>
> > > >>>> @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > >>>> return ret;
> > > >>>> break;
> > > >>>> case KF_ARG_PTR_TO_DYNPTR:
> > > >>>> + {
> > > >>>> + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > >>>> +
> > > >>>> if (reg->type != PTR_TO_STACK &&
> > > >>>> reg->type != CONST_PTR_TO_DYNPTR) {
> > > >>>> verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > >>>> return -EINVAL;
> > > >>>> }
> > > >>>> - ret = process_dynptr_func(env, regno, insn_idx,
> > > >>>> - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > >>>> + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > >>>> + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > >>>> + else
> > > >>>> + dynptr_arg_type |= MEM_RDONLY;
> > > >>>> +
> > > >>>> + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > >>>> + meta->func_id);
> > > >>>> if (ret < 0)
> > > >>>> return ret;
> > > >>>> break;
> > > >>>> + }
> > > >>>> case KF_ARG_PTR_TO_LIST_HEAD:
> > > >>>> if (reg->type != PTR_TO_MAP_VALUE &&
> > > >>>> reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > >>>> @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > >>>> desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > >>>> insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > >>>> *cnt = 1;
> > > >>>> + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > >>>> + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > >>>
> > > >>> Does it need to restore the env->seen_direct_write here also?
> > > >>>
> > > >>> It seems this 'seen_direct_write' saving/restoring is needed now because
> > > >>> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > > >>> actually writing the packet. Some refactoring can help to avoid issue like
> > > >>> this.
> > > >>>
> > > >>> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > > >>> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > > >>> has been written. It should be tracked in 'seen_direct_write'. Take a look
> > > >>> at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > > >>> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > > >>> in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > > >>> PTR_TO_MEM instead of PTR_TO_PACKET.
> > > >>
> > > >> btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > > >> (nothing is being called by the bpf prog).
> > > >> In this case we don't need to repeat this approach. If so we don't need to
> > > >> set seen_direct_write.
> > > >> Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > > >> And technically we don't need to limit it to skb head. It can handle any off/len.
> > > >> It will work for skb, but there is no equivalent for xdp_pull_data().
> > > >> I don't think we can implement xdp_pull_data in all drivers.
> > > >> That's massive amount of work, but we need to be consistent if we want
> > > >> dynptr to wrap both skb and xdp.
> > > >> We can say dynptr_data is for head only, but we've seen bugs where people
> > > >> had to switch from data/data_end to load_bytes.
> > > >>
> > > >> Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > > >> the packet calling that in bpf_dynptr_data is a heavy hammer.
> > > >>
> > > >> It feels that we need to go back to skb_header_pointer-like discussion.
> > > >> Something like:
> > > >> bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > > >> Whether buffer is a part of dynptr or program provided is tbd.
> > > >
> > > > making it hidden within dynptr would make this approach unreliable
> > > > (memory allocations, which can fail, etc). But if we ask users to pass
> > > > it directly, then it should be relatively easy to use in practice with
> > > > some pre-allocated per-CPU buffer:
>
> bpf_skb_pull_data() is even more unreliable, since it's a bigger allocation.
> I like preallocated approach more, so we're in agreement here.
>
> > > >
> > > >
> > > > struct {
> > > > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > > > __int(max_entries, 1);
> > > > __type(key, int);
> > > > __type(value, char[4096]);
> > > > } scratch SEC(".maps");
> > > >
> > > >
> > > > ...
> > > >
> > > >
> > > > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > > > void *p, *buf;
> > > > int zero = 0;
> > > >
> > > > buf = bpf_map_lookup_elem(&scratch, &zero);
> > > > if (!buf) return 0; /* can't happen */
> > > >
> > > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > > if (p == NULL) {
> > > > /* out of range */
> > > > } else {
> > > > /* work with p directly */
> > > > }
> > > >
> > > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > > if (p == buf) {
> > > > bpf_dynptr_write(dp, buf, 16);
> > > > }
> > > >
> > > >
> > > > We'll just need to teach verifier to make sure that buf is at least 16
> > > > byte long.
> > >
> > > A fifth __sz arg may do:
> > > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void
> > > *buffer, u32 buffer__sz);
> >
> > We'll need to make sure that buffer__sz is >= len (or preferably not
> > require extra size at all). We can check that at runtime, of course,
> > but rejecting too small buffer at verification time would be a better
> > experience.
>
> I don't follow. Why two equivalent 'len' args ?
> Just to allow 'len' to be a variable instead of constant ?
> It's unusual for the verifier to have 'len' before 'buffer',
> but this is fixable.
Right, I don't like two lens as well. And no, len can't be variable,
it has to be a constant known at verification time. We could define
bpf_dynptr_slice as
void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, void
*buffer, u32 buffer__sz) and it would follow current conventions,
though feels a bit weird.
But either way we'd have to teach verifier to take buffer__sz and mark
it as the size of PTR_TO_MEM returned from bpf_dynptr_slice.
All this is doable.
>
> How about adding 'rd_only vs rdwr' flag ?
> Then MEM_RDONLY for ret value of bpf_dynptr_slice can be set by the verifier
> and in run-time bpf_dynptr_slice() wouldn't need to check for skb->cloned.
> if (rd_only) return skb_header_pointer()
> if (rdwr) bpf_try_make_writable(); return skb->data + off;
> and final bpf_dynptr_write() is not needed.
>
> But that doesn't work for xdp, since there is no pull.
>
> It's not clear how to deal with BPF_F_RECOMPUTE_CSUM though.
> Expose __skb_postpull_rcsum/__skb_postpush_rcsum as kfuncs?
> But that defeats Andrii's goal to use dynptr as a generic wrapper.
> skb is quite special.
>
> Maybe something like:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> if (skb_cloned()) {
> skb_copy_bits(skb, offset, buffer, len);
> return buffer;
> }
> return skb_header_pointer(...);
> }
>
> When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> No need for rdonly flag, but extra copy is there in case of cloned which
> could have been avoided with extra rd_only flag.
Yep, given we are designing bpf_dynptr_slice for performance, extra
copy on reads is unfortunate. ro/rw flag or have separate
bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
>
> In case of xdp it will be:
> void *bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len,
> void *buffer, u32 buffer__sz)
> {
> ptr = bpf_xdp_pointer(xdp, offset, len);
> if (ptr)
> return ptr;
> bpf_xdp_copy_buf(xdp, offset, buffer, len, false); /* copy into buf */
> return buffer;
> }
>
> bpf_dynptr_write will use bpf_xdp_copy_buf(,true); /* copy into xdp */
>
> > >
> > > The bpf prog usually has buffer in the stack for the common small header parsing.
> >
> > sure, that would work for small chunks
> >
> > >
> > > One side note is the bpf_dynptr_slice() still needs to check if the skb is
> > > cloned or not even the off/len is within the head range.
> >
> > yep, and the above snippet will still do the right thing with
> > bpf_dynptr_write(), right? bpf_dynptr_write() will have to pull
> > anyways, if I understand correctly?
>
> Yes and No. bpf_skb_store_bytes is doing pull followed by memcpy,
> while xdp_store_bytes does scatter gather copy into frags.
> We should probably add similar copy to skb case to avoid allocations and pull.
> Then in case of:
> if (p == buf) {
> bpf_dynptr_write(dp, buf, 16);
> }
>
> the write will guarantee to succeed for both xdp and skb and the user
> doesn't need to add error checking for alloc failures in case of skb.
>
That seems like a nice guarantee, agreed.
> > >
> > > > But I wonder if for simple cases when users are mostly sure that they
> > > > are going to access only header data directly we can have an option
> > > > for bpf_dynptr_from_skb() to specify what should be the behavior for
> > > > bpf_dynptr_slice():
> > > >
> > > > - either return NULL for anything that crosses into frags (no
> > > > surprising perf penalty, but surprising NULLs);
> > > > - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> > > > beyond header (potential perf penalty, but on NULLs, if off+len is
> > > > within packet).
> > > >
> > > > And then bpf_dynptr_from_skb() can accept a flag specifying this
> > > > behavior and store it somewhere in struct bpf_dynptr.
> > >
> > > xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
> > > need the write back handling.
> > >
> >
> > Sure, unfortunately, can't have everything. I'm just thinking how to
> > make bpf_dynptr_data() generically usable. Think about some common BPF
> > routine that calculates hash for all bytes pointed to by dynptr,
> > regardless of underlying dynptr type; it can iterate in small chunks,
> > get memory slice, if possible, but fallback to generic
> > bpf_dynptr_read() if doesn't. This will work for skb, xdp, LOCAL,
> > RINGBUF, any other dynptr type.
>
> It looks to me that dynptr on top of skb, xdp, local can work as generic reader,
> but dynptr as a generic writer doesn't look possible.
> BPF_F_RECOMPUTE_CSUM and BPF_F_INVALIDATE_HASH are special to skb.
> There is also bpf_skb_change_proto and crazy complex bpf_skb_adjust_room.
> I don't think writing into skb vs xdp vs ringbuf are generalizable.
> The prog needs to do a ton more work to write into skb correctly.
If that's the case, then yeah, bpf_dynptr_write() can just return
error for skb/xdp dynptrs?
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-01 0:11 ` Andrii Nakryiko
@ 2023-02-01 0:40 ` Alexei Starovoitov
2023-02-02 1:21 ` Andrii Nakryiko
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-02-01 0:40 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Martin KaFai Lau, Joanne Koong, daniel, andrii, martin.lau, ast,
netdev, memxor, kernel-team, bpf
On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> >
> > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > No need for rdonly flag, but extra copy is there in case of cloned which
> > could have been avoided with extra rd_only flag.
>
> Yep, given we are designing bpf_dynptr_slice for performance, extra
> copy on reads is unfortunate. ro/rw flag or have separate
> bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
Either flag or two kfuncs sound good to me.
> > Yes and No. bpf_skb_store_bytes is doing pull followed by memcpy,
> > while xdp_store_bytes does scatter gather copy into frags.
> > We should probably add similar copy to skb case to avoid allocations and pull.
> > Then in case of:
> > if (p == buf) {
> > bpf_dynptr_write(dp, buf, 16);
> > }
> >
> > the write will guarantee to succeed for both xdp and skb and the user
> > doesn't need to add error checking for alloc failures in case of skb.
> >
>
> That seems like a nice guarantee, agreed.
Just grepped through few projects that use skb_store_bytes.
Everywhere it looks like:
if (bpf_skb_store_byte(...))
return error;
Not a pretty code to read.
I should prioritize bpf_assert() work, so we can assert from inside of
bpf_dynptr_write() eventually and remove all these IFs.
> > > >
> > > > > But I wonder if for simple cases when users are mostly sure that they
> > > > > are going to access only header data directly we can have an option
> > > > > for bpf_dynptr_from_skb() to specify what should be the behavior for
> > > > > bpf_dynptr_slice():
> > > > >
> > > > > - either return NULL for anything that crosses into frags (no
> > > > > surprising perf penalty, but surprising NULLs);
> > > > > - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> > > > > beyond header (potential perf penalty, but on NULLs, if off+len is
> > > > > within packet).
> > > > >
> > > > > And then bpf_dynptr_from_skb() can accept a flag specifying this
> > > > > behavior and store it somewhere in struct bpf_dynptr.
> > > >
> > > > xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
> > > > need the write back handling.
> > > >
> > >
> > > Sure, unfortunately, can't have everything. I'm just thinking how to
> > > make bpf_dynptr_data() generically usable. Think about some common BPF
> > > routine that calculates hash for all bytes pointed to by dynptr,
> > > regardless of underlying dynptr type; it can iterate in small chunks,
> > > get memory slice, if possible, but fallback to generic
> > > bpf_dynptr_read() if doesn't. This will work for skb, xdp, LOCAL,
> > > RINGBUF, any other dynptr type.
> >
> > It looks to me that dynptr on top of skb, xdp, local can work as generic reader,
> > but dynptr as a generic writer doesn't look possible.
> > BPF_F_RECOMPUTE_CSUM and BPF_F_INVALIDATE_HASH are special to skb.
> > There is also bpf_skb_change_proto and crazy complex bpf_skb_adjust_room.
> > I don't think writing into skb vs xdp vs ringbuf are generalizable.
> > The prog needs to do a ton more work to write into skb correctly.
>
> If that's the case, then yeah, bpf_dynptr_write() can just return
> error for skb/xdp dynptrs?
You mean to error when these skb only flags are present, but dynptr->type == xdp ?
Yep. I don't see another option. My point was that dynptr doesn't quite work as an
abstraction for writing into networking things.
While libraries like: parse_http(&dynptr), compute_hash(&dynptr), find_string(&dynptr)
can indeed be generic and work with raw bytes, skb, xdp as an input,
which I think was on top of your wishlist for dynptr.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-01 0:40 ` Alexei Starovoitov
@ 2023-02-02 1:21 ` Andrii Nakryiko
2023-02-02 11:43 ` Alexei Starovoitov
0 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-02-02 1:21 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Martin KaFai Lau, Joanne Koong, daniel, andrii, martin.lau, ast,
netdev, memxor, kernel-team, bpf
On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > >
> > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > could have been avoided with extra rd_only flag.
> >
> > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > copy on reads is unfortunate. ro/rw flag or have separate
> > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
>
> Either flag or two kfuncs sound good to me.
Would it make sense to make bpf_dynptr_slice() as read-only variant,
and bpf_dynptr_slice_rw() for read/write? I think the common case is
read-only, right? And if users mistakenly use bpf_dynptr_slice() for
r/w case, they will get a verifier error when trying to write into the
returned pointer. While if we make bpf_dynptr_slice() as read-write,
users won't realize they are paying a performance penalty for
something that they don't actually need.
>
> > > Yes and No. bpf_skb_store_bytes is doing pull followed by memcpy,
> > > while xdp_store_bytes does scatter gather copy into frags.
> > > We should probably add similar copy to skb case to avoid allocations and pull.
> > > Then in case of:
> > > if (p == buf) {
> > > bpf_dynptr_write(dp, buf, 16);
> > > }
> > >
> > > the write will guarantee to succeed for both xdp and skb and the user
> > > doesn't need to add error checking for alloc failures in case of skb.
> > >
> >
> > That seems like a nice guarantee, agreed.
>
> Just grepped through few projects that use skb_store_bytes.
> Everywhere it looks like:
> if (bpf_skb_store_byte(...))
> return error;
>
> Not a pretty code to read.
> I should prioritize bpf_assert() work, so we can assert from inside of
> bpf_dynptr_write() eventually and remove all these IFs.
>
> > > > >
> > > > > > But I wonder if for simple cases when users are mostly sure that they
> > > > > > are going to access only header data directly we can have an option
> > > > > > for bpf_dynptr_from_skb() to specify what should be the behavior for
> > > > > > bpf_dynptr_slice():
> > > > > >
> > > > > > - either return NULL for anything that crosses into frags (no
> > > > > > surprising perf penalty, but surprising NULLs);
> > > > > > - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> > > > > > beyond header (potential perf penalty, but on NULLs, if off+len is
> > > > > > within packet).
> > > > > >
> > > > > > And then bpf_dynptr_from_skb() can accept a flag specifying this
> > > > > > behavior and store it somewhere in struct bpf_dynptr.
> > > > >
> > > > > xdp does not have the bpf_skb_pull_data() equivalent, so xdp prog will still
> > > > > need the write back handling.
> > > > >
> > > >
> > > > Sure, unfortunately, can't have everything. I'm just thinking how to
> > > > make bpf_dynptr_data() generically usable. Think about some common BPF
> > > > routine that calculates hash for all bytes pointed to by dynptr,
> > > > regardless of underlying dynptr type; it can iterate in small chunks,
> > > > get memory slice, if possible, but fallback to generic
> > > > bpf_dynptr_read() if doesn't. This will work for skb, xdp, LOCAL,
> > > > RINGBUF, any other dynptr type.
> > >
> > > It looks to me that dynptr on top of skb, xdp, local can work as generic reader,
> > > but dynptr as a generic writer doesn't look possible.
> > > BPF_F_RECOMPUTE_CSUM and BPF_F_INVALIDATE_HASH are special to skb.
> > > There is also bpf_skb_change_proto and crazy complex bpf_skb_adjust_room.
> > > I don't think writing into skb vs xdp vs ringbuf are generalizable.
> > > The prog needs to do a ton more work to write into skb correctly.
> >
> > If that's the case, then yeah, bpf_dynptr_write() can just return
> > error for skb/xdp dynptrs?
>
> You mean to error when these skb only flags are present, but dynptr->type == xdp ?
> Yep. I don't see another option. My point was that dynptr doesn't quite work as an
> abstraction for writing into networking things.
agreed
> While libraries like: parse_http(&dynptr), compute_hash(&dynptr), find_string(&dynptr)
> can indeed be generic and work with raw bytes, skb, xdp as an input,
> which I think was on top of your wishlist for dynptr.
yep, it would be a great property
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-02 1:21 ` Andrii Nakryiko
@ 2023-02-02 11:43 ` Alexei Starovoitov
2023-02-03 21:37 ` Andrii Nakryiko
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-02-02 11:43 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Martin KaFai Lau, Joanne Koong, Daniel Borkmann, Andrii Nakryiko,
Martin KaFai Lau, Alexei Starovoitov, Network Development,
Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Wed, Feb 1, 2023 at 5:21 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > > >
> > > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > > could have been avoided with extra rd_only flag.
> > >
> > > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > > copy on reads is unfortunate. ro/rw flag or have separate
> > > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
> >
> > Either flag or two kfuncs sound good to me.
>
> Would it make sense to make bpf_dynptr_slice() as read-only variant,
> and bpf_dynptr_slice_rw() for read/write? I think the common case is
> read-only, right? And if users mistakenly use bpf_dynptr_slice() for
> r/w case, they will get a verifier error when trying to write into the
> returned pointer. While if we make bpf_dynptr_slice() as read-write,
> users won't realize they are paying a performance penalty for
> something that they don't actually need.
Makes sense and it matches skb_header_pointer() usage in the kernel
which is read-only. Since there is no verifier the read-only-ness
is not enforced, but we can do it.
Looks like we've converged on bpf_dynptr_slice() and bpf_dynptr_slice_rw().
The question remains what to do with bpf_dynptr_data() backed by skb/xdp.
Should we return EINVAL to discourage its usage?
Of course, we can come up with sensible behavior for bpf_dynptr_data(),
but it will have quirks that will be not easy to document.
Even with extensive docs the users might be surprised by the behavior.
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-02 11:43 ` Alexei Starovoitov
@ 2023-02-03 21:37 ` Andrii Nakryiko
2023-02-08 2:25 ` Alexei Starovoitov
0 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-02-03 21:37 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Martin KaFai Lau, Joanne Koong, Daniel Borkmann, Andrii Nakryiko,
Martin KaFai Lau, Alexei Starovoitov, Network Development,
Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Thu, Feb 2, 2023 at 3:43 AM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Wed, Feb 1, 2023 at 5:21 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> > On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > > > >
> > > > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > > > could have been avoided with extra rd_only flag.
> > > >
> > > > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > > > copy on reads is unfortunate. ro/rw flag or have separate
> > > > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
> > >
> > > Either flag or two kfuncs sound good to me.
> >
> > Would it make sense to make bpf_dynptr_slice() as read-only variant,
> > and bpf_dynptr_slice_rw() for read/write? I think the common case is
> > read-only, right? And if users mistakenly use bpf_dynptr_slice() for
> > r/w case, they will get a verifier error when trying to write into the
> > returned pointer. While if we make bpf_dynptr_slice() as read-write,
> > users won't realize they are paying a performance penalty for
> > something that they don't actually need.
>
> Makes sense and it matches skb_header_pointer() usage in the kernel
> which is read-only. Since there is no verifier the read-only-ness
> is not enforced, but we can do it.
>
> Looks like we've converged on bpf_dynptr_slice() and bpf_dynptr_slice_rw().
> The question remains what to do with bpf_dynptr_data() backed by skb/xdp.
> Should we return EINVAL to discourage its usage?
> Of course, we can come up with sensible behavior for bpf_dynptr_data(),
> but it will have quirks that will be not easy to document.
> Even with extensive docs the users might be surprised by the behavior.
I feel like having bpf_dynptr_data() working in the common case for
skb/xdp would be nice (e.g., so basically at least work in cases when
we don't need to pull).
But we've been discussing bpf_dynptr_slice() with Joanne today, and we
came to the conclusion that bpf_dynptr_slice()/bpf_dynptr_slice_rw()
should work for any kind of dynptr (LOCAL, RINGBUF, SKB, XDP). So
generic code that wants to work with any dynptr would be able to just
use bpf_dynptr_slice, even for LOCAL/RINGBUF, even though buffer won't
ever be filled for LOCAL/RINGBUF.
In application, though, if I know I'm working with LOCAL or RINGBUF
(or MALLOC, once we have it), I'd use bpf_dynptr_data() to fill out
fixed parts, of course. bpf_dynptr_slice() would be cumbersome for
such cases (especially if I have some huge fixed part that I *know* is
available in RINGBUF/MALLOC case).
With this setup we probably won't ever need bpf_dynptr_data_rdonly(),
because we can say to use bpf_dynptr_slice() for that (even with an
unnecessary buffer).
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-03 21:37 ` Andrii Nakryiko
@ 2023-02-08 2:25 ` Alexei Starovoitov
2023-02-08 20:13 ` Joanne Koong
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-02-08 2:25 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Martin KaFai Lau, Joanne Koong, Daniel Borkmann, Andrii Nakryiko,
Martin KaFai Lau, Alexei Starovoitov, Network Development,
Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Fri, Feb 03, 2023 at 01:37:46PM -0800, Andrii Nakryiko wrote:
> On Thu, Feb 2, 2023 at 3:43 AM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Wed, Feb 1, 2023 at 5:21 PM Andrii Nakryiko
> > <andrii.nakryiko@gmail.com> wrote:
> > >
> > > On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > > >
> > > > On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > > > > >
> > > > > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > > > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > > > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > > > > could have been avoided with extra rd_only flag.
> > > > >
> > > > > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > > > > copy on reads is unfortunate. ro/rw flag or have separate
> > > > > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
> > > >
> > > > Either flag or two kfuncs sound good to me.
> > >
> > > Would it make sense to make bpf_dynptr_slice() as read-only variant,
> > > and bpf_dynptr_slice_rw() for read/write? I think the common case is
> > > read-only, right? And if users mistakenly use bpf_dynptr_slice() for
> > > r/w case, they will get a verifier error when trying to write into the
> > > returned pointer. While if we make bpf_dynptr_slice() as read-write,
> > > users won't realize they are paying a performance penalty for
> > > something that they don't actually need.
> >
> > Makes sense and it matches skb_header_pointer() usage in the kernel
> > which is read-only. Since there is no verifier the read-only-ness
> > is not enforced, but we can do it.
> >
> > Looks like we've converged on bpf_dynptr_slice() and bpf_dynptr_slice_rw().
> > The question remains what to do with bpf_dynptr_data() backed by skb/xdp.
> > Should we return EINVAL to discourage its usage?
> > Of course, we can come up with sensible behavior for bpf_dynptr_data(),
> > but it will have quirks that will be not easy to document.
> > Even with extensive docs the users might be surprised by the behavior.
>
> I feel like having bpf_dynptr_data() working in the common case for
> skb/xdp would be nice (e.g., so basically at least work in cases when
> we don't need to pull).
>
> But we've been discussing bpf_dynptr_slice() with Joanne today, and we
> came to the conclusion that bpf_dynptr_slice()/bpf_dynptr_slice_rw()
> should work for any kind of dynptr (LOCAL, RINGBUF, SKB, XDP). So
> generic code that wants to work with any dynptr would be able to just
> use bpf_dynptr_slice, even for LOCAL/RINGBUF, even though buffer won't
> ever be filled for LOCAL/RINGBUF.
great
> In application, though, if I know I'm working with LOCAL or RINGBUF
> (or MALLOC, once we have it), I'd use bpf_dynptr_data() to fill out
> fixed parts, of course. bpf_dynptr_slice() would be cumbersome for
> such cases (especially if I have some huge fixed part that I *know* is
> available in RINGBUF/MALLOC case).
bpf_dynptr_data() for local and ringbuf is fine, of course.
It already exists and has to continue working.
bpf_dynptr_data() for xdp is probably ok as well,
but bpf_dynptr_data() for skb is problematic.
data/data_end concept looked great back in 2016 when it was introduced
and lots of programs were written, but we underestimated the impact
of driver's copybreak on programs.
Network parsing progs consume headers one by one and would typically
be written as:
if (header > data_end)
return DROP;
Some drivers copybreak fixed number of bytes. Others try to be smart
and copy only headers into linear part of skb.
The drivers also change. At one point we tried to upgrade the kernel
and suddenly bpf firewall started blocking valid traffic.
Turned out the driver copybreak heuristic was changed in that kernel.
The bpf prog was converted to use skb_load_bytes() and the fire was extinguished.
It was a hard lesson.
Others learned the danger of data/data_end the hard way as well.
Take a look at cloudflare's progs/test_cls_redirect.c.
It's a complicated combination of data/data_end and skb_load_bytes().
It's essentially implementing skb_header_pointer.
I wish we could use bpf_dynptr_slice only and remove data/data_end,
but we cannot, since it's uapi.
But we shouldn't repeat the same mistake. If we do bpf_dynptr_data()
that returns linear part people will be hitting the same fragility and
difficult to debug bugs.
bpf_dynptr_data() for XDP is ok-ish, since most of XDP is still
page-per-packet, but patches to split headers in HW are starting to appear.
So even for XDP data/data_end concept may bite us.
Hence my preference is to EINVAL in bpf_dynptr_data() at least for skb,
since bpf_dynptr_slice() is a strictly better alternative.
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-08 2:25 ` Alexei Starovoitov
@ 2023-02-08 20:13 ` Joanne Koong
2023-02-09 0:39 ` Andrii Nakryiko
0 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-02-08 20:13 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Andrii Nakryiko, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Tue, Feb 7, 2023 at 6:25 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Fri, Feb 03, 2023 at 01:37:46PM -0800, Andrii Nakryiko wrote:
> > On Thu, Feb 2, 2023 at 3:43 AM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Wed, Feb 1, 2023 at 5:21 PM Andrii Nakryiko
> > > <andrii.nakryiko@gmail.com> wrote:
> > > >
> > > > On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
> > > > <alexei.starovoitov@gmail.com> wrote:
> > > > >
> > > > > On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > > > > > >
> > > > > > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > > > > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > > > > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > > > > > could have been avoided with extra rd_only flag.
> > > > > >
> > > > > > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > > > > > copy on reads is unfortunate. ro/rw flag or have separate
> > > > > > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
> > > > >
> > > > > Either flag or two kfuncs sound good to me.
> > > >
> > > > Would it make sense to make bpf_dynptr_slice() as read-only variant,
> > > > and bpf_dynptr_slice_rw() for read/write? I think the common case is
> > > > read-only, right? And if users mistakenly use bpf_dynptr_slice() for
> > > > r/w case, they will get a verifier error when trying to write into the
> > > > returned pointer. While if we make bpf_dynptr_slice() as read-write,
> > > > users won't realize they are paying a performance penalty for
> > > > something that they don't actually need.
> > >
> > > Makes sense and it matches skb_header_pointer() usage in the kernel
> > > which is read-only. Since there is no verifier the read-only-ness
> > > is not enforced, but we can do it.
> > >
> > > Looks like we've converged on bpf_dynptr_slice() and bpf_dynptr_slice_rw().
> > > The question remains what to do with bpf_dynptr_data() backed by skb/xdp.
> > > Should we return EINVAL to discourage its usage?
> > > Of course, we can come up with sensible behavior for bpf_dynptr_data(),
> > > but it will have quirks that will be not easy to document.
> > > Even with extensive docs the users might be surprised by the behavior.
> >
> > I feel like having bpf_dynptr_data() working in the common case for
> > skb/xdp would be nice (e.g., so basically at least work in cases when
> > we don't need to pull).
> >
> > But we've been discussing bpf_dynptr_slice() with Joanne today, and we
> > came to the conclusion that bpf_dynptr_slice()/bpf_dynptr_slice_rw()
> > should work for any kind of dynptr (LOCAL, RINGBUF, SKB, XDP). So
> > generic code that wants to work with any dynptr would be able to just
> > use bpf_dynptr_slice, even for LOCAL/RINGBUF, even though buffer won't
> > ever be filled for LOCAL/RINGBUF.
>
> great
>
> > In application, though, if I know I'm working with LOCAL or RINGBUF
> > (or MALLOC, once we have it), I'd use bpf_dynptr_data() to fill out
> > fixed parts, of course. bpf_dynptr_slice() would be cumbersome for
> > such cases (especially if I have some huge fixed part that I *know* is
> > available in RINGBUF/MALLOC case).
>
> bpf_dynptr_data() for local and ringbuf is fine, of course.
> It already exists and has to continue working.
> bpf_dynptr_data() for xdp is probably ok as well,
> but bpf_dynptr_data() for skb is problematic.
> data/data_end concept looked great back in 2016 when it was introduced
> and lots of programs were written, but we underestimated the impact
> of driver's copybreak on programs.
> Network parsing progs consume headers one by one and would typically
> be written as:
> if (header > data_end)
> return DROP;
> Some drivers copybreak fixed number of bytes. Others try to be smart
> and copy only headers into linear part of skb.
> The drivers also change. At one point we tried to upgrade the kernel
> and suddenly bpf firewall started blocking valid traffic.
> Turned out the driver copybreak heuristic was changed in that kernel.
> The bpf prog was converted to use skb_load_bytes() and the fire was extinguished.
> It was a hard lesson.
> Others learned the danger of data/data_end the hard way as well.
> Take a look at cloudflare's progs/test_cls_redirect.c.
> It's a complicated combination of data/data_end and skb_load_bytes().
> It's essentially implementing skb_header_pointer.
> I wish we could use bpf_dynptr_slice only and remove data/data_end,
> but we cannot, since it's uapi.
> But we shouldn't repeat the same mistake. If we do bpf_dynptr_data()
> that returns linear part people will be hitting the same fragility and
> difficult to debug bugs.
> bpf_dynptr_data() for XDP is ok-ish, since most of XDP is still
> page-per-packet, but patches to split headers in HW are starting to appear.
> So even for XDP data/data_end concept may bite us.
> Hence my preference is to EINVAL in bpf_dynptr_data() at least for skb,
> since bpf_dynptr_slice() is a strictly better alternative.
This makes sense to me, I will have the next version of this patchset
return -EINVAL if bpf_dynptr_data() is used on a skb or xdp dynptr
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-02-08 20:13 ` Joanne Koong
@ 2023-02-09 0:39 ` Andrii Nakryiko
0 siblings, 0 replies; 44+ messages in thread
From: Andrii Nakryiko @ 2023-02-09 0:39 UTC (permalink / raw)
To: Joanne Koong
Cc: Alexei Starovoitov, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Wed, Feb 8, 2023 at 12:13 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Tue, Feb 7, 2023 at 6:25 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Fri, Feb 03, 2023 at 01:37:46PM -0800, Andrii Nakryiko wrote:
> > > On Thu, Feb 2, 2023 at 3:43 AM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > > >
> > > > On Wed, Feb 1, 2023 at 5:21 PM Andrii Nakryiko
> > > > <andrii.nakryiko@gmail.com> wrote:
> > > > >
> > > > > On Tue, Jan 31, 2023 at 4:40 PM Alexei Starovoitov
> > > > > <alexei.starovoitov@gmail.com> wrote:
> > > > > >
> > > > > > On Tue, Jan 31, 2023 at 04:11:47PM -0800, Andrii Nakryiko wrote:
> > > > > > > >
> > > > > > > > When prog is just parsing the packet it doesn't need to finalize with bpf_dynptr_write.
> > > > > > > > The prog can always write into the pointer followed by if (p == buf) bpf_dynptr_write.
> > > > > > > > No need for rdonly flag, but extra copy is there in case of cloned which
> > > > > > > > could have been avoided with extra rd_only flag.
> > > > > > >
> > > > > > > Yep, given we are designing bpf_dynptr_slice for performance, extra
> > > > > > > copy on reads is unfortunate. ro/rw flag or have separate
> > > > > > > bpf_dynptr_slice_rw vs bpf_dynptr_slice_ro?
> > > > > >
> > > > > > Either flag or two kfuncs sound good to me.
> > > > >
> > > > > Would it make sense to make bpf_dynptr_slice() as read-only variant,
> > > > > and bpf_dynptr_slice_rw() for read/write? I think the common case is
> > > > > read-only, right? And if users mistakenly use bpf_dynptr_slice() for
> > > > > r/w case, they will get a verifier error when trying to write into the
> > > > > returned pointer. While if we make bpf_dynptr_slice() as read-write,
> > > > > users won't realize they are paying a performance penalty for
> > > > > something that they don't actually need.
> > > >
> > > > Makes sense and it matches skb_header_pointer() usage in the kernel
> > > > which is read-only. Since there is no verifier the read-only-ness
> > > > is not enforced, but we can do it.
> > > >
> > > > Looks like we've converged on bpf_dynptr_slice() and bpf_dynptr_slice_rw().
> > > > The question remains what to do with bpf_dynptr_data() backed by skb/xdp.
> > > > Should we return EINVAL to discourage its usage?
> > > > Of course, we can come up with sensible behavior for bpf_dynptr_data(),
> > > > but it will have quirks that will be not easy to document.
> > > > Even with extensive docs the users might be surprised by the behavior.
> > >
> > > I feel like having bpf_dynptr_data() working in the common case for
> > > skb/xdp would be nice (e.g., so basically at least work in cases when
> > > we don't need to pull).
> > >
> > > But we've been discussing bpf_dynptr_slice() with Joanne today, and we
> > > came to the conclusion that bpf_dynptr_slice()/bpf_dynptr_slice_rw()
> > > should work for any kind of dynptr (LOCAL, RINGBUF, SKB, XDP). So
> > > generic code that wants to work with any dynptr would be able to just
> > > use bpf_dynptr_slice, even for LOCAL/RINGBUF, even though buffer won't
> > > ever be filled for LOCAL/RINGBUF.
> >
> > great
> >
> > > In application, though, if I know I'm working with LOCAL or RINGBUF
> > > (or MALLOC, once we have it), I'd use bpf_dynptr_data() to fill out
> > > fixed parts, of course. bpf_dynptr_slice() would be cumbersome for
> > > such cases (especially if I have some huge fixed part that I *know* is
> > > available in RINGBUF/MALLOC case).
> >
> > bpf_dynptr_data() for local and ringbuf is fine, of course.
> > It already exists and has to continue working.
> > bpf_dynptr_data() for xdp is probably ok as well,
> > but bpf_dynptr_data() for skb is problematic.
> > data/data_end concept looked great back in 2016 when it was introduced
> > and lots of programs were written, but we underestimated the impact
> > of driver's copybreak on programs.
> > Network parsing progs consume headers one by one and would typically
> > be written as:
> > if (header > data_end)
> > return DROP;
> > Some drivers copybreak fixed number of bytes. Others try to be smart
> > and copy only headers into linear part of skb.
> > The drivers also change. At one point we tried to upgrade the kernel
> > and suddenly bpf firewall started blocking valid traffic.
> > Turned out the driver copybreak heuristic was changed in that kernel.
> > The bpf prog was converted to use skb_load_bytes() and the fire was extinguished.
> > It was a hard lesson.
> > Others learned the danger of data/data_end the hard way as well.
> > Take a look at cloudflare's progs/test_cls_redirect.c.
> > It's a complicated combination of data/data_end and skb_load_bytes().
> > It's essentially implementing skb_header_pointer.
> > I wish we could use bpf_dynptr_slice only and remove data/data_end,
> > but we cannot, since it's uapi.
> > But we shouldn't repeat the same mistake. If we do bpf_dynptr_data()
> > that returns linear part people will be hitting the same fragility and
> > difficult to debug bugs.
> > bpf_dynptr_data() for XDP is ok-ish, since most of XDP is still
> > page-per-packet, but patches to split headers in HW are starting to appear.
> > So even for XDP data/data_end concept may bite us.
> > Hence my preference is to EINVAL in bpf_dynptr_data() at least for skb,
> > since bpf_dynptr_slice() is a strictly better alternative.
>
> This makes sense to me, I will have the next version of this patchset
> return -EINVAL if bpf_dynptr_data() is used on a skb or xdp dynptr
+1, sounds reasonable to me as well
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 1:04 ` Andrii Nakryiko
2023-01-31 1:49 ` Martin KaFai Lau
@ 2023-01-31 18:30 ` Joanne Koong
2023-01-31 19:58 ` Alexei Starovoitov
1 sibling, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 18:30 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: Alexei Starovoitov, Martin KaFai Lau, daniel, andrii, martin.lau,
ast, netdev, memxor, kernel-team, bpf
On Mon, Jan 30, 2023 at 5:04 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> <alexei.starovoitov@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > > On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > > + bool seen_direct_write = env->seen_direct_write;
> > > > +
> > > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > > + else
> > > > + /*
> > > > + * Calling may_access_direct_pkt_data() will set
> > > > + * env->seen_direct_write to true if the skb is
> > > > + * writable. As an optimization, we can ignore
> > > > + * setting env->seen_direct_write.
> > > > + *
> > > > + * env->seen_direct_write is used by skb
> > > > + * programs to determine whether the skb's page
> > > > + * buffers should be cloned. Since data slice
> > > > + * writes would only be to the head, we can skip
> > > > + * this.
> > > > + */
> > > > + env->seen_direct_write = seen_direct_write;
> > > > + }
> > >
> > > [ ... ]
> > >
> > > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > > return ret;
> > > > break;
> > > > case KF_ARG_PTR_TO_DYNPTR:
> > > > + {
> > > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > > +
> > > > if (reg->type != PTR_TO_STACK &&
> > > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > > return -EINVAL;
> > > > }
> > > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > > + else
> > > > + dynptr_arg_type |= MEM_RDONLY;
> > > > +
> > > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > > + meta->func_id);
> > > > if (ret < 0)
> > > > return ret;
> > > > break;
> > > > + }
> > > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > > *cnt = 1;
> > > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > >
> > > Does it need to restore the env->seen_direct_write here also?
> > >
> > > It seems this 'seen_direct_write' saving/restoring is needed now because
> > > 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > > actually writing the packet. Some refactoring can help to avoid issue like
> > > this.
> > >
> > > While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > > needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > > has been written. It should be tracked in 'seen_direct_write'. Take a look
> > > at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > > check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > > in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > > PTR_TO_MEM instead of PTR_TO_PACKET.
> >
> > btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > (nothing is being called by the bpf prog).
> > In this case we don't need to repeat this approach. If so we don't need to
> > set seen_direct_write.
> > Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > And technically we don't need to limit it to skb head. It can handle any off/len.
> > It will work for skb, but there is no equivalent for xdp_pull_data().
> > I don't think we can implement xdp_pull_data in all drivers.
> > That's massive amount of work, but we need to be consistent if we want
> > dynptr to wrap both skb and xdp.
> > We can say dynptr_data is for head only, but we've seen bugs where people
> > had to switch from data/data_end to load_bytes.
> >
> > Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > the packet calling that in bpf_dynptr_data is a heavy hammer.
> >
> > It feels that we need to go back to skb_header_pointer-like discussion.
> > Something like:
> > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > Whether buffer is a part of dynptr or program provided is tbd.
>
> making it hidden within dynptr would make this approach unreliable
> (memory allocations, which can fail, etc). But if we ask users to pass
> it directly, then it should be relatively easy to use in practice with
> some pre-allocated per-CPU buffer:
>
>
> struct {
> __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> __int(max_entries, 1);
> __type(key, int);
> __type(value, char[4096]);
> } scratch SEC(".maps");
>
>
> ...
>
>
> struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> void *p, *buf;
> int zero = 0;
>
> buf = bpf_map_lookup_elem(&scratch, &zero);
> if (!buf) return 0; /* can't happen */
>
> p = bpf_dynptr_slice(dp, off, 16, buf);
> if (p == NULL) {
> /* out of range */
> } else {
> /* work with p directly */
> }
>
> /* if we wrote something to p and it was copied to buffer, write it back */
> if (p == buf) {
> bpf_dynptr_write(dp, buf, 16);
> }
>
>
> We'll just need to teach verifier to make sure that buf is at least 16
> byte long.
I'm confused what the benefit of passing in the buffer is. If it's to
avoid the uncloning, this will still need to happen if the user writes
back the data to the skb (which will be the majority of cases). If
it's to avoid uncloning if the user only reads the data of a writable
prog, then we could add logic in the verifier so that we don't pull
the data in this case; the uncloning might still happen regardless if
another part of the program does a direct write. If the benefit is to
avoid needing to pull the data, then can't the user just use
bpf_dynptr_read, which takes in a buffer?
>
>
> But I wonder if for simple cases when users are mostly sure that they
> are going to access only header data directly we can have an option
> for bpf_dynptr_from_skb() to specify what should be the behavior for
> bpf_dynptr_slice():
>
> - either return NULL for anything that crosses into frags (no
> surprising perf penalty, but surprising NULLs);
> - do bpf_skb_pull_data() if bpf_dynptr_data() needs to point to data
> beyond header (potential perf penalty, but on NULLs, if off+len is
> within packet).
>
> And then bpf_dynptr_from_skb() can accept a flag specifying this
> behavior and store it somewhere in struct bpf_dynptr.
>
> Thoughts?
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 18:30 ` Joanne Koong
@ 2023-01-31 19:58 ` Alexei Starovoitov
2023-01-31 20:47 ` Joanne Koong
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-31 19:58 UTC (permalink / raw)
To: Joanne Koong
Cc: Andrii Nakryiko, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Tue, Jan 31, 2023 at 10:30 AM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 5:04 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > <alexei.starovoitov@gmail.com> wrote:
> > >
> > > On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > > > On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > > > + bool seen_direct_write = env->seen_direct_write;
> > > > > +
> > > > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > > > + else
> > > > > + /*
> > > > > + * Calling may_access_direct_pkt_data() will set
> > > > > + * env->seen_direct_write to true if the skb is
> > > > > + * writable. As an optimization, we can ignore
> > > > > + * setting env->seen_direct_write.
> > > > > + *
> > > > > + * env->seen_direct_write is used by skb
> > > > > + * programs to determine whether the skb's page
> > > > > + * buffers should be cloned. Since data slice
> > > > > + * writes would only be to the head, we can skip
> > > > > + * this.
> > > > > + */
> > > > > + env->seen_direct_write = seen_direct_write;
> > > > > + }
> > > >
> > > > [ ... ]
> > > >
> > > > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > > > return ret;
> > > > > break;
> > > > > case KF_ARG_PTR_TO_DYNPTR:
> > > > > + {
> > > > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > > > +
> > > > > if (reg->type != PTR_TO_STACK &&
> > > > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > > > return -EINVAL;
> > > > > }
> > > > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > > > + else
> > > > > + dynptr_arg_type |= MEM_RDONLY;
> > > > > +
> > > > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > > > + meta->func_id);
> > > > > if (ret < 0)
> > > > > return ret;
> > > > > break;
> > > > > + }
> > > > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > > > *cnt = 1;
> > > > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > >
> > > > Does it need to restore the env->seen_direct_write here also?
> > > >
> > > > It seems this 'seen_direct_write' saving/restoring is needed now because
> > > > 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > > > actually writing the packet. Some refactoring can help to avoid issue like
> > > > this.
> > > >
> > > > While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > > > needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > > > has been written. It should be tracked in 'seen_direct_write'. Take a look
> > > > at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > > > check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > > > in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > > > PTR_TO_MEM instead of PTR_TO_PACKET.
> > >
> > > btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > > (nothing is being called by the bpf prog).
> > > In this case we don't need to repeat this approach. If so we don't need to
> > > set seen_direct_write.
> > > Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > > And technically we don't need to limit it to skb head. It can handle any off/len.
> > > It will work for skb, but there is no equivalent for xdp_pull_data().
> > > I don't think we can implement xdp_pull_data in all drivers.
> > > That's massive amount of work, but we need to be consistent if we want
> > > dynptr to wrap both skb and xdp.
> > > We can say dynptr_data is for head only, but we've seen bugs where people
> > > had to switch from data/data_end to load_bytes.
> > >
> > > Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > > the packet calling that in bpf_dynptr_data is a heavy hammer.
> > >
> > > It feels that we need to go back to skb_header_pointer-like discussion.
> > > Something like:
> > > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > > Whether buffer is a part of dynptr or program provided is tbd.
> >
> > making it hidden within dynptr would make this approach unreliable
> > (memory allocations, which can fail, etc). But if we ask users to pass
> > it directly, then it should be relatively easy to use in practice with
> > some pre-allocated per-CPU buffer:
> >
> >
> > struct {
> > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > __int(max_entries, 1);
> > __type(key, int);
> > __type(value, char[4096]);
> > } scratch SEC(".maps");
> >
> >
> > ...
> >
> >
> > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > void *p, *buf;
> > int zero = 0;
> >
> > buf = bpf_map_lookup_elem(&scratch, &zero);
> > if (!buf) return 0; /* can't happen */
> >
> > p = bpf_dynptr_slice(dp, off, 16, buf);
> > if (p == NULL) {
> > /* out of range */
> > } else {
> > /* work with p directly */
> > }
> >
> > /* if we wrote something to p and it was copied to buffer, write it back */
> > if (p == buf) {
> > bpf_dynptr_write(dp, buf, 16);
> > }
> >
> >
> > We'll just need to teach verifier to make sure that buf is at least 16
> > byte long.
>
> I'm confused what the benefit of passing in the buffer is. If it's to
> avoid the uncloning, this will still need to happen if the user writes
> back the data to the skb (which will be the majority of cases). If
> it's to avoid uncloning if the user only reads the data of a writable
> prog, then we could add logic in the verifier so that we don't pull
> the data in this case; the uncloning might still happen regardless if
> another part of the program does a direct write. If the benefit is to
> avoid needing to pull the data, then can't the user just use
> bpf_dynptr_read, which takes in a buffer?
There is no unclone and there is no pull in xdp.
The main idea of this semantics of bpf_dynptr_slice is to make it
work the same way on skb and xdp for _read_ case.
Writes are going to be different between skb and xdp anyway.
In some rare cases the writes can be the same for skb and xdp
with this bpf_dynptr_slice + bpf_dynptr_write logic,
but that's a minor feature addition of the api.
I'd say in skb cases the progs do reads and either drop
or forward the skb.
Writes to skb are done from time to time too, because
they're a pain to do correctly.
nat is the main use case for skb rewrites.
In xdp cases the progs do parse, drop, rewrite, xmit more or less equally.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 19:58 ` Alexei Starovoitov
@ 2023-01-31 20:47 ` Joanne Koong
2023-01-31 21:10 ` Alexei Starovoitov
0 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 20:47 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Andrii Nakryiko, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Tue, Jan 31, 2023 at 11:59 AM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 10:30 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 5:04 PM Andrii Nakryiko
> > <andrii.nakryiko@gmail.com> wrote:
> > >
> > > On Mon, Jan 30, 2023 at 2:31 PM Alexei Starovoitov
> > > <alexei.starovoitov@gmail.com> wrote:
> > > >
> > > > On Mon, Jan 30, 2023 at 02:04:08PM -0800, Martin KaFai Lau wrote:
> > > > > On 1/27/23 11:17 AM, Joanne Koong wrote:
> > > > > > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > > > > > mark_reg_known_zero(env, regs, BPF_REG_0);
> > > > > > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > > > > > regs[BPF_REG_0].mem_size = meta.mem_size;
> > > > > > + if (func_id == BPF_FUNC_dynptr_data &&
> > > > > > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > > > > > + bool seen_direct_write = env->seen_direct_write;
> > > > > > +
> > > > > > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > > > > > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > > > > > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > > > > > + else
> > > > > > + /*
> > > > > > + * Calling may_access_direct_pkt_data() will set
> > > > > > + * env->seen_direct_write to true if the skb is
> > > > > > + * writable. As an optimization, we can ignore
> > > > > > + * setting env->seen_direct_write.
> > > > > > + *
> > > > > > + * env->seen_direct_write is used by skb
> > > > > > + * programs to determine whether the skb's page
> > > > > > + * buffers should be cloned. Since data slice
> > > > > > + * writes would only be to the head, we can skip
> > > > > > + * this.
> > > > > > + */
> > > > > > + env->seen_direct_write = seen_direct_write;
> > > > > > + }
> > > > >
> > > > > [ ... ]
> > > > >
> > > > > > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > > > > > return ret;
> > > > > > break;
> > > > > > case KF_ARG_PTR_TO_DYNPTR:
> > > > > > + {
> > > > > > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > > > > > +
> > > > > > if (reg->type != PTR_TO_STACK &&
> > > > > > reg->type != CONST_PTR_TO_DYNPTR) {
> > > > > > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > > > > > return -EINVAL;
> > > > > > }
> > > > > > - ret = process_dynptr_func(env, regno, insn_idx,
> > > > > > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > > > > > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > > > > > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > > > > > + else
> > > > > > + dynptr_arg_type |= MEM_RDONLY;
> > > > > > +
> > > > > > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > > > > > + meta->func_id);
> > > > > > if (ret < 0)
> > > > > > return ret;
> > > > > > break;
> > > > > > + }
> > > > > > case KF_ARG_PTR_TO_LIST_HEAD:
> > > > > > if (reg->type != PTR_TO_MAP_VALUE &&
> > > > > > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > > > > > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > > > > > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > > > > > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > > > > > *cnt = 1;
> > > > > > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > > > > > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
> > > > >
> > > > > Does it need to restore the env->seen_direct_write here also?
> > > > >
> > > > > It seems this 'seen_direct_write' saving/restoring is needed now because
> > > > > 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is
> > > > > actually writing the packet. Some refactoring can help to avoid issue like
> > > > > this.
> > > > >
> > > > > While at 'seen_direct_write', Alexei has also pointed out that the verifier
> > > > > needs to track whether the (packet) 'slice' returned by bpf_dynptr_data()
> > > > > has been written. It should be tracked in 'seen_direct_write'. Take a look
> > > > > at how reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> > > > > check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere
> > > > > in v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> > > > > PTR_TO_MEM instead of PTR_TO_PACKET.
> > > >
> > > > btw tc progs are using gen_prologue() approach because data/data_end are not kfuncs
> > > > (nothing is being called by the bpf prog).
> > > > In this case we don't need to repeat this approach. If so we don't need to
> > > > set seen_direct_write.
> > > > Instead bpf_dynptr_data() can call bpf_skb_pull_data() directly.
> > > > And technically we don't need to limit it to skb head. It can handle any off/len.
> > > > It will work for skb, but there is no equivalent for xdp_pull_data().
> > > > I don't think we can implement xdp_pull_data in all drivers.
> > > > That's massive amount of work, but we need to be consistent if we want
> > > > dynptr to wrap both skb and xdp.
> > > > We can say dynptr_data is for head only, but we've seen bugs where people
> > > > had to switch from data/data_end to load_bytes.
> > > >
> > > > Also bpf_skb_pull_data is quite heavy. For progs that only want to parse
> > > > the packet calling that in bpf_dynptr_data is a heavy hammer.
> > > >
> > > > It feels that we need to go back to skb_header_pointer-like discussion.
> > > > Something like:
> > > > bpf_dynptr_slice(const struct bpf_dynptr *ptr, u32 offset, u32 len, void *buffer)
> > > > Whether buffer is a part of dynptr or program provided is tbd.
> > >
> > > making it hidden within dynptr would make this approach unreliable
> > > (memory allocations, which can fail, etc). But if we ask users to pass
> > > it directly, then it should be relatively easy to use in practice with
> > > some pre-allocated per-CPU buffer:
> > >
> > >
> > > struct {
> > > __int(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> > > __int(max_entries, 1);
> > > __type(key, int);
> > > __type(value, char[4096]);
> > > } scratch SEC(".maps");
> > >
> > >
> > > ...
> > >
> > >
> > > struct dyn_ptr *dp = bpf_dynptr_from_skb(...).
> > > void *p, *buf;
> > > int zero = 0;
> > >
> > > buf = bpf_map_lookup_elem(&scratch, &zero);
> > > if (!buf) return 0; /* can't happen */
> > >
> > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > if (p == NULL) {
> > > /* out of range */
> > > } else {
> > > /* work with p directly */
> > > }
> > >
> > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > if (p == buf) {
> > > bpf_dynptr_write(dp, buf, 16);
> > > }
> > >
> > >
> > > We'll just need to teach verifier to make sure that buf is at least 16
> > > byte long.
> >
> > I'm confused what the benefit of passing in the buffer is. If it's to
> > avoid the uncloning, this will still need to happen if the user writes
> > back the data to the skb (which will be the majority of cases). If
> > it's to avoid uncloning if the user only reads the data of a writable
> > prog, then we could add logic in the verifier so that we don't pull
> > the data in this case; the uncloning might still happen regardless if
> > another part of the program does a direct write. If the benefit is to
> > avoid needing to pull the data, then can't the user just use
> > bpf_dynptr_read, which takes in a buffer?
>
> There is no unclone and there is no pull in xdp.
> The main idea of this semantics of bpf_dynptr_slice is to make it
> work the same way on skb and xdp for _read_ case.
> Writes are going to be different between skb and xdp anyway.
> In some rare cases the writes can be the same for skb and xdp
> with this bpf_dynptr_slice + bpf_dynptr_write logic,
> but that's a minor feature addition of the api.
bpf_dynptr_read works the same way on skb and xdp. bpf_dynptr_read
takes in a buffer as well, so what is the added benefit of
bpf_dynptr_slice?
>
> I'd say in skb cases the progs do reads and either drop
> or forward the skb.
> Writes to skb are done from time to time too, because
> they're a pain to do correctly.
> nat is the main use case for skb rewrites.
> In xdp cases the progs do parse, drop, rewrite, xmit more or less equally.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 20:47 ` Joanne Koong
@ 2023-01-31 21:10 ` Alexei Starovoitov
2023-01-31 21:33 ` Joanne Koong
0 siblings, 1 reply; 44+ messages in thread
From: Alexei Starovoitov @ 2023-01-31 21:10 UTC (permalink / raw)
To: Joanne Koong
Cc: Andrii Nakryiko, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Tue, Jan 31, 2023 at 12:48 PM Joanne Koong <joannelkoong@gmail.com> wrote:
> > > >
> > > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > > if (p == NULL) {
> > > > /* out of range */
> > > > } else {
> > > > /* work with p directly */
> > > > }
> > > >
> > > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > > if (p == buf) {
> > > > bpf_dynptr_write(dp, buf, 16);
> > > > }
> > > >
> > > >
> > > > We'll just need to teach verifier to make sure that buf is at least 16
> > > > byte long.
> > >
> > > I'm confused what the benefit of passing in the buffer is. If it's to
> > > avoid the uncloning, this will still need to happen if the user writes
> > > back the data to the skb (which will be the majority of cases). If
> > > it's to avoid uncloning if the user only reads the data of a writable
> > > prog, then we could add logic in the verifier so that we don't pull
> > > the data in this case; the uncloning might still happen regardless if
> > > another part of the program does a direct write. If the benefit is to
> > > avoid needing to pull the data, then can't the user just use
> > > bpf_dynptr_read, which takes in a buffer?
> >
> > There is no unclone and there is no pull in xdp.
> > The main idea of this semantics of bpf_dynptr_slice is to make it
> > work the same way on skb and xdp for _read_ case.
> > Writes are going to be different between skb and xdp anyway.
> > In some rare cases the writes can be the same for skb and xdp
> > with this bpf_dynptr_slice + bpf_dynptr_write logic,
> > but that's a minor feature addition of the api.
>
> bpf_dynptr_read works the same way on skb and xdp. bpf_dynptr_read
> takes in a buffer as well, so what is the added benefit of
> bpf_dynptr_slice?
That it doesn't copy most of the time.
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 21:10 ` Alexei Starovoitov
@ 2023-01-31 21:33 ` Joanne Koong
0 siblings, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 21:33 UTC (permalink / raw)
To: Alexei Starovoitov
Cc: Andrii Nakryiko, Martin KaFai Lau, Daniel Borkmann,
Andrii Nakryiko, Martin KaFai Lau, Alexei Starovoitov,
Network Development, Kumar Kartikeya Dwivedi, Kernel Team, bpf
On Tue, Jan 31, 2023 at 1:11 PM Alexei Starovoitov
<alexei.starovoitov@gmail.com> wrote:
>
> On Tue, Jan 31, 2023 at 12:48 PM Joanne Koong <joannelkoong@gmail.com> wrote:
> > > > >
> > > > > p = bpf_dynptr_slice(dp, off, 16, buf);
> > > > > if (p == NULL) {
> > > > > /* out of range */
> > > > > } else {
> > > > > /* work with p directly */
> > > > > }
> > > > >
> > > > > /* if we wrote something to p and it was copied to buffer, write it back */
> > > > > if (p == buf) {
> > > > > bpf_dynptr_write(dp, buf, 16);
> > > > > }
> > > > >
> > > > >
> > > > > We'll just need to teach verifier to make sure that buf is at least 16
> > > > > byte long.
> > > >
> > > > I'm confused what the benefit of passing in the buffer is. If it's to
> > > > avoid the uncloning, this will still need to happen if the user writes
> > > > back the data to the skb (which will be the majority of cases). If
> > > > it's to avoid uncloning if the user only reads the data of a writable
> > > > prog, then we could add logic in the verifier so that we don't pull
> > > > the data in this case; the uncloning might still happen regardless if
> > > > another part of the program does a direct write. If the benefit is to
> > > > avoid needing to pull the data, then can't the user just use
> > > > bpf_dynptr_read, which takes in a buffer?
> > >
> > > There is no unclone and there is no pull in xdp.
> > > The main idea of this semantics of bpf_dynptr_slice is to make it
> > > work the same way on skb and xdp for _read_ case.
> > > Writes are going to be different between skb and xdp anyway.
> > > In some rare cases the writes can be the same for skb and xdp
> > > with this bpf_dynptr_slice + bpf_dynptr_write logic,
> > > but that's a minor feature addition of the api.
> >
> > bpf_dynptr_read works the same way on skb and xdp. bpf_dynptr_read
> > takes in a buffer as well, so what is the added benefit of
> > bpf_dynptr_slice?
>
> That it doesn't copy most of the time.
Ohh I see, I missed that bpf_dynptr_slice also returns back a ptr.
This makes sense to me now, thanks for clarifying.
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-30 22:04 ` Martin KaFai Lau
2023-01-30 22:31 ` Alexei Starovoitov
@ 2023-01-31 18:18 ` Joanne Koong
1 sibling, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 18:18 UTC (permalink / raw)
To: Martin KaFai Lau
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team, bpf
On Mon, Jan 30, 2023 at 2:04 PM Martin KaFai Lau <martin.lau@linux.dev> wrote:
>
> On 1/27/23 11:17 AM, Joanne Koong wrote:
> > @@ -8243,6 +8316,28 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
> > mark_reg_known_zero(env, regs, BPF_REG_0);
> > regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
> > regs[BPF_REG_0].mem_size = meta.mem_size;
> > + if (func_id == BPF_FUNC_dynptr_data &&
> > + dynptr_type == BPF_DYNPTR_TYPE_SKB) {
> > + bool seen_direct_write = env->seen_direct_write;
> > +
> > + regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
> > + if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
> > + regs[BPF_REG_0].type |= MEM_RDONLY;
> > + else
> > + /*
> > + * Calling may_access_direct_pkt_data() will set
> > + * env->seen_direct_write to true if the skb is
> > + * writable. As an optimization, we can ignore
> > + * setting env->seen_direct_write.
> > + *
> > + * env->seen_direct_write is used by skb
> > + * programs to determine whether the skb's page
> > + * buffers should be cloned. Since data slice
> > + * writes would only be to the head, we can skip
> > + * this.
> > + */
> > + env->seen_direct_write = seen_direct_write;
> > + }
>
> [ ... ]
>
> > @@ -9263,17 +9361,26 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
> > return ret;
> > break;
> > case KF_ARG_PTR_TO_DYNPTR:
> > + {
> > + enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
> > +
> > if (reg->type != PTR_TO_STACK &&
> > reg->type != CONST_PTR_TO_DYNPTR) {
> > verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
> > return -EINVAL;
> > }
> >
> > - ret = process_dynptr_func(env, regno, insn_idx,
> > - ARG_PTR_TO_DYNPTR | MEM_RDONLY);
> > + if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
> > + dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
> > + else
> > + dynptr_arg_type |= MEM_RDONLY;
> > +
> > + ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type,
> > + meta->func_id);
> > if (ret < 0)
> > return ret;
> > break;
> > + }
> > case KF_ARG_PTR_TO_LIST_HEAD:
> > if (reg->type != PTR_TO_MAP_VALUE &&
> > reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
> > @@ -15857,6 +15964,14 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
> > desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
> > insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
> > *cnt = 1;
> > + } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
> > + bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
>
> Does it need to restore the env->seen_direct_write here also?
>
> It seems this 'seen_direct_write' saving/restoring is needed now because
> 'may_access_direct_pkt_data(BPF_WRITE)' is not only called when it is actually
> writing the packet. Some refactoring can help to avoid issue like this.
Yes! Great catch! I'll submit a patch that refactors this, so that
env->seen_direct_write isn't set implicitly within
may_access_direct_pkt_data()
>
> While at 'seen_direct_write', Alexei has also pointed out that the verifier
> needs to track whether the (packet) 'slice' returned by bpf_dynptr_data() has
> been written. It should be tracked in 'seen_direct_write'. Take a look at how
> reg_is_pkt_pointer() and may_access_direct_pkt_data() are done in
> check_mem_access(). iirc, this reg_is_pkt_pointer() part got loss somewhere in
> v5 (or v4?) when bpf_dynptr_data() was changed to return register typed
> PTR_TO_MEM instead of PTR_TO_PACKET.
>
The verifier right now does track whether the dynptr skb 'slice' is
writable or not and sets seen_direct_write accordingly. However, it
currently does it in check_helper_call() where if the bpf program is
writable, then the env->seen_direct_write is set (regardless of
whether actual writes occur or not), so I like your idea of moving
this to check_mem_access(). The PTR_TO_MEM that gets returned for the
data slice will need to be tagged with DYNPTR_TYPE_SKB.
>
> [ ... ]
>
> > +int bpf_dynptr_from_skb(struct sk_buff *skb, u64 flags,
> > + struct bpf_dynptr_kern *ptr, int is_rdonly)
>
> hmm... this exposed kfunc takes "int is_rdonly".
>
> What if the bpf prog calls it like bpf_dynptr_from_skb(..., false) in some hook
> that is not writable to packet?
If the bpf prog tries to do this, their "false" value will be ignored,
because the "int is_rdonly" arg value gets set by the verifier (in
fixup_kfunc_call() in line 15969)
>
> > +{
> > + if (flags) {
> > + bpf_dynptr_set_null(ptr);
> > + return -EINVAL;
> > + }
> > +
> > + bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
> > +
> > + if (is_rdonly)
> > + bpf_dynptr_set_rdonly(ptr);
> > +
> > + return 0;
> > +}
> > +
> > BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
> > {
> > return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
> > @@ -11607,3 +11634,28 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
> >
> > return func;
> > }
> > +
> > +BTF_SET8_START(bpf_kfunc_check_set_skb)
> > +BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
> > +BTF_SET8_END(bpf_kfunc_check_set_skb)
> > +
> > +static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
> > + .owner = THIS_MODULE,
> > + .set = &bpf_kfunc_check_set_skb,
> > +};
> > +
> > +static int __init bpf_kfunc_init(void)
> > +{
> > + int ret;
> > +
> > + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
> > + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
> > + return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
> > +}
> > +late_initcall(bpf_kfunc_init);
>
>
^ permalink raw reply [flat|nested] 44+ messages in thread
* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-27 19:17 ` [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs Joanne Koong
2023-01-29 23:39 ` Alexei Starovoitov
2023-01-30 22:04 ` Martin KaFai Lau
@ 2023-01-31 0:48 ` Andrii Nakryiko
2023-01-31 0:55 ` Joanne Koong
2 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 0:48 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> Add skb dynptrs, which are dynptrs whose underlying pointer points
> to a skb. The dynptr acts on skb data. skb dynptrs have two main
> benefits. One is that they allow operations on sizes that are not
> statically known at compile-time (eg variable-sized accesses).
> Another is that parsing the packet data through dynptrs (instead of
> through direct access of skb->data and skb->data_end) can be more
> ergonomic and less brittle (eg does not need manual if checking for
> being within bounds of data_end).
>
> For bpf prog types that don't support writes on skb data, the dynptr is
> read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> will return a data slice that is read-only where any writes to it will
> be rejected by the verifier).
>
> For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> interfaces, reading and writing from/to data in the head as well as from/to
> non-linear paged buffers is supported. For data slices (through the
> bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> must first call bpf_skb_pull_data() to pull the data into the linear
> portion.
>
> Any bpf_dynptr_write() automatically invalidates any prior data slices
> to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> to data in a paged buffer, so it will need to pull the buffer first into
> the head. The reason it needs to be pulled instead of writing directly to
> the paged buffers is because they may be cloned (only the head of the skb
> is by default uncloned). As such, any bpf_dynptr_write() will
> automatically have its prior data slices invalidated, even if the write
> is to data in the skb head (the verifier has no way of differentiating
> whether the write is to the head or paged buffers during program load
> time). Please note as well that any other helper calls that change the
> underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> slices of the skb dynptr as well. The stack trace for this is
> check_helper_call() -> clear_all_pkt_pointers() ->
> __clear_all_pkt_pointers() -> mark_reg_unknown().
>
> For examples of how skb dynptrs can be used, please see the attached
> selftests.
>
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
> include/linux/bpf.h | 82 +++++++++------
> include/linux/filter.h | 18 ++++
> include/uapi/linux/bpf.h | 37 +++++--
> kernel/bpf/btf.c | 18 ++++
> kernel/bpf/helpers.c | 95 ++++++++++++++---
> kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> net/core/filter.c | 60 ++++++++++-
> tools/include/uapi/linux/bpf.h | 37 +++++--
> 8 files changed, 432 insertions(+), 100 deletions(-)
>
[...]
> static const struct bpf_func_proto bpf_dynptr_write_proto = {
> @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
>
> BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> {
> + enum bpf_dynptr_type type;
> + void *data;
> int err;
>
> if (!ptr->data)
> @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> if (err)
> return 0;
>
> - if (bpf_dynptr_is_rdonly(ptr))
> - return 0;
> + type = bpf_dynptr_get_type(ptr);
> +
> + switch (type) {
> + case BPF_DYNPTR_TYPE_LOCAL:
> + case BPF_DYNPTR_TYPE_RINGBUF:
> + if (bpf_dynptr_is_rdonly(ptr))
> + return 0;
will something break if we return ptr->data for read-only LOCAL/RINGBUF dynptr?
> +
> + data = ptr->data;
> + break;
> + case BPF_DYNPTR_TYPE_SKB:
> + {
> + struct sk_buff *skb = ptr->data;
>
[...]
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 0:48 ` Andrii Nakryiko
@ 2023-01-31 0:55 ` Joanne Koong
2023-01-31 1:06 ` Andrii Nakryiko
0 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 0:55 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 4:48 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > benefits. One is that they allow operations on sizes that are not
> > statically known at compile-time (eg variable-sized accesses).
> > Another is that parsing the packet data through dynptrs (instead of
> > through direct access of skb->data and skb->data_end) can be more
> > ergonomic and less brittle (eg does not need manual if checking for
> > being within bounds of data_end).
> >
> > For bpf prog types that don't support writes on skb data, the dynptr is
> > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > will return a data slice that is read-only where any writes to it will
> > be rejected by the verifier).
> >
> > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > interfaces, reading and writing from/to data in the head as well as from/to
> > non-linear paged buffers is supported. For data slices (through the
> > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > must first call bpf_skb_pull_data() to pull the data into the linear
> > portion.
> >
> > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > to data in a paged buffer, so it will need to pull the buffer first into
> > the head. The reason it needs to be pulled instead of writing directly to
> > the paged buffers is because they may be cloned (only the head of the skb
> > is by default uncloned). As such, any bpf_dynptr_write() will
> > automatically have its prior data slices invalidated, even if the write
> > is to data in the skb head (the verifier has no way of differentiating
> > whether the write is to the head or paged buffers during program load
> > time). Please note as well that any other helper calls that change the
> > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > slices of the skb dynptr as well. The stack trace for this is
> > check_helper_call() -> clear_all_pkt_pointers() ->
> > __clear_all_pkt_pointers() -> mark_reg_unknown().
> >
> > For examples of how skb dynptrs can be used, please see the attached
> > selftests.
> >
> > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > ---
> > include/linux/bpf.h | 82 +++++++++------
> > include/linux/filter.h | 18 ++++
> > include/uapi/linux/bpf.h | 37 +++++--
> > kernel/bpf/btf.c | 18 ++++
> > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > net/core/filter.c | 60 ++++++++++-
> > tools/include/uapi/linux/bpf.h | 37 +++++--
> > 8 files changed, 432 insertions(+), 100 deletions(-)
> >
>
> [...]
>
> > static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
> >
> > BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> > {
> > + enum bpf_dynptr_type type;
> > + void *data;
> > int err;
> >
> > if (!ptr->data)
> > @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> > if (err)
> > return 0;
> >
> > - if (bpf_dynptr_is_rdonly(ptr))
> > - return 0;
> > + type = bpf_dynptr_get_type(ptr);
> > +
> > + switch (type) {
> > + case BPF_DYNPTR_TYPE_LOCAL:
> > + case BPF_DYNPTR_TYPE_RINGBUF:
> > + if (bpf_dynptr_is_rdonly(ptr))
> > + return 0;
>
> will something break if we return ptr->data for read-only LOCAL/RINGBUF dynptr?
There will be nothing guarding against direct writes into read-only
LOCAL/RINGBUF dynptrs if we return ptr->data. For skb type dynptrs,
it's guarded by the ptr->data return pointer being marked as
MEM_RDONLY in the verifier if the skb is non-writable.
>
> > +
> > + data = ptr->data;
> > + break;
> > + case BPF_DYNPTR_TYPE_SKB:
> > + {
> > + struct sk_buff *skb = ptr->data;
> >
>
> [...]
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 0:55 ` Joanne Koong
@ 2023-01-31 1:06 ` Andrii Nakryiko
2023-01-31 1:13 ` Joanne Koong
0 siblings, 1 reply; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 1:06 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 4:56 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 4:48 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> > On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> > >
> > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > benefits. One is that they allow operations on sizes that are not
> > > statically known at compile-time (eg variable-sized accesses).
> > > Another is that parsing the packet data through dynptrs (instead of
> > > through direct access of skb->data and skb->data_end) can be more
> > > ergonomic and less brittle (eg does not need manual if checking for
> > > being within bounds of data_end).
> > >
> > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > will return a data slice that is read-only where any writes to it will
> > > be rejected by the verifier).
> > >
> > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > interfaces, reading and writing from/to data in the head as well as from/to
> > > non-linear paged buffers is supported. For data slices (through the
> > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > portion.
> > >
> > > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > > to data in a paged buffer, so it will need to pull the buffer first into
> > > the head. The reason it needs to be pulled instead of writing directly to
> > > the paged buffers is because they may be cloned (only the head of the skb
> > > is by default uncloned). As such, any bpf_dynptr_write() will
> > > automatically have its prior data slices invalidated, even if the write
> > > is to data in the skb head (the verifier has no way of differentiating
> > > whether the write is to the head or paged buffers during program load
> > > time). Please note as well that any other helper calls that change the
> > > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > > slices of the skb dynptr as well. The stack trace for this is
> > > check_helper_call() -> clear_all_pkt_pointers() ->
> > > __clear_all_pkt_pointers() -> mark_reg_unknown().
> > >
> > > For examples of how skb dynptrs can be used, please see the attached
> > > selftests.
> > >
> > > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > > ---
> > > include/linux/bpf.h | 82 +++++++++------
> > > include/linux/filter.h | 18 ++++
> > > include/uapi/linux/bpf.h | 37 +++++--
> > > kernel/bpf/btf.c | 18 ++++
> > > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > > net/core/filter.c | 60 ++++++++++-
> > > tools/include/uapi/linux/bpf.h | 37 +++++--
> > > 8 files changed, 432 insertions(+), 100 deletions(-)
> > >
> >
> > [...]
> >
> > > static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > > @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > >
> > > BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> > > {
> > > + enum bpf_dynptr_type type;
> > > + void *data;
> > > int err;
> > >
> > > if (!ptr->data)
> > > @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> > > if (err)
> > > return 0;
> > >
> > > - if (bpf_dynptr_is_rdonly(ptr))
> > > - return 0;
> > > + type = bpf_dynptr_get_type(ptr);
> > > +
> > > + switch (type) {
> > > + case BPF_DYNPTR_TYPE_LOCAL:
> > > + case BPF_DYNPTR_TYPE_RINGBUF:
> > > + if (bpf_dynptr_is_rdonly(ptr))
> > > + return 0;
> >
> > will something break if we return ptr->data for read-only LOCAL/RINGBUF dynptr?
>
> There will be nothing guarding against direct writes into read-only
> LOCAL/RINGBUF dynptrs if we return ptr->data. For skb type dynptrs,
> it's guarded by the ptr->data return pointer being marked as
> MEM_RDONLY in the verifier if the skb is non-writable.
>
Ah, so we won't add MEM_RDONLY for bpf_dynptr_data()'s returned
PTR_TO_MEM if we know (statically) that dynptr is read-only?
Ok, not a big deal, just something that we might want to improve in the future.
> >
> > > +
> > > + data = ptr->data;
> > > + break;
> > > + case BPF_DYNPTR_TYPE_SKB:
> > > + {
> > > + struct sk_buff *skb = ptr->data;
> > >
> >
> > [...]
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 1:06 ` Andrii Nakryiko
@ 2023-01-31 1:13 ` Joanne Koong
2023-01-31 1:19 ` Andrii Nakryiko
0 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-31 1:13 UTC (permalink / raw)
To: Andrii Nakryiko
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 5:06 PM Andrii Nakryiko
<andrii.nakryiko@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 4:56 PM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 4:48 PM Andrii Nakryiko
> > <andrii.nakryiko@gmail.com> wrote:
> > >
> > > On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> > > >
> > > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > > benefits. One is that they allow operations on sizes that are not
> > > > statically known at compile-time (eg variable-sized accesses).
> > > > Another is that parsing the packet data through dynptrs (instead of
> > > > through direct access of skb->data and skb->data_end) can be more
> > > > ergonomic and less brittle (eg does not need manual if checking for
> > > > being within bounds of data_end).
> > > >
> > > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > > will return a data slice that is read-only where any writes to it will
> > > > be rejected by the verifier).
> > > >
> > > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > > interfaces, reading and writing from/to data in the head as well as from/to
> > > > non-linear paged buffers is supported. For data slices (through the
> > > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > > portion.
> > > >
> > > > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > > > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > > > to data in a paged buffer, so it will need to pull the buffer first into
> > > > the head. The reason it needs to be pulled instead of writing directly to
> > > > the paged buffers is because they may be cloned (only the head of the skb
> > > > is by default uncloned). As such, any bpf_dynptr_write() will
> > > > automatically have its prior data slices invalidated, even if the write
> > > > is to data in the skb head (the verifier has no way of differentiating
> > > > whether the write is to the head or paged buffers during program load
> > > > time). Please note as well that any other helper calls that change the
> > > > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > > > slices of the skb dynptr as well. The stack trace for this is
> > > > check_helper_call() -> clear_all_pkt_pointers() ->
> > > > __clear_all_pkt_pointers() -> mark_reg_unknown().
> > > >
> > > > For examples of how skb dynptrs can be used, please see the attached
> > > > selftests.
> > > >
> > > > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > > > ---
> > > > include/linux/bpf.h | 82 +++++++++------
> > > > include/linux/filter.h | 18 ++++
> > > > include/uapi/linux/bpf.h | 37 +++++--
> > > > kernel/bpf/btf.c | 18 ++++
> > > > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > > > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > > > net/core/filter.c | 60 ++++++++++-
> > > > tools/include/uapi/linux/bpf.h | 37 +++++--
> > > > 8 files changed, 432 insertions(+), 100 deletions(-)
> > > >
> > >
> > > [...]
> > >
> > > > static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > > > @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > > >
> > > > BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> > > > {
> > > > + enum bpf_dynptr_type type;
> > > > + void *data;
> > > > int err;
> > > >
> > > > if (!ptr->data)
> > > > @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> > > > if (err)
> > > > return 0;
> > > >
> > > > - if (bpf_dynptr_is_rdonly(ptr))
> > > > - return 0;
> > > > + type = bpf_dynptr_get_type(ptr);
> > > > +
> > > > + switch (type) {
> > > > + case BPF_DYNPTR_TYPE_LOCAL:
> > > > + case BPF_DYNPTR_TYPE_RINGBUF:
> > > > + if (bpf_dynptr_is_rdonly(ptr))
> > > > + return 0;
> > >
> > > will something break if we return ptr->data for read-only LOCAL/RINGBUF dynptr?
> >
> > There will be nothing guarding against direct writes into read-only
> > LOCAL/RINGBUF dynptrs if we return ptr->data. For skb type dynptrs,
> > it's guarded by the ptr->data return pointer being marked as
> > MEM_RDONLY in the verifier if the skb is non-writable.
> >
>
> Ah, so we won't add MEM_RDONLY for bpf_dynptr_data()'s returned
> PTR_TO_MEM if we know (statically) that dynptr is read-only?
I think you meant will, not won't? If so, then yes we only add
MEM_RDONLY for the returned data slice if we can pre-determine that
the dynptr is read-only, else bpf_dynptr_data() will return null.
>
> Ok, not a big deal, just something that we might want to improve in the future.
I'm curious to hear how you think this could be improved. If we're not
able to know statically whether the dynptr is read-only or writable,
then there's no way to enforce it in the verifier before the bpf
program runs. Or is there some way to do this?
>
> > >
> > > > +
> > > > + data = ptr->data;
> > > > + break;
> > > > + case BPF_DYNPTR_TYPE_SKB:
> > > > + {
> > > > + struct sk_buff *skb = ptr->data;
> > > >
> > >
> > > [...]
^ permalink raw reply [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs
2023-01-31 1:13 ` Joanne Koong
@ 2023-01-31 1:19 ` Andrii Nakryiko
0 siblings, 0 replies; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 1:19 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Mon, Jan 30, 2023 at 5:13 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> On Mon, Jan 30, 2023 at 5:06 PM Andrii Nakryiko
> <andrii.nakryiko@gmail.com> wrote:
> >
> > On Mon, Jan 30, 2023 at 4:56 PM Joanne Koong <joannelkoong@gmail.com> wrote:
> > >
> > > On Mon, Jan 30, 2023 at 4:48 PM Andrii Nakryiko
> > > <andrii.nakryiko@gmail.com> wrote:
> > > >
> > > > On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
> > > > >
> > > > > Add skb dynptrs, which are dynptrs whose underlying pointer points
> > > > > to a skb. The dynptr acts on skb data. skb dynptrs have two main
> > > > > benefits. One is that they allow operations on sizes that are not
> > > > > statically known at compile-time (eg variable-sized accesses).
> > > > > Another is that parsing the packet data through dynptrs (instead of
> > > > > through direct access of skb->data and skb->data_end) can be more
> > > > > ergonomic and less brittle (eg does not need manual if checking for
> > > > > being within bounds of data_end).
> > > > >
> > > > > For bpf prog types that don't support writes on skb data, the dynptr is
> > > > > read-only (bpf_dynptr_write() will return an error and bpf_dynptr_data()
> > > > > will return a data slice that is read-only where any writes to it will
> > > > > be rejected by the verifier).
> > > > >
> > > > > For reads and writes through the bpf_dynptr_read() and bpf_dynptr_write()
> > > > > interfaces, reading and writing from/to data in the head as well as from/to
> > > > > non-linear paged buffers is supported. For data slices (through the
> > > > > bpf_dynptr_data() interface), if the data is in a paged buffer, the user
> > > > > must first call bpf_skb_pull_data() to pull the data into the linear
> > > > > portion.
> > > > >
> > > > > Any bpf_dynptr_write() automatically invalidates any prior data slices
> > > > > to the skb dynptr. This is because a bpf_dynptr_write() may be writing
> > > > > to data in a paged buffer, so it will need to pull the buffer first into
> > > > > the head. The reason it needs to be pulled instead of writing directly to
> > > > > the paged buffers is because they may be cloned (only the head of the skb
> > > > > is by default uncloned). As such, any bpf_dynptr_write() will
> > > > > automatically have its prior data slices invalidated, even if the write
> > > > > is to data in the skb head (the verifier has no way of differentiating
> > > > > whether the write is to the head or paged buffers during program load
> > > > > time). Please note as well that any other helper calls that change the
> > > > > underlying packet buffer (eg bpf_skb_pull_data()) invalidates any data
> > > > > slices of the skb dynptr as well. The stack trace for this is
> > > > > check_helper_call() -> clear_all_pkt_pointers() ->
> > > > > __clear_all_pkt_pointers() -> mark_reg_unknown().
> > > > >
> > > > > For examples of how skb dynptrs can be used, please see the attached
> > > > > selftests.
> > > > >
> > > > > Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> > > > > ---
> > > > > include/linux/bpf.h | 82 +++++++++------
> > > > > include/linux/filter.h | 18 ++++
> > > > > include/uapi/linux/bpf.h | 37 +++++--
> > > > > kernel/bpf/btf.c | 18 ++++
> > > > > kernel/bpf/helpers.c | 95 ++++++++++++++---
> > > > > kernel/bpf/verifier.c | 185 ++++++++++++++++++++++++++-------
> > > > > net/core/filter.c | 60 ++++++++++-
> > > > > tools/include/uapi/linux/bpf.h | 37 +++++--
> > > > > 8 files changed, 432 insertions(+), 100 deletions(-)
> > > > >
> > > >
> > > > [...]
> > > >
> > > > > static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > > > > @@ -1560,6 +1595,8 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
> > > > >
> > > > > BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u32, len)
> > > > > {
> > > > > + enum bpf_dynptr_type type;
> > > > > + void *data;
> > > > > int err;
> > > > >
> > > > > if (!ptr->data)
> > > > > @@ -1569,10 +1606,36 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
> > > > > if (err)
> > > > > return 0;
> > > > >
> > > > > - if (bpf_dynptr_is_rdonly(ptr))
> > > > > - return 0;
> > > > > + type = bpf_dynptr_get_type(ptr);
> > > > > +
> > > > > + switch (type) {
> > > > > + case BPF_DYNPTR_TYPE_LOCAL:
> > > > > + case BPF_DYNPTR_TYPE_RINGBUF:
> > > > > + if (bpf_dynptr_is_rdonly(ptr))
> > > > > + return 0;
> > > >
> > > > will something break if we return ptr->data for read-only LOCAL/RINGBUF dynptr?
> > >
> > > There will be nothing guarding against direct writes into read-only
> > > LOCAL/RINGBUF dynptrs if we return ptr->data. For skb type dynptrs,
> > > it's guarded by the ptr->data return pointer being marked as
> > > MEM_RDONLY in the verifier if the skb is non-writable.
> > >
> >
> > Ah, so we won't add MEM_RDONLY for bpf_dynptr_data()'s returned
> > PTR_TO_MEM if we know (statically) that dynptr is read-only?
>
> I think you meant will, not won't? If so, then yes we only add
> MEM_RDONLY for the returned data slice if we can pre-determine that
> the dynptr is read-only, else bpf_dynptr_data() will return null.
>
> >
> > Ok, not a big deal, just something that we might want to improve in the future.
>
> I'm curious to hear how you think this could be improved. If we're not
> able to know statically whether the dynptr is read-only or writable,
> then there's no way to enforce it in the verifier before the bpf
> program runs. Or is there some way to do this?
I might be just confused, I thought the conclusion from previous
discussions were that we do know statically if dynptr is read-only? If
that's not the case, then yeah, we can't really do much about this.
Either way, I think this is a small thing, as in practice
LOCAL/RINGBUF dynptrs will always be read-write, right?
>
> >
> > > >
> > > > > +
> > > > > + data = ptr->data;
> > > > > + break;
> > > > > + case BPF_DYNPTR_TYPE_SKB:
> > > > > + {
> > > > > + struct sk_buff *skb = ptr->data;
> > > > >
> > > >
> > > > [...]
^ permalink raw reply [flat|nested] 44+ messages in thread
* [PATCH v9 bpf-next 4/5] bpf: Add xdp dynptrs
2023-01-27 19:16 [PATCH v9 bpf-next 0/5] Add skb + xdp dynptrs Joanne Koong
` (2 preceding siblings ...)
2023-01-27 19:17 ` [PATCH v9 bpf-next 3/5] bpf: Add skb dynptrs Joanne Koong
@ 2023-01-27 19:17 ` Joanne Koong
2023-01-27 19:17 ` [PATCH v9 bpf-next 5/5] selftests/bpf: tests for using dynptrs to parse skb and xdp buffers Joanne Koong
4 siblings, 0 replies; 44+ messages in thread
From: Joanne Koong @ 2023-01-27 19:17 UTC (permalink / raw)
To: bpf
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team,
Joanne Koong
Add xdp dynptrs, which are dynptrs whose underlying pointer points
to a xdp_buff. The dynptr acts on xdp data. xdp dynptrs have two main
benefits. One is that they allow operations on sizes that are not
statically known at compile-time (eg variable-sized accesses).
Another is that parsing the packet data through dynptrs (instead of
through direct access of xdp->data and xdp->data_end) can be more
ergonomic and less brittle (eg does not need manual if checking for
being within bounds of data_end).
For reads and writes on the dynptr, this includes reading/writing
from/to and across fragments. For data slices, direct access to
data in fragments is also permitted, but access across fragments
is not.
Any helper calls that change the underlying packet buffer (eg
bpf_xdp_adjust_head) invalidates any data slices of the associated
dynptr. The stack trace for this is check_helper_call() ->
clear_all_pkt_pointers() -> __clear_all_pkt_pointers() ->
mark_reg_unknown().
For examples of how xdp dynptrs can be used, please see the attached
selftests.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
include/linux/bpf.h | 8 ++++-
include/linux/filter.h | 20 ++++++++++++
include/uapi/linux/bpf.h | 10 ++++--
kernel/bpf/helpers.c | 12 +++++++
kernel/bpf/verifier.c | 57 +++++++++++++++++++++-------------
net/core/filter.c | 46 ++++++++++++++++++++++-----
tools/include/uapi/linux/bpf.h | 10 ++++--
7 files changed, 127 insertions(+), 36 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 1ac061b64582..94e910d9598e 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -578,11 +578,15 @@ enum bpf_type_flag {
/* DYNPTR points to sk_buff */
DYNPTR_TYPE_SKB = BIT(14 + BPF_BASE_TYPE_BITS),
+ /* DYNPTR points to xdp_buff */
+ DYNPTR_TYPE_XDP = BIT(15 + BPF_BASE_TYPE_BITS),
+
__BPF_TYPE_FLAG_MAX,
__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
};
-#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB)
+#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
+ | DYNPTR_TYPE_XDP)
/* Max number of base types. */
#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
@@ -1109,6 +1113,8 @@ enum bpf_dynptr_type {
BPF_DYNPTR_TYPE_RINGBUF,
/* Underlying data is a sk_buff */
BPF_DYNPTR_TYPE_SKB,
+ /* Underlying data is a xdp_buff */
+ BPF_DYNPTR_TYPE_XDP,
};
int bpf_dynptr_check_size(u32 size);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index c87d13954d89..674795ea9d20 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1545,6 +1545,9 @@ static __always_inline int __bpf_xdp_redirect_map(struct bpf_map *map, u64 index
int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
u32 len, u64 flags);
+int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
+int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len);
+void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len);
#else /* CONFIG_NET */
static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset,
void *to, u32 len)
@@ -1557,6 +1560,23 @@ static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset,
{
return -EOPNOTSUPP;
}
+
+static inline int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset,
+ void *buf, u32 len)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset,
+ void *buf, u32 len)
+{
+ return -EOPNOTSUPP;
+}
+
+static inline void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
+{
+ return NULL;
+}
#endif /* CONFIG_NET */
#endif /* __LINUX_FILTER_H__ */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index f6910392d339..9cc988e063de 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5352,13 +5352,17 @@ union bpf_attr {
* and try again.
*
* * The data slice is automatically invalidated anytime
- * **bpf_dynptr_write**\ () or a helper call that changes
- * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
+ * **bpf_dynptr_write**\ () is called.
+ *
+ * For skb-type and xdp-type dynptrs:
+ * * The data slice is automatically invalidated anytime a
+ * helper call that changes the underlying packet buffer
+ * (eg **bpf_skb_pull_data**\ (), **bpf_xdp_adjust_head**\ ())
* is called.
* Return
* Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
* or if the offset and length is out of bounds or in a paged buffer for
- * skb-type dynptrs.
+ * skb-type dynptrs or across fragments for xdp-type dynptrs.
*
* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
* Description
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index a79d522b3a26..ede199107016 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1530,6 +1530,8 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern
return 0;
case BPF_DYNPTR_TYPE_SKB:
return __bpf_skb_load_bytes(src->data, src->offset + offset, dst, len);
+ case BPF_DYNPTR_TYPE_XDP:
+ return __bpf_xdp_load_bytes(src->data, src->offset + offset, dst, len);
default:
WARN_ONCE(true, "bpf_dynptr_read: unknown dynptr type %d\n", type);
return -EFAULT;
@@ -1576,6 +1578,10 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v
case BPF_DYNPTR_TYPE_SKB:
return __bpf_skb_store_bytes(dst->data, dst->offset + offset, src, len,
flags);
+ case BPF_DYNPTR_TYPE_XDP:
+ if (flags)
+ return -EINVAL;
+ return __bpf_xdp_store_bytes(dst->data, dst->offset + offset, src, len);
default:
WARN_ONCE(true, "bpf_dynptr_write: unknown dynptr type %d\n", type);
return -EFAULT;
@@ -1631,6 +1637,12 @@ BPF_CALL_3(bpf_dynptr_data, const struct bpf_dynptr_kern *, ptr, u32, offset, u3
data = skb->data;
break;
}
+ case BPF_DYNPTR_TYPE_XDP:
+ /* if the requested data in across fragments, then it cannot
+ * be accessed directly - bpf_xdp_pointer will return NULL
+ */
+ return (unsigned long)bpf_xdp_pointer(ptr->data,
+ ptr->offset + offset, len);
default:
WARN_ONCE(true, "bpf_dynptr_data: unknown dynptr type %d\n", type);
return 0;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3b022abc34e3..fce06cb44598 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -743,6 +743,8 @@ static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
return BPF_DYNPTR_TYPE_RINGBUF;
case DYNPTR_TYPE_SKB:
return BPF_DYNPTR_TYPE_SKB;
+ case DYNPTR_TYPE_XDP:
+ return BPF_DYNPTR_TYPE_XDP;
default:
return BPF_DYNPTR_TYPE_INVALID;
}
@@ -1630,7 +1632,7 @@ static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
{
return base_type(reg->type) == PTR_TO_MEM &&
- reg->type & DYNPTR_TYPE_SKB;
+ (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP);
}
/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
@@ -6244,6 +6246,9 @@ int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
case DYNPTR_TYPE_SKB:
err_extra = "skb ";
break;
+ case DYNPTR_TYPE_XDP:
+ err_extra = "xdp ";
+ break;
default:
err_extra = "<unknown>";
break;
@@ -7319,7 +7324,7 @@ static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
* are now invalid, so turn them into unknown SCALAR_VALUE.
*
- * This also applies to dynptr slices belonging to skb dynptrs,
+ * This also applies to dynptr slices belonging to skb or xdp dynptrs,
* since these slices point to packet data.
*/
static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
@@ -8316,27 +8321,30 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
regs[BPF_REG_0].mem_size = meta.mem_size;
- if (func_id == BPF_FUNC_dynptr_data &&
- dynptr_type == BPF_DYNPTR_TYPE_SKB) {
- bool seen_direct_write = env->seen_direct_write;
+ if (func_id == BPF_FUNC_dynptr_data) {
+ if (dynptr_type == BPF_DYNPTR_TYPE_SKB) {
+ bool seen_direct_write = env->seen_direct_write;
- regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
- if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
- regs[BPF_REG_0].type |= MEM_RDONLY;
- else
- /*
- * Calling may_access_direct_pkt_data() will set
- * env->seen_direct_write to true if the skb is
- * writable. As an optimization, we can ignore
- * setting env->seen_direct_write.
- *
- * env->seen_direct_write is used by skb
- * programs to determine whether the skb's page
- * buffers should be cloned. Since data slice
- * writes would only be to the head, we can skip
- * this.
- */
- env->seen_direct_write = seen_direct_write;
+ regs[BPF_REG_0].type |= DYNPTR_TYPE_SKB;
+ if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE))
+ regs[BPF_REG_0].type |= MEM_RDONLY;
+ else
+ /*
+ * Calling may_access_direct_pkt_data() will set
+ * env->seen_direct_write to true if the skb is
+ * writable. As an optimization, we can ignore
+ * setting env->seen_direct_write.
+ *
+ * env->seen_direct_write is used by skb
+ * programs to determine whether the skb's page
+ * buffers should be cloned. Since data slice
+ * writes would only be to the head, we can skip
+ * this.
+ */
+ env->seen_direct_write = seen_direct_write;
+ } else if (dynptr_type == BPF_DYNPTR_TYPE_XDP) {
+ regs[BPF_REG_0].type |= DYNPTR_TYPE_XDP;
+ }
}
break;
case RET_PTR_TO_MEM_OR_BTF_ID:
@@ -8745,6 +8753,7 @@ enum special_kfunc_type {
KF_bpf_cast_to_kern_ctx,
KF_bpf_rdonly_cast,
KF_bpf_dynptr_from_skb,
+ KF_bpf_dynptr_from_xdp,
KF_bpf_rcu_read_lock,
KF_bpf_rcu_read_unlock,
};
@@ -8759,6 +8768,7 @@ BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
BTF_ID(func, bpf_dynptr_from_skb)
+BTF_ID(func, bpf_dynptr_from_xdp)
BTF_SET_END(special_kfunc_set)
BTF_ID_LIST(special_kfunc_list)
@@ -8771,6 +8781,7 @@ BTF_ID(func, bpf_list_pop_back)
BTF_ID(func, bpf_cast_to_kern_ctx)
BTF_ID(func, bpf_rdonly_cast)
BTF_ID(func, bpf_dynptr_from_skb)
+BTF_ID(func, bpf_dynptr_from_xdp)
BTF_ID(func, bpf_rcu_read_lock)
BTF_ID(func, bpf_rcu_read_unlock)
@@ -9372,6 +9383,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb])
dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_SKB;
+ else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp])
+ dynptr_arg_type |= MEM_UNINIT | DYNPTR_TYPE_XDP;
else
dynptr_arg_type |= MEM_RDONLY;
diff --git a/net/core/filter.c b/net/core/filter.c
index ddb47126071a..fb77d0c9bad6 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3855,7 +3855,19 @@ static const struct bpf_func_proto sk_skb_change_head_proto = {
.arg3_type = ARG_ANYTHING,
};
-BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
+int bpf_dynptr_from_xdp(struct xdp_buff *xdp, u64 flags, struct bpf_dynptr_kern *ptr)
+{
+ if (flags) {
+ bpf_dynptr_set_null(ptr);
+ return -EINVAL;
+ }
+
+ bpf_dynptr_init(ptr, xdp, BPF_DYNPTR_TYPE_XDP, 0, xdp_get_buff_len(xdp));
+
+ return 0;
+}
+
+BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
{
return xdp_get_buff_len(xdp);
}
@@ -3957,7 +3969,7 @@ static void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
}
}
-static void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
+void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
{
struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
u32 size = xdp->data_end - xdp->data;
@@ -3988,8 +4000,7 @@ static void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
return offset + len <= size ? addr + offset : NULL;
}
-BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
- void *, buf, u32, len)
+int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
{
void *ptr;
@@ -4005,6 +4016,12 @@ BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
return 0;
}
+BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
+ void *, buf, u32, len)
+{
+ return __bpf_xdp_load_bytes(xdp, offset, buf, len);
+}
+
static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
.func = bpf_xdp_load_bytes,
.gpl_only = false,
@@ -4015,8 +4032,7 @@ static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
.arg4_type = ARG_CONST_SIZE,
};
-BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
- void *, buf, u32, len)
+int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
{
void *ptr;
@@ -4032,6 +4048,12 @@ BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
return 0;
}
+BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
+ void *, buf, u32, len)
+{
+ return __bpf_xdp_store_bytes(xdp, offset, buf, len);
+}
+
static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {
.func = bpf_xdp_store_bytes,
.gpl_only = false,
@@ -11635,6 +11657,10 @@ bpf_sk_base_func_proto(enum bpf_func_id func_id)
return func;
}
+BTF_SET8_START(bpf_kfunc_check_set_xdp)
+BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
+BTF_SET8_END(bpf_kfunc_check_set_xdp)
+
BTF_SET8_START(bpf_kfunc_check_set_skb)
BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
BTF_SET8_END(bpf_kfunc_check_set_skb)
@@ -11644,6 +11670,11 @@ static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
.set = &bpf_kfunc_check_set_skb,
};
+static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
+ .owner = THIS_MODULE,
+ .set = &bpf_kfunc_check_set_xdp,
+};
+
static int __init bpf_kfunc_init(void)
{
int ret;
@@ -11656,6 +11687,7 @@ static int __init bpf_kfunc_init(void)
ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
- return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
+ ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
+ return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
}
late_initcall(bpf_kfunc_init);
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6b58e5a75fc5..04aa7d0b6437 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5352,13 +5352,17 @@ union bpf_attr {
* and try again.
*
* * The data slice is automatically invalidated anytime
- * **bpf_dynptr_write**\ () or a helper call that changes
- * the underlying packet buffer (eg **bpf_skb_pull_data**\ ())
+ * **bpf_dynptr_write**\ () is called.
+ *
+ * For skb-type and xdp-type dynptrs:
+ * * The data slice is automatically invalidated anytime a
+ * helper call that changes the underlying packet buffer
+ * (eg **bpf_skb_pull_data**\ (), **bpf_xdp_adjust_head**\ ())
* is called.
* Return
* Pointer to the underlying dynptr data, NULL if the dynptr is invalid,
* or if the offset and length is out of bounds or in a paged buffer for
- * skb-type dynptrs.
+ * skb-type dynptrs or across fragments for xdp-type dynptrs.
*
* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len)
* Description
--
2.30.2
^ permalink raw reply related [flat|nested] 44+ messages in thread* [PATCH v9 bpf-next 5/5] selftests/bpf: tests for using dynptrs to parse skb and xdp buffers
2023-01-27 19:16 [PATCH v9 bpf-next 0/5] Add skb + xdp dynptrs Joanne Koong
` (3 preceding siblings ...)
2023-01-27 19:17 ` [PATCH v9 bpf-next 4/5] bpf: Add xdp dynptrs Joanne Koong
@ 2023-01-27 19:17 ` Joanne Koong
2023-01-31 0:49 ` Andrii Nakryiko
4 siblings, 1 reply; 44+ messages in thread
From: Joanne Koong @ 2023-01-27 19:17 UTC (permalink / raw)
To: bpf
Cc: daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team,
Joanne Koong
Test skb and xdp dynptr functionality in the following ways:
1) progs/test_cls_redirect_dynptr.c
* Rewrite "progs/test_cls_redirect.c" test to use dynptrs to parse
skb data
* This is a great example of how dynptrs can be used to simplify a
lot of the parsing logic for non-statically known values.
When measuring the user + system time between the original version
vs. using dynptrs, and averaging the time for 10 runs (using
"time ./test_progs -t cls_redirect"):
original version: 0.092 sec
with dynptrs: 0.078 sec
2) progs/test_xdp_dynptr.c
* Rewrite "progs/test_xdp.c" test to use dynptrs to parse xdp data
When measuring the user + system time between the original version
vs. using dynptrs, and averaging the time for 10 runs (using
"time ./test_progs -t xdp_attach"):
original version: 0.118 sec
with dynptrs: 0.094 sec
3) progs/test_l4lb_noinline_dynptr.c
* Rewrite "progs/test_l4lb_noinline.c" test to use dynptrs to parse
skb data
When measuring the user + system time between the original version
vs. using dynptrs, and averaging the time for 10 runs (using
"time ./test_progs -t l4lb_all"):
original version: 0.062 sec
with dynptrs: 0.081 sec
For number of processed verifier instructions:
original version: 6268 insns
with dynptrs: 2588 insns
4) progs/test_parse_tcp_hdr_opt_dynptr.c
* Add sample code for parsing tcp hdr opt lookup using dynptrs.
This logic is lifted from a real-world use case of packet parsing
in katran [0], a layer 4 load balancer. The original version
"progs/test_parse_tcp_hdr_opt.c" (not using dynptrs) is included
here as well, for comparison.
When measuring the user + system time between the original version
vs. using dynptrs, and averaging the time for 10 runs (using
"time ./test_progs -t parse_tcp_hdr_opt"):
original version: 0.031 sec
with dynptrs: 0.045 sec
5) progs/dynptr_success.c
* Add test case "test_skb_readonly" for testing attempts at writes /
data slices on a prog type with read-only skb ctx.
6) progs/dynptr_fail.c
* Add test cases "skb_invalid_data_slice{1,2}" and
"xdp_invalid_data_slice" for testing that helpers that modify the
underlying packet buffer automatically invalidate the associated
data slice.
* Add test cases "skb_invalid_ctx" and "xdp_invalid_ctx" for testing
that prog types that do not support bpf_dynptr_from_skb/xdp don't
have access to the API.
* Add test case "skb_invalid_write" for testing that writes to a
read-only data slice are rejected by the verifier.
[0]
https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/pckt_parsing.h
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
---
.../selftests/bpf/prog_tests/cls_redirect.c | 25 +
.../testing/selftests/bpf/prog_tests/dynptr.c | 63 +-
.../selftests/bpf/prog_tests/l4lb_all.c | 2 +
.../bpf/prog_tests/parse_tcp_hdr_opt.c | 93 ++
.../selftests/bpf/prog_tests/xdp_attach.c | 11 +-
.../testing/selftests/bpf/progs/dynptr_fail.c | 124 +++
.../selftests/bpf/progs/dynptr_success.c | 28 +
.../bpf/progs/test_cls_redirect_dynptr.c | 973 ++++++++++++++++++
.../bpf/progs/test_l4lb_noinline_dynptr.c | 474 +++++++++
.../bpf/progs/test_parse_tcp_hdr_opt.c | 119 +++
.../bpf/progs/test_parse_tcp_hdr_opt_dynptr.c | 112 ++
.../selftests/bpf/progs/test_xdp_dynptr.c | 237 +++++
.../selftests/bpf/test_tcp_hdr_options.h | 1 +
13 files changed, 2248 insertions(+), 14 deletions(-)
create mode 100644 tools/testing/selftests/bpf/prog_tests/parse_tcp_hdr_opt.c
create mode 100644 tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c
create mode 100644 tools/testing/selftests/bpf/progs/test_l4lb_noinline_dynptr.c
create mode 100644 tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt.c
create mode 100644 tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt_dynptr.c
create mode 100644 tools/testing/selftests/bpf/progs/test_xdp_dynptr.c
diff --git a/tools/testing/selftests/bpf/prog_tests/cls_redirect.c b/tools/testing/selftests/bpf/prog_tests/cls_redirect.c
index 224f016b0a53..2a55f717fc07 100644
--- a/tools/testing/selftests/bpf/prog_tests/cls_redirect.c
+++ b/tools/testing/selftests/bpf/prog_tests/cls_redirect.c
@@ -13,6 +13,7 @@
#include "progs/test_cls_redirect.h"
#include "test_cls_redirect.skel.h"
+#include "test_cls_redirect_dynptr.skel.h"
#include "test_cls_redirect_subprogs.skel.h"
#define ENCAP_IP INADDR_LOOPBACK
@@ -446,6 +447,28 @@ static void test_cls_redirect_common(struct bpf_program *prog)
close_fds((int *)conns, sizeof(conns) / sizeof(conns[0][0]));
}
+static void test_cls_redirect_dynptr(void)
+{
+ struct test_cls_redirect_dynptr *skel;
+ int err;
+
+ skel = test_cls_redirect_dynptr__open();
+ if (!ASSERT_OK_PTR(skel, "skel_open"))
+ return;
+
+ skel->rodata->ENCAPSULATION_IP = htonl(ENCAP_IP);
+ skel->rodata->ENCAPSULATION_PORT = htons(ENCAP_PORT);
+
+ err = test_cls_redirect_dynptr__load(skel);
+ if (!ASSERT_OK(err, "skel_load"))
+ goto cleanup;
+
+ test_cls_redirect_common(skel->progs.cls_redirect);
+
+cleanup:
+ test_cls_redirect_dynptr__destroy(skel);
+}
+
static void test_cls_redirect_inlined(void)
{
struct test_cls_redirect *skel;
@@ -496,4 +519,6 @@ void test_cls_redirect(void)
test_cls_redirect_inlined();
if (test__start_subtest("cls_redirect_subprogs"))
test_cls_redirect_subprogs();
+ if (test__start_subtest("cls_redirect_dynptr"))
+ test_cls_redirect_dynptr();
}
diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c
index 7faaf6d9e0d4..4b6102a39e69 100644
--- a/tools/testing/selftests/bpf/prog_tests/dynptr.c
+++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c
@@ -2,20 +2,29 @@
/* Copyright (c) 2022 Facebook */
#include <test_progs.h>
+#include <network_helpers.h>
#include "dynptr_fail.skel.h"
#include "dynptr_success.skel.h"
+enum test_setup_type {
+ /* no set up is required. the prog will just be loaded */
+ SETUP_NONE,
+ SETUP_SYSCALL_SLEEP,
+ SETUP_SKB_PROG,
+};
+
static struct {
const char *prog_name;
- const char *expected_err_msg;
+ enum test_setup_type type;
} dynptr_tests[] = {
/* success cases */
- {"test_read_write", NULL},
- {"test_data_slice", NULL},
- {"test_ringbuf", NULL},
+ {"test_read_write", SETUP_SYSCALL_SLEEP},
+ {"test_data_slice", SETUP_SYSCALL_SLEEP},
+ {"test_ringbuf", SETUP_SYSCALL_SLEEP},
+ {"test_skb_readonly", SETUP_SKB_PROG},
};
-static void verify_success(const char *prog_name)
+static void verify_success(const char *prog_name, enum test_setup_type setup_type)
{
struct dynptr_success *skel;
struct bpf_program *prog;
@@ -35,15 +44,45 @@ static void verify_success(const char *prog_name)
if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
goto cleanup;
- link = bpf_program__attach(prog);
- if (!ASSERT_OK_PTR(link, "bpf_program__attach"))
- goto cleanup;
+ switch (setup_type) {
+ case SETUP_SYSCALL_SLEEP:
+ link = bpf_program__attach(prog);
+ if (!ASSERT_OK_PTR(link, "bpf_program__attach"))
+ goto cleanup;
- usleep(1);
+ usleep(1);
- ASSERT_EQ(skel->bss->err, 0, "err");
+ bpf_link__destroy(link);
+ break;
+ case SETUP_SKB_PROG:
+ {
+ int prog_fd, err;
+ char buf[64];
+
+ LIBBPF_OPTS(bpf_test_run_opts, topts,
+ .data_in = &pkt_v4,
+ .data_size_in = sizeof(pkt_v4),
+ .data_out = buf,
+ .data_size_out = sizeof(buf),
+ .repeat = 1,
+ );
+
+ prog_fd = bpf_program__fd(prog);
+ if (!ASSERT_GE(prog_fd, 0, "prog_fd"))
+ goto cleanup;
- bpf_link__destroy(link);
+ err = bpf_prog_test_run_opts(prog_fd, &topts);
+
+ if (!ASSERT_OK(err, "test_run"))
+ goto cleanup;
+
+ break;
+ }
+ case SETUP_NONE:
+ ASSERT_EQ(0, 1, "internal error: SETUP_NONE unimplemented");
+ }
+
+ ASSERT_EQ(skel->bss->err, 0, "err");
cleanup:
dynptr_success__destroy(skel);
@@ -57,7 +96,7 @@ void test_dynptr(void)
if (!test__start_subtest(dynptr_tests[i].prog_name))
continue;
- verify_success(dynptr_tests[i].prog_name);
+ verify_success(dynptr_tests[i].prog_name, dynptr_tests[i].type);
}
RUN_TESTS(dynptr_fail);
diff --git a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
index 9c1a18573ffd..1eab286b14fe 100644
--- a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
+++ b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
@@ -93,4 +93,6 @@ void test_l4lb_all(void)
test_l4lb("test_l4lb.bpf.o");
if (test__start_subtest("l4lb_noinline"))
test_l4lb("test_l4lb_noinline.bpf.o");
+ if (test__start_subtest("l4lb_noinline_dynptr"))
+ test_l4lb("test_l4lb_noinline_dynptr.bpf.o");
}
diff --git a/tools/testing/selftests/bpf/prog_tests/parse_tcp_hdr_opt.c b/tools/testing/selftests/bpf/prog_tests/parse_tcp_hdr_opt.c
new file mode 100644
index 000000000000..ab042fdcfe55
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/parse_tcp_hdr_opt.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include <network_helpers.h>
+#include "test_parse_tcp_hdr_opt.skel.h"
+#include "test_parse_tcp_hdr_opt_dynptr.skel.h"
+#include "test_tcp_hdr_options.h"
+
+struct test_pkt {
+ struct ipv6_packet pk6_v6;
+ u8 options[16];
+} __packed;
+
+struct test_pkt pkt = {
+ .pk6_v6.eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
+ .pk6_v6.iph.nexthdr = IPPROTO_TCP,
+ .pk6_v6.iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
+ .pk6_v6.tcp.urg_ptr = 123,
+ .pk6_v6.tcp.doff = 9, /* 16 bytes of options */
+
+ .options = {
+ TCPOPT_MSS, 4, 0x05, 0xB4, TCPOPT_NOP, TCPOPT_NOP,
+ 0, 6, 0, 0, 0, 9, TCPOPT_EOL
+ },
+};
+
+static void test_parse_opt(void)
+{
+ struct test_parse_tcp_hdr_opt *skel;
+ struct bpf_program *prog;
+ char buf[128];
+ int err;
+
+ LIBBPF_OPTS(bpf_test_run_opts, topts,
+ .data_in = &pkt,
+ .data_size_in = sizeof(pkt),
+ .data_out = buf,
+ .data_size_out = sizeof(buf),
+ .repeat = 3,
+ );
+
+ skel = test_parse_tcp_hdr_opt__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+ return;
+
+ pkt.options[6] = skel->rodata->tcp_hdr_opt_kind_tpr;
+ prog = skel->progs.xdp_ingress_v6;
+
+ err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts);
+ ASSERT_OK(err, "ipv6 test_run");
+ ASSERT_EQ(topts.retval, XDP_PASS, "ipv6 test_run retval");
+ ASSERT_EQ(skel->bss->server_id, 0x9000000, "server id");
+
+ test_parse_tcp_hdr_opt__destroy(skel);
+}
+
+static void test_parse_opt_dynptr(void)
+{
+ struct test_parse_tcp_hdr_opt_dynptr *skel;
+ struct bpf_program *prog;
+ char buf[128];
+ int err;
+
+ LIBBPF_OPTS(bpf_test_run_opts, topts,
+ .data_in = &pkt,
+ .data_size_in = sizeof(pkt),
+ .data_out = buf,
+ .data_size_out = sizeof(buf),
+ .repeat = 3,
+ );
+
+ skel = test_parse_tcp_hdr_opt_dynptr__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+ return;
+
+ pkt.options[6] = skel->rodata->tcp_hdr_opt_kind_tpr;
+ prog = skel->progs.xdp_ingress_v6;
+
+ err = bpf_prog_test_run_opts(bpf_program__fd(prog), &topts);
+ ASSERT_OK(err, "ipv6 test_run");
+ ASSERT_EQ(topts.retval, XDP_PASS, "ipv6 test_run retval");
+ ASSERT_EQ(skel->bss->server_id, 0x9000000, "server id");
+
+ test_parse_tcp_hdr_opt_dynptr__destroy(skel);
+}
+
+void test_parse_tcp_hdr_opt(void)
+{
+ if (test__start_subtest("parse_tcp_hdr_opt"))
+ test_parse_opt();
+ if (test__start_subtest("parse_tcp_hdr_opt_dynptr"))
+ test_parse_opt_dynptr();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
index 062fbc8c8e5e..28c453bbb84a 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
@@ -4,11 +4,10 @@
#define IFINDEX_LO 1
#define XDP_FLAGS_REPLACE (1U << 4)
-void serial_test_xdp_attach(void)
+static void serial_test_xdp_attach(const char *file)
{
__u32 duration = 0, id1, id2, id0 = 0, len;
struct bpf_object *obj1, *obj2, *obj3;
- const char *file = "./test_xdp.bpf.o";
struct bpf_prog_info info = {};
int err, fd1, fd2, fd3;
LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
@@ -85,3 +84,11 @@ void serial_test_xdp_attach(void)
out_1:
bpf_object__close(obj1);
}
+
+void test_xdp_attach(void)
+{
+ if (test__start_subtest("xdp_attach"))
+ serial_test_xdp_attach("./test_xdp.bpf.o");
+ if (test__start_subtest("xdp_attach_dynptr"))
+ serial_test_xdp_attach("./test_xdp_dynptr.bpf.o");
+}
diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c
index 5950ad6ec2e6..8cbc3451056e 100644
--- a/tools/testing/selftests/bpf/progs/dynptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c
@@ -5,10 +5,18 @@
#include <string.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
+#include <linux/if_ether.h>
#include "bpf_misc.h"
char _license[] SEC("license") = "GPL";
+extern int bpf_dynptr_from_skb(struct __sk_buff *skb, __u64 flags,
+ struct bpf_dynptr *ptr, int rd_only) __ksym;
+extern int bpf_dynptr_from_xdp(struct xdp_md *xdp, __u64 flags,
+ struct bpf_dynptr *ptr) __ksym;
+
+#define bpf_dynptr_from_skb(skb, flags, ptr) bpf_dynptr_from_skb(skb, flags, ptr, 0)
+
struct test_info {
int x;
struct bpf_dynptr ptr;
@@ -1044,6 +1052,89 @@ int dynptr_read_into_slot(void *ctx)
return 0;
}
+/* The data slice is invalidated whenever a helper changes packet data */
+SEC("?tc")
+__failure __msg("invalid mem access 'scalar'")
+int skb_invalid_data_slice1(struct __sk_buff *skb)
+{
+ struct bpf_dynptr ptr;
+ struct ethhdr *hdr;
+
+ bpf_dynptr_from_skb(skb, 0, &ptr);
+ hdr = bpf_dynptr_data(&ptr, 0, sizeof(*hdr));
+
+ if (bpf_skb_pull_data(skb, skb->len))
+ return SK_DROP;
+
+ if (!hdr)
+ return SK_DROP;
+
+ /* this should fail */
+ hdr->h_proto = 1;
+
+ return SK_PASS;
+}
+
+/* The data slice is invalidated whenever bpf_dynptr_write() is called */
+SEC("?tc")
+__failure __msg("invalid mem access 'scalar'")
+int skb_invalid_data_slice2(struct __sk_buff *skb)
+{
+ char write_data[64] = "hello there, world!!";
+ struct bpf_dynptr ptr;
+ struct ethhdr *hdr;
+
+ bpf_dynptr_from_skb(skb, 0, &ptr);
+ hdr = bpf_dynptr_data(&ptr, 0, sizeof(*hdr));
+
+ bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
+
+ if (!hdr)
+ return SK_DROP;
+
+ /* this should fail */
+ hdr->h_proto = 1;
+
+ return SK_PASS;
+}
+
+/* The data slice is invalidated whenever a helper changes packet data */
+SEC("?xdp")
+__failure __msg("invalid mem access 'scalar'")
+int xdp_invalid_data_slice(struct xdp_md *xdp)
+{
+ struct bpf_dynptr ptr;
+ struct ethhdr *hdr;
+
+ bpf_dynptr_from_xdp(xdp, 0, &ptr);
+ hdr = bpf_dynptr_data(&ptr, 0, sizeof(*hdr));
+ if (!hdr)
+ return SK_DROP;
+
+ hdr->h_proto = 9;
+
+ if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(*hdr)))
+ return XDP_DROP;
+
+ /* this should fail */
+ hdr->h_proto = 1;
+
+ return XDP_PASS;
+}
+
+/* Only supported prog type can create skb-type dynptrs */
+SEC("?raw_tp")
+__failure __msg("calling kernel function bpf_dynptr_from_skb is not allowed")
+int skb_invalid_ctx(void *ctx)
+{
+ struct bpf_dynptr ptr;
+
+ /* this should fail */
+ bpf_dynptr_from_skb(ctx, 0, &ptr);
+
+ return 0;
+}
+
/* Reject writes to dynptr slot for uninit arg */
SEC("?raw_tp")
__failure __msg("potential write to dynptr at off=-16")
@@ -1061,6 +1152,19 @@ int uninit_write_into_slot(void *ctx)
return 0;
}
+/* Only supported prog type can create xdp-type dynptrs */
+SEC("?raw_tp")
+__failure __msg("calling kernel function bpf_dynptr_from_xdp is not allowed")
+int xdp_invalid_ctx(void *ctx)
+{
+ struct bpf_dynptr ptr;
+
+ /* this should fail */
+ bpf_dynptr_from_xdp(ctx, 0, &ptr);
+
+ return 0;
+}
+
static int callback(__u32 index, void *data)
{
*(__u32 *)data = 123;
@@ -1092,3 +1196,23 @@ int invalid_data_slices(void *ctx)
return 0;
}
+
+/* Read-only skb packet buffers can't be written to through data slices */
+SEC("?cgroup_skb/egress")
+__failure __msg("cannot write into rdonly_mem")
+int skb_invalid_write(struct __sk_buff *skb)
+{
+ struct bpf_dynptr ptr;
+ __u64 *data;
+
+ bpf_dynptr_from_skb(skb, 0, &ptr);
+
+ data = bpf_dynptr_data(&ptr, 0, sizeof(*data));
+ if (!data)
+ return 0;
+
+ /* this should fail */
+ *data = 123;
+
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/dynptr_success.c b/tools/testing/selftests/bpf/progs/dynptr_success.c
index 35db7c6c1fc7..7341598bb4bf 100644
--- a/tools/testing/selftests/bpf/progs/dynptr_success.c
+++ b/tools/testing/selftests/bpf/progs/dynptr_success.c
@@ -9,6 +9,11 @@
char _license[] SEC("license") = "GPL";
+extern int bpf_dynptr_from_skb(struct __sk_buff *skb, __u64 flags,
+ struct bpf_dynptr *ptr, int rd_only) __ksym;
+
+#define bpf_dynptr_from_skb(skb, flags, ptr) bpf_dynptr_from_skb(skb, flags, ptr, 0)
+
int pid, err, val;
struct sample {
@@ -163,3 +168,26 @@ int test_ringbuf(void *ctx)
bpf_ringbuf_discard_dynptr(&ptr, 0);
return 0;
}
+
+SEC("cgroup_skb/egress")
+int test_skb_readonly(struct __sk_buff *skb)
+{
+ __u8 write_data[2] = {1, 2};
+ struct bpf_dynptr ptr;
+ __u64 *data;
+ int ret;
+
+ if (bpf_dynptr_from_skb(skb, 0, &ptr)) {
+ err = 1;
+ return 0;
+ }
+
+ /* since cgroup skbs are read only, writes should fail */
+ ret = bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
+ if (ret != -EINVAL) {
+ err = 2;
+ return 0;
+ }
+
+ return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c b/tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c
new file mode 100644
index 000000000000..7add7997d862
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c
@@ -0,0 +1,973 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+// Copyright (c) 2019, 2020 Cloudflare
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <linux/bpf.h>
+#include <linux/icmp.h>
+#include <linux/icmpv6.h>
+#include <linux/if_ether.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/pkt_cls.h>
+#include <linux/tcp.h>
+#include <linux/udp.h>
+
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+
+#include "test_cls_redirect.h"
+
+#define offsetofend(TYPE, MEMBER) \
+ (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER)))
+
+#define IP_OFFSET_MASK (0x1FFF)
+#define IP_MF (0x2000)
+
+char _license[] SEC("license") = "Dual BSD/GPL";
+
+extern int bpf_dynptr_from_skb(struct __sk_buff *skb, __u64 flags,
+ struct bpf_dynptr *ptr, int rd_only) __ksym;
+
+#define bpf_dynptr_from_skb(skb, flags, ptr) bpf_dynptr_from_skb(skb, flags, ptr, 0)
+
+/**
+ * Destination port and IP used for UDP encapsulation.
+ */
+volatile const __be16 ENCAPSULATION_PORT;
+volatile const __be32 ENCAPSULATION_IP;
+
+typedef struct {
+ uint64_t processed_packets_total;
+ uint64_t l3_protocol_packets_total_ipv4;
+ uint64_t l3_protocol_packets_total_ipv6;
+ uint64_t l4_protocol_packets_total_tcp;
+ uint64_t l4_protocol_packets_total_udp;
+ uint64_t accepted_packets_total_syn;
+ uint64_t accepted_packets_total_syn_cookies;
+ uint64_t accepted_packets_total_last_hop;
+ uint64_t accepted_packets_total_icmp_echo_request;
+ uint64_t accepted_packets_total_established;
+ uint64_t forwarded_packets_total_gue;
+ uint64_t forwarded_packets_total_gre;
+
+ uint64_t errors_total_unknown_l3_proto;
+ uint64_t errors_total_unknown_l4_proto;
+ uint64_t errors_total_malformed_ip;
+ uint64_t errors_total_fragmented_ip;
+ uint64_t errors_total_malformed_icmp;
+ uint64_t errors_total_unwanted_icmp;
+ uint64_t errors_total_malformed_icmp_pkt_too_big;
+ uint64_t errors_total_malformed_tcp;
+ uint64_t errors_total_malformed_udp;
+ uint64_t errors_total_icmp_echo_replies;
+ uint64_t errors_total_malformed_encapsulation;
+ uint64_t errors_total_encap_adjust_failed;
+ uint64_t errors_total_encap_buffer_too_small;
+ uint64_t errors_total_redirect_loop;
+ uint64_t errors_total_encap_mtu_violate;
+} metrics_t;
+
+typedef enum {
+ INVALID = 0,
+ UNKNOWN,
+ ECHO_REQUEST,
+ SYN,
+ SYN_COOKIE,
+ ESTABLISHED,
+} verdict_t;
+
+typedef struct {
+ uint16_t src, dst;
+} flow_ports_t;
+
+_Static_assert(
+ sizeof(flow_ports_t) !=
+ offsetofend(struct bpf_sock_tuple, ipv4.dport) -
+ offsetof(struct bpf_sock_tuple, ipv4.sport) - 1,
+ "flow_ports_t must match sport and dport in struct bpf_sock_tuple");
+_Static_assert(
+ sizeof(flow_ports_t) !=
+ offsetofend(struct bpf_sock_tuple, ipv6.dport) -
+ offsetof(struct bpf_sock_tuple, ipv6.sport) - 1,
+ "flow_ports_t must match sport and dport in struct bpf_sock_tuple");
+
+struct iphdr_info {
+ void *hdr;
+ __u64 len;
+};
+
+typedef int ret_t;
+
+/* This is a bit of a hack. We need a return value which allows us to
+ * indicate that the regular flow of the program should continue,
+ * while allowing functions to use XDP_PASS and XDP_DROP, etc.
+ */
+static const ret_t CONTINUE_PROCESSING = -1;
+
+/* Convenience macro to call functions which return ret_t.
+ */
+#define MAYBE_RETURN(x) \
+ do { \
+ ret_t __ret = x; \
+ if (__ret != CONTINUE_PROCESSING) \
+ return __ret; \
+ } while (0)
+
+static bool ipv4_is_fragment(const struct iphdr *ip)
+{
+ uint16_t frag_off = ip->frag_off & bpf_htons(IP_OFFSET_MASK);
+ return (ip->frag_off & bpf_htons(IP_MF)) != 0 || frag_off > 0;
+}
+
+static int pkt_parse_ipv4(struct bpf_dynptr *dynptr, __u64 *offset, struct iphdr *iphdr)
+{
+ if (bpf_dynptr_read(iphdr, sizeof(*iphdr), dynptr, *offset, 0))
+ return -1;
+
+ *offset += sizeof(*iphdr);
+
+ if (iphdr->ihl < 5)
+ return -1;
+
+ /* skip ipv4 options */
+ *offset += (iphdr->ihl - 5) * 4;
+
+ return 0;
+}
+
+/* Parse the L4 ports from a packet, assuming a layout like TCP or UDP. */
+static bool pkt_parse_icmp_l4_ports(struct bpf_dynptr *dynptr, __u64 *offset, flow_ports_t *ports)
+{
+ if (bpf_dynptr_read(ports, sizeof(*ports), dynptr, *offset, 0))
+ return false;
+
+ *offset += sizeof(*ports);
+
+ /* Ports in the L4 headers are reversed, since we are parsing an ICMP
+ * payload which is going towards the eyeball.
+ */
+ uint16_t dst = ports->src;
+ ports->src = ports->dst;
+ ports->dst = dst;
+ return true;
+}
+
+static uint16_t pkt_checksum_fold(uint32_t csum)
+{
+ /* The highest reasonable value for an IPv4 header
+ * checksum requires two folds, so we just do that always.
+ */
+ csum = (csum & 0xffff) + (csum >> 16);
+ csum = (csum & 0xffff) + (csum >> 16);
+ return (uint16_t)~csum;
+}
+
+static void pkt_ipv4_checksum(struct iphdr *iph)
+{
+ iph->check = 0;
+
+ /* An IP header without options is 20 bytes. Two of those
+ * are the checksum, which we always set to zero. Hence,
+ * the maximum accumulated value is 18 / 2 * 0xffff = 0x8fff7,
+ * which fits in 32 bit.
+ */
+ _Static_assert(sizeof(struct iphdr) == 20, "iphdr must be 20 bytes");
+ uint32_t acc = 0;
+ uint16_t *ipw = (uint16_t *)iph;
+
+ for (size_t i = 0; i < sizeof(struct iphdr) / 2; i++)
+ acc += ipw[i];
+
+ iph->check = pkt_checksum_fold(acc);
+}
+
+static bool pkt_skip_ipv6_extension_headers(struct bpf_dynptr *dynptr, __u64 *offset,
+ const struct ipv6hdr *ipv6, uint8_t *upper_proto,
+ bool *is_fragment)
+{
+ /* We understand five extension headers.
+ * https://tools.ietf.org/html/rfc8200#section-4.1 states that all
+ * headers should occur once, except Destination Options, which may
+ * occur twice. Hence we give up after 6 headers.
+ */
+ struct {
+ uint8_t next;
+ uint8_t len;
+ } exthdr = {
+ .next = ipv6->nexthdr,
+ };
+ *is_fragment = false;
+
+ for (int i = 0; i < 6; i++) {
+ switch (exthdr.next) {
+ case IPPROTO_FRAGMENT:
+ *is_fragment = true;
+ /* NB: We don't check that hdrlen == 0 as per spec. */
+ /* fallthrough; */
+
+ case IPPROTO_HOPOPTS:
+ case IPPROTO_ROUTING:
+ case IPPROTO_DSTOPTS:
+ case IPPROTO_MH:
+ if (bpf_dynptr_read(&exthdr, sizeof(exthdr), dynptr, *offset, 0))
+ return false;
+
+ /* hdrlen is in 8-octet units, and excludes the first 8 octets. */
+ *offset += (exthdr.len + 1) * 8;
+
+ /* Decode next header */
+ break;
+
+ default:
+ /* The next header is not one of the known extension
+ * headers, treat it as the upper layer header.
+ *
+ * This handles IPPROTO_NONE.
+ *
+ * Encapsulating Security Payload (50) and Authentication
+ * Header (51) also end up here (and will trigger an
+ * unknown proto error later). They have a custom header
+ * format and seem too esoteric to care about.
+ */
+ *upper_proto = exthdr.next;
+ return true;
+ }
+ }
+
+ /* We never found an upper layer header. */
+ return false;
+}
+
+static int pkt_parse_ipv6(struct bpf_dynptr *dynptr, __u64 *offset, struct ipv6hdr *ipv6,
+ uint8_t *proto, bool *is_fragment)
+{
+ if (bpf_dynptr_read(ipv6, sizeof(*ipv6), dynptr, *offset, 0))
+ return -1;
+
+ *offset += sizeof(*ipv6);
+
+ if (!pkt_skip_ipv6_extension_headers(dynptr, offset, ipv6, proto, is_fragment))
+ return -1;
+
+ return 0;
+}
+
+/* Global metrics, per CPU
+ */
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, unsigned int);
+ __type(value, metrics_t);
+} metrics_map SEC(".maps");
+
+static metrics_t *get_global_metrics(void)
+{
+ uint64_t key = 0;
+ return bpf_map_lookup_elem(&metrics_map, &key);
+}
+
+static ret_t accept_locally(struct __sk_buff *skb, encap_headers_t *encap)
+{
+ const int payload_off =
+ sizeof(*encap) +
+ sizeof(struct in_addr) * encap->unigue.hop_count;
+ int32_t encap_overhead = payload_off - sizeof(struct ethhdr);
+
+ /* Changing the ethertype if the encapsulated packet is ipv6 */
+ if (encap->gue.proto_ctype == IPPROTO_IPV6)
+ encap->eth.h_proto = bpf_htons(ETH_P_IPV6);
+
+ if (bpf_skb_adjust_room(skb, -encap_overhead, BPF_ADJ_ROOM_MAC,
+ BPF_F_ADJ_ROOM_FIXED_GSO |
+ BPF_F_ADJ_ROOM_NO_CSUM_RESET) ||
+ bpf_csum_level(skb, BPF_CSUM_LEVEL_DEC))
+ return TC_ACT_SHOT;
+
+ return bpf_redirect(skb->ifindex, BPF_F_INGRESS);
+}
+
+static ret_t forward_with_gre(struct __sk_buff *skb, struct bpf_dynptr *dynptr,
+ encap_headers_t *encap, struct in_addr *next_hop,
+ metrics_t *metrics)
+{
+ const int payload_off =
+ sizeof(*encap) +
+ sizeof(struct in_addr) * encap->unigue.hop_count;
+ int32_t encap_overhead =
+ payload_off - sizeof(struct ethhdr) - sizeof(struct iphdr);
+ int32_t delta = sizeof(struct gre_base_hdr) - encap_overhead;
+ uint16_t proto = ETH_P_IP;
+ uint32_t mtu_len = 0;
+ encap_gre_t *encap_gre;
+
+ metrics->forwarded_packets_total_gre++;
+
+ /* Loop protection: the inner packet's TTL is decremented as a safeguard
+ * against any forwarding loop. As the only interesting field is the TTL
+ * hop limit for IPv6, it is easier to use bpf_skb_load_bytes/bpf_skb_store_bytes
+ * as they handle the split packets if needed (no need for the data to be
+ * in the linear section).
+ */
+ if (encap->gue.proto_ctype == IPPROTO_IPV6) {
+ proto = ETH_P_IPV6;
+ uint8_t ttl;
+ int rc;
+
+ rc = bpf_skb_load_bytes(
+ skb, payload_off + offsetof(struct ipv6hdr, hop_limit),
+ &ttl, 1);
+ if (rc != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (ttl == 0) {
+ metrics->errors_total_redirect_loop++;
+ return TC_ACT_SHOT;
+ }
+
+ ttl--;
+ rc = bpf_skb_store_bytes(
+ skb, payload_off + offsetof(struct ipv6hdr, hop_limit),
+ &ttl, 1, 0);
+ if (rc != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+ } else {
+ uint8_t ttl;
+ int rc;
+
+ rc = bpf_skb_load_bytes(
+ skb, payload_off + offsetof(struct iphdr, ttl), &ttl,
+ 1);
+ if (rc != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (ttl == 0) {
+ metrics->errors_total_redirect_loop++;
+ return TC_ACT_SHOT;
+ }
+
+ /* IPv4 also has a checksum to patch. While the TTL is only one byte,
+ * this function only works for 2 and 4 bytes arguments (the result is
+ * the same).
+ */
+ rc = bpf_l3_csum_replace(
+ skb, payload_off + offsetof(struct iphdr, check), ttl,
+ ttl - 1, 2);
+ if (rc != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ ttl--;
+ rc = bpf_skb_store_bytes(
+ skb, payload_off + offsetof(struct iphdr, ttl), &ttl, 1,
+ 0);
+ if (rc != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+ }
+
+ if (bpf_check_mtu(skb, skb->ifindex, &mtu_len, delta, 0)) {
+ metrics->errors_total_encap_mtu_violate++;
+ return TC_ACT_SHOT;
+ }
+
+ if (bpf_skb_adjust_room(skb, delta, BPF_ADJ_ROOM_NET,
+ BPF_F_ADJ_ROOM_FIXED_GSO |
+ BPF_F_ADJ_ROOM_NO_CSUM_RESET) ||
+ bpf_csum_level(skb, BPF_CSUM_LEVEL_INC)) {
+ metrics->errors_total_encap_adjust_failed++;
+ return TC_ACT_SHOT;
+ }
+
+ if (bpf_skb_pull_data(skb, sizeof(encap_gre_t))) {
+ metrics->errors_total_encap_buffer_too_small++;
+ return TC_ACT_SHOT;
+ }
+
+ encap_gre = bpf_dynptr_data(dynptr, 0, sizeof(encap_gre_t));
+ if (!encap_gre) {
+ metrics->errors_total_encap_buffer_too_small++;
+ return TC_ACT_SHOT;
+ }
+
+ encap_gre->ip.protocol = IPPROTO_GRE;
+ encap_gre->ip.daddr = next_hop->s_addr;
+ encap_gre->ip.saddr = ENCAPSULATION_IP;
+ encap_gre->ip.tot_len =
+ bpf_htons(bpf_ntohs(encap_gre->ip.tot_len) + delta);
+ encap_gre->gre.flags = 0;
+ encap_gre->gre.protocol = bpf_htons(proto);
+ pkt_ipv4_checksum((void *)&encap_gre->ip);
+
+ return bpf_redirect(skb->ifindex, 0);
+}
+
+static ret_t forward_to_next_hop(struct __sk_buff *skb, struct bpf_dynptr *dynptr,
+ encap_headers_t *encap, struct in_addr *next_hop,
+ metrics_t *metrics)
+{
+ /* swap L2 addresses */
+ /* This assumes that packets are received from a router.
+ * So just swapping the MAC addresses here will make the packet go back to
+ * the router, which will send it to the appropriate machine.
+ */
+ unsigned char temp[ETH_ALEN];
+ memcpy(temp, encap->eth.h_dest, sizeof(temp));
+ memcpy(encap->eth.h_dest, encap->eth.h_source,
+ sizeof(encap->eth.h_dest));
+ memcpy(encap->eth.h_source, temp, sizeof(encap->eth.h_source));
+
+ if (encap->unigue.next_hop == encap->unigue.hop_count - 1 &&
+ encap->unigue.last_hop_gre) {
+ return forward_with_gre(skb, dynptr, encap, next_hop, metrics);
+ }
+
+ metrics->forwarded_packets_total_gue++;
+ uint32_t old_saddr = encap->ip.saddr;
+ encap->ip.saddr = encap->ip.daddr;
+ encap->ip.daddr = next_hop->s_addr;
+ if (encap->unigue.next_hop < encap->unigue.hop_count) {
+ encap->unigue.next_hop++;
+ }
+
+ /* Remove ip->saddr, add next_hop->s_addr */
+ const uint64_t off = offsetof(typeof(*encap), ip.check);
+ int ret = bpf_l3_csum_replace(skb, off, old_saddr, next_hop->s_addr, 4);
+ if (ret < 0) {
+ return TC_ACT_SHOT;
+ }
+
+ return bpf_redirect(skb->ifindex, 0);
+}
+
+static ret_t skip_next_hops(__u64 *offset, int n)
+{
+ __u32 res;
+ switch (n) {
+ case 1:
+ *offset += sizeof(struct in_addr);
+ case 0:
+ return CONTINUE_PROCESSING;
+
+ default:
+ return TC_ACT_SHOT;
+ }
+}
+
+/* Get the next hop from the GLB header.
+ *
+ * Sets next_hop->s_addr to 0 if there are no more hops left.
+ * pkt is positioned just after the variable length GLB header
+ * iff the call is successful.
+ */
+static ret_t get_next_hop(struct bpf_dynptr *dynptr, __u64 *offset, encap_headers_t *encap,
+ struct in_addr *next_hop)
+{
+ if (encap->unigue.next_hop > encap->unigue.hop_count)
+ return TC_ACT_SHOT;
+
+ /* Skip "used" next hops. */
+ MAYBE_RETURN(skip_next_hops(offset, encap->unigue.next_hop));
+
+ if (encap->unigue.next_hop == encap->unigue.hop_count) {
+ /* No more next hops, we are at the end of the GLB header. */
+ next_hop->s_addr = 0;
+ return CONTINUE_PROCESSING;
+ }
+
+ if (bpf_dynptr_read(next_hop, sizeof(*next_hop), dynptr, *offset, 0))
+ return TC_ACT_SHOT;
+
+ *offset += sizeof(*next_hop);
+
+ /* Skip the remainig next hops (may be zero). */
+ return skip_next_hops(offset, encap->unigue.hop_count - encap->unigue.next_hop - 1);
+}
+
+/* Fill a bpf_sock_tuple to be used with the socket lookup functions.
+ * This is a kludge that let's us work around verifier limitations:
+ *
+ * fill_tuple(&t, foo, sizeof(struct iphdr), 123, 321)
+ *
+ * clang will substitue a costant for sizeof, which allows the verifier
+ * to track it's value. Based on this, it can figure out the constant
+ * return value, and calling code works while still being "generic" to
+ * IPv4 and IPv6.
+ */
+static uint64_t fill_tuple(struct bpf_sock_tuple *tuple, void *iph,
+ uint64_t iphlen, uint16_t sport, uint16_t dport)
+{
+ switch (iphlen) {
+ case sizeof(struct iphdr): {
+ struct iphdr *ipv4 = (struct iphdr *)iph;
+ tuple->ipv4.daddr = ipv4->daddr;
+ tuple->ipv4.saddr = ipv4->saddr;
+ tuple->ipv4.sport = sport;
+ tuple->ipv4.dport = dport;
+ return sizeof(tuple->ipv4);
+ }
+
+ case sizeof(struct ipv6hdr): {
+ struct ipv6hdr *ipv6 = (struct ipv6hdr *)iph;
+ memcpy(&tuple->ipv6.daddr, &ipv6->daddr,
+ sizeof(tuple->ipv6.daddr));
+ memcpy(&tuple->ipv6.saddr, &ipv6->saddr,
+ sizeof(tuple->ipv6.saddr));
+ tuple->ipv6.sport = sport;
+ tuple->ipv6.dport = dport;
+ return sizeof(tuple->ipv6);
+ }
+
+ default:
+ return 0;
+ }
+}
+
+static verdict_t classify_tcp(struct __sk_buff *skb, struct bpf_sock_tuple *tuple,
+ uint64_t tuplen, void *iph, struct tcphdr *tcp)
+{
+ struct bpf_sock *sk =
+ bpf_skc_lookup_tcp(skb, tuple, tuplen, BPF_F_CURRENT_NETNS, 0);
+
+ if (sk == NULL)
+ return UNKNOWN;
+
+ if (sk->state != BPF_TCP_LISTEN) {
+ bpf_sk_release(sk);
+ return ESTABLISHED;
+ }
+
+ if (iph != NULL && tcp != NULL) {
+ /* Kludge: we've run out of arguments, but need the length of the ip header. */
+ uint64_t iphlen = sizeof(struct iphdr);
+
+ if (tuplen == sizeof(tuple->ipv6))
+ iphlen = sizeof(struct ipv6hdr);
+
+ if (bpf_tcp_check_syncookie(sk, iph, iphlen, tcp,
+ sizeof(*tcp)) == 0) {
+ bpf_sk_release(sk);
+ return SYN_COOKIE;
+ }
+ }
+
+ bpf_sk_release(sk);
+ return UNKNOWN;
+}
+
+static verdict_t classify_udp(struct __sk_buff *skb, struct bpf_sock_tuple *tuple, uint64_t tuplen)
+{
+ struct bpf_sock *sk =
+ bpf_sk_lookup_udp(skb, tuple, tuplen, BPF_F_CURRENT_NETNS, 0);
+
+ if (sk == NULL)
+ return UNKNOWN;
+
+ if (sk->state == BPF_TCP_ESTABLISHED) {
+ bpf_sk_release(sk);
+ return ESTABLISHED;
+ }
+
+ bpf_sk_release(sk);
+ return UNKNOWN;
+}
+
+static verdict_t classify_icmp(struct __sk_buff *skb, uint8_t proto, struct bpf_sock_tuple *tuple,
+ uint64_t tuplen, metrics_t *metrics)
+{
+ switch (proto) {
+ case IPPROTO_TCP:
+ return classify_tcp(skb, tuple, tuplen, NULL, NULL);
+
+ case IPPROTO_UDP:
+ return classify_udp(skb, tuple, tuplen);
+
+ default:
+ metrics->errors_total_malformed_icmp++;
+ return INVALID;
+ }
+}
+
+static verdict_t process_icmpv4(struct __sk_buff *skb, struct bpf_dynptr *dynptr, __u64 *offset,
+ metrics_t *metrics)
+{
+ struct icmphdr icmp;
+ struct iphdr ipv4;
+
+ if (bpf_dynptr_read(&icmp, sizeof(icmp), dynptr, *offset, 0)) {
+ metrics->errors_total_malformed_icmp++;
+ return INVALID;
+ }
+
+ *offset += sizeof(icmp);
+
+ /* We should never receive encapsulated echo replies. */
+ if (icmp.type == ICMP_ECHOREPLY) {
+ metrics->errors_total_icmp_echo_replies++;
+ return INVALID;
+ }
+
+ if (icmp.type == ICMP_ECHO)
+ return ECHO_REQUEST;
+
+ if (icmp.type != ICMP_DEST_UNREACH || icmp.code != ICMP_FRAG_NEEDED) {
+ metrics->errors_total_unwanted_icmp++;
+ return INVALID;
+ }
+
+ if (pkt_parse_ipv4(dynptr, offset, &ipv4)) {
+ metrics->errors_total_malformed_icmp_pkt_too_big++;
+ return INVALID;
+ }
+
+ /* The source address in the outer IP header is from the entity that
+ * originated the ICMP message. Use the original IP header to restore
+ * the correct flow tuple.
+ */
+ struct bpf_sock_tuple tuple;
+ tuple.ipv4.saddr = ipv4.daddr;
+ tuple.ipv4.daddr = ipv4.saddr;
+
+ if (!pkt_parse_icmp_l4_ports(dynptr, offset, (flow_ports_t *)&tuple.ipv4.sport)) {
+ metrics->errors_total_malformed_icmp_pkt_too_big++;
+ return INVALID;
+ }
+
+ return classify_icmp(skb, ipv4.protocol, &tuple,
+ sizeof(tuple.ipv4), metrics);
+}
+
+static verdict_t process_icmpv6(struct bpf_dynptr *dynptr, __u64 *offset, struct __sk_buff *skb,
+ metrics_t *metrics)
+{
+ struct bpf_sock_tuple tuple;
+ struct ipv6hdr ipv6;
+ struct icmp6hdr icmp6;
+ bool is_fragment;
+ uint8_t l4_proto;
+
+ if (bpf_dynptr_read(&icmp6, sizeof(icmp6), dynptr, *offset, 0)) {
+ metrics->errors_total_malformed_icmp++;
+ return INVALID;
+ }
+
+ /* We should never receive encapsulated echo replies. */
+ if (icmp6.icmp6_type == ICMPV6_ECHO_REPLY) {
+ metrics->errors_total_icmp_echo_replies++;
+ return INVALID;
+ }
+
+ if (icmp6.icmp6_type == ICMPV6_ECHO_REQUEST) {
+ return ECHO_REQUEST;
+ }
+
+ if (icmp6.icmp6_type != ICMPV6_PKT_TOOBIG) {
+ metrics->errors_total_unwanted_icmp++;
+ return INVALID;
+ }
+
+ if (pkt_parse_ipv6(dynptr, offset, &ipv6, &l4_proto, &is_fragment)) {
+ metrics->errors_total_malformed_icmp_pkt_too_big++;
+ return INVALID;
+ }
+
+ if (is_fragment) {
+ metrics->errors_total_fragmented_ip++;
+ return INVALID;
+ }
+
+ /* Swap source and dest addresses. */
+ memcpy(&tuple.ipv6.saddr, &ipv6.daddr, sizeof(tuple.ipv6.saddr));
+ memcpy(&tuple.ipv6.daddr, &ipv6.saddr, sizeof(tuple.ipv6.daddr));
+
+ if (!pkt_parse_icmp_l4_ports(dynptr, offset, (flow_ports_t *)&tuple.ipv6.sport)) {
+ metrics->errors_total_malformed_icmp_pkt_too_big++;
+ return INVALID;
+ }
+
+ return classify_icmp(skb, l4_proto, &tuple, sizeof(tuple.ipv6),
+ metrics);
+}
+
+static verdict_t process_tcp(struct bpf_dynptr *dynptr, __u64 *offset, struct __sk_buff *skb,
+ struct iphdr_info *info, metrics_t *metrics)
+{
+ struct bpf_sock_tuple tuple;
+ struct tcphdr tcp;
+ uint64_t tuplen;
+
+ metrics->l4_protocol_packets_total_tcp++;
+
+ if (bpf_dynptr_read(&tcp, sizeof(tcp), dynptr, *offset, 0)) {
+ metrics->errors_total_malformed_tcp++;
+ return INVALID;
+ }
+
+ *offset += sizeof(tcp);
+
+ if (tcp.syn)
+ return SYN;
+
+ tuplen = fill_tuple(&tuple, info->hdr, info->len, tcp.source, tcp.dest);
+ return classify_tcp(skb, &tuple, tuplen, info->hdr, &tcp);
+}
+
+static verdict_t process_udp(struct bpf_dynptr *dynptr, __u64 *offset, struct __sk_buff *skb,
+ struct iphdr_info *info, metrics_t *metrics)
+{
+ struct bpf_sock_tuple tuple;
+ struct udphdr udph;
+ uint64_t tuplen;
+
+ metrics->l4_protocol_packets_total_udp++;
+
+ if (bpf_dynptr_read(&udph, sizeof(udph), dynptr, *offset, 0)) {
+ metrics->errors_total_malformed_udp++;
+ return INVALID;
+ }
+ *offset += sizeof(udph);
+
+ tuplen = fill_tuple(&tuple, info->hdr, info->len, udph.source, udph.dest);
+ return classify_udp(skb, &tuple, tuplen);
+}
+
+static verdict_t process_ipv4(struct __sk_buff *skb, struct bpf_dynptr *dynptr,
+ __u64 *offset, metrics_t *metrics)
+{
+ struct iphdr ipv4;
+ struct iphdr_info info = {
+ .hdr = &ipv4,
+ .len = sizeof(ipv4),
+ };
+
+ metrics->l3_protocol_packets_total_ipv4++;
+
+ if (pkt_parse_ipv4(dynptr, offset, &ipv4)) {
+ metrics->errors_total_malformed_ip++;
+ return INVALID;
+ }
+
+ if (ipv4.version != 4) {
+ metrics->errors_total_malformed_ip++;
+ return INVALID;
+ }
+
+ if (ipv4_is_fragment(&ipv4)) {
+ metrics->errors_total_fragmented_ip++;
+ return INVALID;
+ }
+
+ switch (ipv4.protocol) {
+ case IPPROTO_ICMP:
+ return process_icmpv4(skb, dynptr, offset, metrics);
+
+ case IPPROTO_TCP:
+ return process_tcp(dynptr, offset, skb, &info, metrics);
+
+ case IPPROTO_UDP:
+ return process_udp(dynptr, offset, skb, &info, metrics);
+
+ default:
+ metrics->errors_total_unknown_l4_proto++;
+ return INVALID;
+ }
+}
+
+static verdict_t process_ipv6(struct __sk_buff *skb, struct bpf_dynptr *dynptr,
+ __u64 *offset, metrics_t *metrics)
+{
+ struct ipv6hdr ipv6;
+ struct iphdr_info info = {
+ .hdr = &ipv6,
+ .len = sizeof(ipv6),
+ };
+ uint8_t l4_proto;
+ bool is_fragment;
+
+ metrics->l3_protocol_packets_total_ipv6++;
+
+ if (pkt_parse_ipv6(dynptr, offset, &ipv6, &l4_proto, &is_fragment)) {
+ metrics->errors_total_malformed_ip++;
+ return INVALID;
+ }
+
+ if (ipv6.version != 6) {
+ metrics->errors_total_malformed_ip++;
+ return INVALID;
+ }
+
+ if (is_fragment) {
+ metrics->errors_total_fragmented_ip++;
+ return INVALID;
+ }
+
+ switch (l4_proto) {
+ case IPPROTO_ICMPV6:
+ return process_icmpv6(dynptr, offset, skb, metrics);
+
+ case IPPROTO_TCP:
+ return process_tcp(dynptr, offset, skb, &info, metrics);
+
+ case IPPROTO_UDP:
+ return process_udp(dynptr, offset, skb, &info, metrics);
+
+ default:
+ metrics->errors_total_unknown_l4_proto++;
+ return INVALID;
+ }
+}
+
+SEC("tc")
+int cls_redirect(struct __sk_buff *skb)
+{
+ struct bpf_dynptr dynptr;
+ struct in_addr next_hop;
+ /* Tracks offset of the dynptr. This will be unnecessary once
+ * bpf_dynptr_advance() is available.
+ */
+ __u64 off = 0;
+
+ bpf_dynptr_from_skb(skb, 0, &dynptr);
+
+ metrics_t *metrics = get_global_metrics();
+ if (metrics == NULL)
+ return TC_ACT_SHOT;
+
+ metrics->processed_packets_total++;
+
+ /* Pass bogus packets as long as we're not sure they're
+ * destined for us.
+ */
+ if (skb->protocol != bpf_htons(ETH_P_IP))
+ return TC_ACT_OK;
+
+ encap_headers_t *encap;
+
+ /* Make sure that all encapsulation headers are available in
+ * the linear portion of the skb. This makes it easy to manipulate them.
+ */
+ if (bpf_skb_pull_data(skb, sizeof(*encap)))
+ return TC_ACT_OK;
+
+ encap = bpf_dynptr_data(&dynptr, 0, sizeof(*encap));
+ if (!encap)
+ return TC_ACT_OK;
+
+ off += sizeof(*encap);
+
+ if (encap->ip.ihl != 5)
+ /* We never have any options. */
+ return TC_ACT_OK;
+
+ if (encap->ip.daddr != ENCAPSULATION_IP ||
+ encap->ip.protocol != IPPROTO_UDP)
+ return TC_ACT_OK;
+
+ /* TODO Check UDP length? */
+ if (encap->udp.dest != ENCAPSULATION_PORT)
+ return TC_ACT_OK;
+
+ /* We now know that the packet is destined to us, we can
+ * drop bogus ones.
+ */
+ if (ipv4_is_fragment((void *)&encap->ip)) {
+ metrics->errors_total_fragmented_ip++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->gue.variant != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->gue.control != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->gue.flags != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->gue.hlen !=
+ sizeof(encap->unigue) / 4 + encap->unigue.hop_count) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->unigue.version != 0) {
+ metrics->errors_total_malformed_encapsulation++;
+ return TC_ACT_SHOT;
+ }
+
+ if (encap->unigue.reserved != 0)
+ return TC_ACT_SHOT;
+
+ MAYBE_RETURN(get_next_hop(&dynptr, &off, encap, &next_hop));
+
+ if (next_hop.s_addr == 0) {
+ metrics->accepted_packets_total_last_hop++;
+ return accept_locally(skb, encap);
+ }
+
+ verdict_t verdict;
+ switch (encap->gue.proto_ctype) {
+ case IPPROTO_IPIP:
+ verdict = process_ipv4(skb, &dynptr, &off, metrics);
+ break;
+
+ case IPPROTO_IPV6:
+ verdict = process_ipv6(skb, &dynptr, &off, metrics);
+ break;
+
+ default:
+ metrics->errors_total_unknown_l3_proto++;
+ return TC_ACT_SHOT;
+ }
+
+ switch (verdict) {
+ case INVALID:
+ /* metrics have already been bumped */
+ return TC_ACT_SHOT;
+
+ case UNKNOWN:
+ return forward_to_next_hop(skb, &dynptr, encap, &next_hop, metrics);
+
+ case ECHO_REQUEST:
+ metrics->accepted_packets_total_icmp_echo_request++;
+ break;
+
+ case SYN:
+ if (encap->unigue.forward_syn) {
+ return forward_to_next_hop(skb, &dynptr, encap, &next_hop,
+ metrics);
+ }
+
+ metrics->accepted_packets_total_syn++;
+ break;
+
+ case SYN_COOKIE:
+ metrics->accepted_packets_total_syn_cookies++;
+ break;
+
+ case ESTABLISHED:
+ metrics->accepted_packets_total_established++;
+ break;
+ }
+
+ return accept_locally(skb, encap);
+}
diff --git a/tools/testing/selftests/bpf/progs/test_l4lb_noinline_dynptr.c b/tools/testing/selftests/bpf/progs/test_l4lb_noinline_dynptr.c
new file mode 100644
index 000000000000..b364adb5434c
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_l4lb_noinline_dynptr.c
@@ -0,0 +1,474 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017 Facebook
+#include <stddef.h>
+#include <stdbool.h>
+#include <string.h>
+#include <linux/pkt_cls.h>
+#include <linux/bpf.h>
+#include <linux/in.h>
+#include <linux/if_ether.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/icmp.h>
+#include <linux/icmpv6.h>
+#include <linux/tcp.h>
+#include <linux/udp.h>
+#include <bpf/bpf_helpers.h>
+#include "test_iptunnel_common.h"
+#include <bpf/bpf_endian.h>
+
+extern int bpf_dynptr_from_skb(struct __sk_buff *skb, __u64 flags,
+ struct bpf_dynptr *ptr, int rd_only) __ksym;
+
+#define bpf_dynptr_from_skb(skb, flags, ptr) bpf_dynptr_from_skb(skb, flags, ptr, 0)
+
+static __always_inline __u32 rol32(__u32 word, unsigned int shift)
+{
+ return (word << shift) | (word >> ((-shift) & 31));
+}
+
+/* copy paste of jhash from kernel sources to make sure llvm
+ * can compile it into valid sequence of bpf instructions
+ */
+#define __jhash_mix(a, b, c) \
+{ \
+ a -= c; a ^= rol32(c, 4); c += b; \
+ b -= a; b ^= rol32(a, 6); a += c; \
+ c -= b; c ^= rol32(b, 8); b += a; \
+ a -= c; a ^= rol32(c, 16); c += b; \
+ b -= a; b ^= rol32(a, 19); a += c; \
+ c -= b; c ^= rol32(b, 4); b += a; \
+}
+
+#define __jhash_final(a, b, c) \
+{ \
+ c ^= b; c -= rol32(b, 14); \
+ a ^= c; a -= rol32(c, 11); \
+ b ^= a; b -= rol32(a, 25); \
+ c ^= b; c -= rol32(b, 16); \
+ a ^= c; a -= rol32(c, 4); \
+ b ^= a; b -= rol32(a, 14); \
+ c ^= b; c -= rol32(b, 24); \
+}
+
+#define JHASH_INITVAL 0xdeadbeef
+
+typedef unsigned int u32;
+
+static __noinline u32 jhash(const void *key, u32 length, u32 initval)
+{
+ u32 a, b, c;
+ const unsigned char *k = key;
+
+ a = b = c = JHASH_INITVAL + length + initval;
+
+ while (length > 12) {
+ a += *(u32 *)(k);
+ b += *(u32 *)(k + 4);
+ c += *(u32 *)(k + 8);
+ __jhash_mix(a, b, c);
+ length -= 12;
+ k += 12;
+ }
+ switch (length) {
+ case 12: c += (u32)k[11]<<24;
+ case 11: c += (u32)k[10]<<16;
+ case 10: c += (u32)k[9]<<8;
+ case 9: c += k[8];
+ case 8: b += (u32)k[7]<<24;
+ case 7: b += (u32)k[6]<<16;
+ case 6: b += (u32)k[5]<<8;
+ case 5: b += k[4];
+ case 4: a += (u32)k[3]<<24;
+ case 3: a += (u32)k[2]<<16;
+ case 2: a += (u32)k[1]<<8;
+ case 1: a += k[0];
+ __jhash_final(a, b, c);
+ case 0: /* Nothing left to add */
+ break;
+ }
+
+ return c;
+}
+
+static __noinline u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
+{
+ a += initval;
+ b += initval;
+ c += initval;
+ __jhash_final(a, b, c);
+ return c;
+}
+
+static __noinline u32 jhash_2words(u32 a, u32 b, u32 initval)
+{
+ return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
+}
+
+#define PCKT_FRAGMENTED 65343
+#define IPV4_HDR_LEN_NO_OPT 20
+#define IPV4_PLUS_ICMP_HDR 28
+#define IPV6_PLUS_ICMP_HDR 48
+#define RING_SIZE 2
+#define MAX_VIPS 12
+#define MAX_REALS 5
+#define CTL_MAP_SIZE 16
+#define CH_RINGS_SIZE (MAX_VIPS * RING_SIZE)
+#define F_IPV6 (1 << 0)
+#define F_HASH_NO_SRC_PORT (1 << 0)
+#define F_ICMP (1 << 0)
+#define F_SYN_SET (1 << 1)
+
+struct packet_description {
+ union {
+ __be32 src;
+ __be32 srcv6[4];
+ };
+ union {
+ __be32 dst;
+ __be32 dstv6[4];
+ };
+ union {
+ __u32 ports;
+ __u16 port16[2];
+ };
+ __u8 proto;
+ __u8 flags;
+};
+
+struct ctl_value {
+ union {
+ __u64 value;
+ __u32 ifindex;
+ __u8 mac[6];
+ };
+};
+
+struct vip_meta {
+ __u32 flags;
+ __u32 vip_num;
+};
+
+struct real_definition {
+ union {
+ __be32 dst;
+ __be32 dstv6[4];
+ };
+ __u8 flags;
+};
+
+struct vip_stats {
+ __u64 bytes;
+ __u64 pkts;
+};
+
+struct eth_hdr {
+ unsigned char eth_dest[ETH_ALEN];
+ unsigned char eth_source[ETH_ALEN];
+ unsigned short eth_proto;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, MAX_VIPS);
+ __type(key, struct vip);
+ __type(value, struct vip_meta);
+} vip_map SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, CH_RINGS_SIZE);
+ __type(key, __u32);
+ __type(value, __u32);
+} ch_rings SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, MAX_REALS);
+ __type(key, __u32);
+ __type(value, struct real_definition);
+} reals SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(max_entries, MAX_VIPS);
+ __type(key, __u32);
+ __type(value, struct vip_stats);
+} stats SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, CTL_MAP_SIZE);
+ __type(key, __u32);
+ __type(value, struct ctl_value);
+} ctl_array SEC(".maps");
+
+static __noinline __u32 get_packet_hash(struct packet_description *pckt, bool ipv6)
+{
+ if (ipv6)
+ return jhash_2words(jhash(pckt->srcv6, 16, MAX_VIPS),
+ pckt->ports, CH_RINGS_SIZE);
+ else
+ return jhash_2words(pckt->src, pckt->ports, CH_RINGS_SIZE);
+}
+
+static __noinline bool get_packet_dst(struct real_definition **real,
+ struct packet_description *pckt,
+ struct vip_meta *vip_info,
+ bool is_ipv6)
+{
+ __u32 hash = get_packet_hash(pckt, is_ipv6);
+ __u32 key = RING_SIZE * vip_info->vip_num + hash % RING_SIZE;
+ __u32 *real_pos;
+
+ if (hash != 0x358459b7 /* jhash of ipv4 packet */ &&
+ hash != 0x2f4bc6bb /* jhash of ipv6 packet */)
+ return false;
+
+ real_pos = bpf_map_lookup_elem(&ch_rings, &key);
+ if (!real_pos)
+ return false;
+ key = *real_pos;
+ *real = bpf_map_lookup_elem(&reals, &key);
+ if (!(*real))
+ return false;
+ return true;
+}
+
+static __noinline int parse_icmpv6(struct bpf_dynptr *skb_ptr, __u64 off,
+ struct packet_description *pckt)
+{
+ struct icmp6hdr *icmp_hdr;
+ struct ipv6hdr *ip6h;
+
+ icmp_hdr = bpf_dynptr_data(skb_ptr, off, sizeof(*icmp_hdr));
+ if (!icmp_hdr)
+ return TC_ACT_SHOT;
+
+ if (icmp_hdr->icmp6_type != ICMPV6_PKT_TOOBIG)
+ return TC_ACT_OK;
+ off += sizeof(struct icmp6hdr);
+ ip6h = (struct ipv6hdr *)bpf_dynptr_data(skb_ptr, off, sizeof(*ip6h));
+ if (!ip6h)
+ return TC_ACT_SHOT;
+ pckt->proto = ip6h->nexthdr;
+ pckt->flags |= F_ICMP;
+ memcpy(pckt->srcv6, ip6h->daddr.s6_addr32, 16);
+ memcpy(pckt->dstv6, ip6h->saddr.s6_addr32, 16);
+ return TC_ACT_UNSPEC;
+}
+
+static __noinline int parse_icmp(struct bpf_dynptr *skb_ptr, __u64 off,
+ struct packet_description *pckt)
+{
+ struct icmphdr *icmp_hdr;
+ struct iphdr *iph;
+
+ icmp_hdr = bpf_dynptr_data(skb_ptr, off, sizeof(*icmp_hdr));
+ if (!icmp_hdr)
+ return TC_ACT_SHOT;
+ if (icmp_hdr->type != ICMP_DEST_UNREACH ||
+ icmp_hdr->code != ICMP_FRAG_NEEDED)
+ return TC_ACT_OK;
+ off += sizeof(struct icmphdr);
+ iph = bpf_dynptr_data(skb_ptr, off, sizeof(*iph));
+ if (!iph || iph->ihl != 5)
+ return TC_ACT_SHOT;
+ pckt->proto = iph->protocol;
+ pckt->flags |= F_ICMP;
+ pckt->src = iph->daddr;
+ pckt->dst = iph->saddr;
+ return TC_ACT_UNSPEC;
+}
+
+static __noinline bool parse_udp(struct bpf_dynptr *skb_ptr, __u64 off,
+ struct packet_description *pckt)
+{
+ struct udphdr *udp;
+
+ udp = bpf_dynptr_data(skb_ptr, off, sizeof(*udp));
+ if (!udp)
+ return false;
+
+ if (!(pckt->flags & F_ICMP)) {
+ pckt->port16[0] = udp->source;
+ pckt->port16[1] = udp->dest;
+ } else {
+ pckt->port16[0] = udp->dest;
+ pckt->port16[1] = udp->source;
+ }
+ return true;
+}
+
+static __noinline bool parse_tcp(struct bpf_dynptr *skb_ptr, __u64 off,
+ struct packet_description *pckt)
+{
+ struct tcphdr *tcp;
+
+ tcp = bpf_dynptr_data(skb_ptr, off, sizeof(*tcp));
+ if (!tcp)
+ return false;
+
+ if (tcp->syn)
+ pckt->flags |= F_SYN_SET;
+
+ if (!(pckt->flags & F_ICMP)) {
+ pckt->port16[0] = tcp->source;
+ pckt->port16[1] = tcp->dest;
+ } else {
+ pckt->port16[0] = tcp->dest;
+ pckt->port16[1] = tcp->source;
+ }
+ return true;
+}
+
+static __noinline int process_packet(struct bpf_dynptr *skb_ptr,
+ struct eth_hdr *eth, __u64 off,
+ bool is_ipv6, struct __sk_buff *skb)
+{
+ struct packet_description pckt = {};
+ struct bpf_tunnel_key tkey = {};
+ struct vip_stats *data_stats;
+ struct real_definition *dst;
+ struct vip_meta *vip_info;
+ struct ctl_value *cval;
+ __u32 v4_intf_pos = 1;
+ __u32 v6_intf_pos = 2;
+ struct ipv6hdr *ip6h;
+ struct vip vip = {};
+ struct iphdr *iph;
+ int tun_flag = 0;
+ __u16 pkt_bytes;
+ __u64 iph_len;
+ __u32 ifindex;
+ __u8 protocol;
+ __u32 vip_num;
+ int action;
+
+ tkey.tunnel_ttl = 64;
+ if (is_ipv6) {
+ ip6h = bpf_dynptr_data(skb_ptr, off, sizeof(*ip6h));
+ if (!ip6h)
+ return TC_ACT_SHOT;
+
+ iph_len = sizeof(struct ipv6hdr);
+ protocol = ip6h->nexthdr;
+ pckt.proto = protocol;
+ pkt_bytes = bpf_ntohs(ip6h->payload_len);
+ off += iph_len;
+ if (protocol == IPPROTO_FRAGMENT) {
+ return TC_ACT_SHOT;
+ } else if (protocol == IPPROTO_ICMPV6) {
+ action = parse_icmpv6(skb_ptr, off, &pckt);
+ if (action >= 0)
+ return action;
+ off += IPV6_PLUS_ICMP_HDR;
+ } else {
+ memcpy(pckt.srcv6, ip6h->saddr.s6_addr32, 16);
+ memcpy(pckt.dstv6, ip6h->daddr.s6_addr32, 16);
+ }
+ } else {
+ iph = bpf_dynptr_data(skb_ptr, off, sizeof(*iph));
+ if (!iph || iph->ihl != 5)
+ return TC_ACT_SHOT;
+
+ protocol = iph->protocol;
+ pckt.proto = protocol;
+ pkt_bytes = bpf_ntohs(iph->tot_len);
+ off += IPV4_HDR_LEN_NO_OPT;
+
+ if (iph->frag_off & PCKT_FRAGMENTED)
+ return TC_ACT_SHOT;
+ if (protocol == IPPROTO_ICMP) {
+ action = parse_icmp(skb_ptr, off, &pckt);
+ if (action >= 0)
+ return action;
+ off += IPV4_PLUS_ICMP_HDR;
+ } else {
+ pckt.src = iph->saddr;
+ pckt.dst = iph->daddr;
+ }
+ }
+ protocol = pckt.proto;
+
+ if (protocol == IPPROTO_TCP) {
+ if (!parse_tcp(skb_ptr, off, &pckt))
+ return TC_ACT_SHOT;
+ } else if (protocol == IPPROTO_UDP) {
+ if (!parse_udp(skb_ptr, off, &pckt))
+ return TC_ACT_SHOT;
+ } else {
+ return TC_ACT_SHOT;
+ }
+
+ if (is_ipv6)
+ memcpy(vip.daddr.v6, pckt.dstv6, 16);
+ else
+ vip.daddr.v4 = pckt.dst;
+
+ vip.dport = pckt.port16[1];
+ vip.protocol = pckt.proto;
+ vip_info = bpf_map_lookup_elem(&vip_map, &vip);
+ if (!vip_info) {
+ vip.dport = 0;
+ vip_info = bpf_map_lookup_elem(&vip_map, &vip);
+ if (!vip_info)
+ return TC_ACT_SHOT;
+ pckt.port16[1] = 0;
+ }
+
+ if (vip_info->flags & F_HASH_NO_SRC_PORT)
+ pckt.port16[0] = 0;
+
+ if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6))
+ return TC_ACT_SHOT;
+
+ if (dst->flags & F_IPV6) {
+ cval = bpf_map_lookup_elem(&ctl_array, &v6_intf_pos);
+ if (!cval)
+ return TC_ACT_SHOT;
+ ifindex = cval->ifindex;
+ memcpy(tkey.remote_ipv6, dst->dstv6, 16);
+ tun_flag = BPF_F_TUNINFO_IPV6;
+ } else {
+ cval = bpf_map_lookup_elem(&ctl_array, &v4_intf_pos);
+ if (!cval)
+ return TC_ACT_SHOT;
+ ifindex = cval->ifindex;
+ tkey.remote_ipv4 = dst->dst;
+ }
+ vip_num = vip_info->vip_num;
+ data_stats = bpf_map_lookup_elem(&stats, &vip_num);
+ if (!data_stats)
+ return TC_ACT_SHOT;
+ data_stats->pkts++;
+ data_stats->bytes += pkt_bytes;
+ bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), tun_flag);
+ *(u32 *)eth->eth_dest = tkey.remote_ipv4;
+ return bpf_redirect(ifindex, 0);
+}
+
+SEC("tc")
+int balancer_ingress(struct __sk_buff *ctx)
+{
+ struct bpf_dynptr ptr;
+ struct eth_hdr *eth;
+ __u32 eth_proto;
+ __u32 nh_off;
+
+ nh_off = sizeof(struct eth_hdr);
+
+ bpf_dynptr_from_skb(ctx, 0, &ptr);
+ eth = bpf_dynptr_data(&ptr, 0, sizeof(*eth));
+ if (!eth)
+ return TC_ACT_SHOT;
+ eth_proto = eth->eth_proto;
+ if (eth_proto == bpf_htons(ETH_P_IP))
+ return process_packet(&ptr, eth, nh_off, false, ctx);
+ else if (eth_proto == bpf_htons(ETH_P_IPV6))
+ return process_packet(&ptr, eth, nh_off, true, ctx);
+ else
+ return TC_ACT_SHOT;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt.c b/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt.c
new file mode 100644
index 000000000000..79bab9b50e9e
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* This parsing logic is taken from the open source library katran, a layer 4
+ * load balancer.
+ *
+ * This code logic using dynptrs can be found in test_parse_tcp_hdr_opt_dynptr.c
+ *
+ * https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/pckt_parsing.h
+ */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <linux/tcp.h>
+#include <stdbool.h>
+#include <linux/ipv6.h>
+#include <linux/if_ether.h>
+#include "test_tcp_hdr_options.h"
+
+char _license[] SEC("license") = "GPL";
+
+/* Kind number used for experiments */
+const __u32 tcp_hdr_opt_kind_tpr = 0xFD;
+/* Length of the tcp header option */
+const __u32 tcp_hdr_opt_len_tpr = 6;
+/* maximum number of header options to check to lookup server_id */
+const __u32 tcp_hdr_opt_max_opt_checks = 15;
+
+__u32 server_id;
+
+struct hdr_opt_state {
+ __u32 server_id;
+ __u8 byte_offset;
+ __u8 hdr_bytes_remaining;
+};
+
+static int parse_hdr_opt(const struct xdp_md *xdp, struct hdr_opt_state *state)
+{
+ const void *data = (void *)(long)xdp->data;
+ const void *data_end = (void *)(long)xdp->data_end;
+ __u8 *tcp_opt, kind, hdr_len;
+
+ tcp_opt = (__u8 *)(data + state->byte_offset);
+ if (tcp_opt + 1 > data_end)
+ return -1;
+
+ kind = tcp_opt[0];
+
+ if (kind == TCPOPT_EOL)
+ return -1;
+
+ if (kind == TCPOPT_NOP) {
+ state->hdr_bytes_remaining--;
+ state->byte_offset++;
+ return 0;
+ }
+
+ if (state->hdr_bytes_remaining < 2 ||
+ tcp_opt + sizeof(__u8) + sizeof(__u8) > data_end)
+ return -1;
+
+ hdr_len = tcp_opt[1];
+ if (hdr_len > state->hdr_bytes_remaining)
+ return -1;
+
+ if (kind == tcp_hdr_opt_kind_tpr) {
+ if (hdr_len != tcp_hdr_opt_len_tpr)
+ return -1;
+
+ if (tcp_opt + tcp_hdr_opt_len_tpr > data_end)
+ return -1;
+
+ state->server_id = *(__u32 *)&tcp_opt[2];
+ return 1;
+ }
+
+ state->hdr_bytes_remaining -= hdr_len;
+ state->byte_offset += hdr_len;
+ return 0;
+}
+
+SEC("xdp")
+int xdp_ingress_v6(struct xdp_md *xdp)
+{
+ const void *data = (void *)(long)xdp->data;
+ const void *data_end = (void *)(long)xdp->data_end;
+ struct hdr_opt_state opt_state = {};
+ __u8 tcp_hdr_opt_len = 0;
+ struct tcphdr *tcp_hdr;
+ __u64 tcp_offset = 0;
+ __u32 off;
+ int err;
+
+ tcp_offset = sizeof(struct ethhdr) + sizeof(struct ipv6hdr);
+ tcp_hdr = (struct tcphdr *)(data + tcp_offset);
+ if (tcp_hdr + 1 > data_end)
+ return XDP_DROP;
+
+ tcp_hdr_opt_len = (tcp_hdr->doff * 4) - sizeof(struct tcphdr);
+ if (tcp_hdr_opt_len < tcp_hdr_opt_len_tpr)
+ return XDP_DROP;
+
+ opt_state.hdr_bytes_remaining = tcp_hdr_opt_len;
+ opt_state.byte_offset = sizeof(struct tcphdr) + tcp_offset;
+
+ /* max number of bytes of options in tcp header is 40 bytes */
+ for (int i = 0; i < tcp_hdr_opt_max_opt_checks; i++) {
+ err = parse_hdr_opt(xdp, &opt_state);
+
+ if (err || !opt_state.hdr_bytes_remaining)
+ break;
+ }
+
+ if (!opt_state.server_id)
+ return XDP_DROP;
+
+ server_id = opt_state.server_id;
+
+ return XDP_PASS;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt_dynptr.c b/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt_dynptr.c
new file mode 100644
index 000000000000..2053b933da6a
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt_dynptr.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* This logic is lifted from a real-world use case of packet parsing, used in
+ * the open source library katran, a layer 4 load balancer.
+ *
+ * This test demonstrates how to parse packet contents using dynptrs. The
+ * original code (parsing without dynptrs) can be found in test_parse_tcp_hdr_opt.c
+ */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <linux/tcp.h>
+#include <stdbool.h>
+#include <linux/ipv6.h>
+#include <linux/if_ether.h>
+#include "test_tcp_hdr_options.h"
+
+char _license[] SEC("license") = "GPL";
+
+extern int bpf_dynptr_from_xdp(struct xdp_md *xdp, __u64 flags, struct bpf_dynptr *ptr) __ksym;
+
+/* Kind number used for experiments */
+const __u32 tcp_hdr_opt_kind_tpr = 0xFD;
+/* Length of the tcp header option */
+const __u32 tcp_hdr_opt_len_tpr = 6;
+/* maximum number of header options to check to lookup server_id */
+const __u32 tcp_hdr_opt_max_opt_checks = 15;
+
+__u32 server_id;
+
+static int parse_hdr_opt(struct bpf_dynptr *ptr, __u32 *off, __u8 *hdr_bytes_remaining,
+ __u32 *server_id)
+{
+ __u8 *tcp_opt, kind, hdr_len;
+ __u8 *data;
+
+ data = bpf_dynptr_data(ptr, *off, sizeof(kind) + sizeof(hdr_len) +
+ sizeof(*server_id));
+ if (!data)
+ return -1;
+
+ kind = data[0];
+
+ if (kind == TCPOPT_EOL)
+ return -1;
+
+ if (kind == TCPOPT_NOP) {
+ *off += 1;
+ *hdr_bytes_remaining -= 1;
+ return 0;
+ }
+
+ if (*hdr_bytes_remaining < 2)
+ return -1;
+
+ hdr_len = data[1];
+ if (hdr_len > *hdr_bytes_remaining)
+ return -1;
+
+ if (kind == tcp_hdr_opt_kind_tpr) {
+ if (hdr_len != tcp_hdr_opt_len_tpr)
+ return -1;
+
+ __builtin_memcpy(server_id, (__u32 *)(data + 2), sizeof(*server_id));
+ return 1;
+ }
+
+ *off += hdr_len;
+ *hdr_bytes_remaining -= hdr_len;
+ return 0;
+}
+
+SEC("xdp")
+int xdp_ingress_v6(struct xdp_md *xdp)
+{
+ __u8 hdr_bytes_remaining;
+ struct tcphdr *tcp_hdr;
+ __u8 tcp_hdr_opt_len;
+ int err = 0;
+ __u32 off;
+
+ struct bpf_dynptr ptr;
+
+ bpf_dynptr_from_xdp(xdp, 0, &ptr);
+
+ off = sizeof(struct ethhdr) + sizeof(struct ipv6hdr);
+
+ tcp_hdr = bpf_dynptr_data(&ptr, off, sizeof(*tcp_hdr));
+ if (!tcp_hdr)
+ return XDP_DROP;
+
+ tcp_hdr_opt_len = (tcp_hdr->doff * 4) - sizeof(struct tcphdr);
+ if (tcp_hdr_opt_len < tcp_hdr_opt_len_tpr)
+ return XDP_DROP;
+
+ hdr_bytes_remaining = tcp_hdr_opt_len;
+
+ off += sizeof(struct tcphdr);
+
+ /* max number of bytes of options in tcp header is 40 bytes */
+ for (int i = 0; i < tcp_hdr_opt_max_opt_checks; i++) {
+ err = parse_hdr_opt(&ptr, &off, &hdr_bytes_remaining, &server_id);
+
+ if (err || !hdr_bytes_remaining)
+ break;
+ }
+
+ if (!server_id)
+ return XDP_DROP;
+
+ return XDP_PASS;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_xdp_dynptr.c b/tools/testing/selftests/bpf/progs/test_xdp_dynptr.c
new file mode 100644
index 000000000000..e635aa2ac43f
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_xdp_dynptr.c
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta */
+#include <stddef.h>
+#include <string.h>
+#include <linux/bpf.h>
+#include <linux/if_ether.h>
+#include <linux/if_packet.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/in.h>
+#include <linux/udp.h>
+#include <linux/tcp.h>
+#include <linux/pkt_cls.h>
+#include <sys/socket.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include "test_iptunnel_common.h"
+
+extern int bpf_dynptr_from_xdp(struct xdp_md *xdp, __u64 flags, struct bpf_dynptr *ptr) __ksym;
+
+const size_t tcphdr_sz = sizeof(struct tcphdr);
+const size_t udphdr_sz = sizeof(struct udphdr);
+const size_t ethhdr_sz = sizeof(struct ethhdr);
+const size_t iphdr_sz = sizeof(struct iphdr);
+const size_t ipv6hdr_sz = sizeof(struct ipv6hdr);
+
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(max_entries, 256);
+ __type(key, __u32);
+ __type(value, __u64);
+} rxcnt SEC(".maps");
+
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, MAX_IPTNL_ENTRIES);
+ __type(key, struct vip);
+ __type(value, struct iptnl_info);
+} vip2tnl SEC(".maps");
+
+static __always_inline void count_tx(__u32 protocol)
+{
+ __u64 *rxcnt_count;
+
+ rxcnt_count = bpf_map_lookup_elem(&rxcnt, &protocol);
+ if (rxcnt_count)
+ *rxcnt_count += 1;
+}
+
+static __always_inline int get_dport(void *trans_data, __u8 protocol)
+{
+ struct tcphdr *th;
+ struct udphdr *uh;
+
+ switch (protocol) {
+ case IPPROTO_TCP:
+ th = (struct tcphdr *)trans_data;
+ return th->dest;
+ case IPPROTO_UDP:
+ uh = (struct udphdr *)trans_data;
+ return uh->dest;
+ default:
+ return 0;
+ }
+}
+
+static __always_inline void set_ethhdr(struct ethhdr *new_eth,
+ const struct ethhdr *old_eth,
+ const struct iptnl_info *tnl,
+ __be16 h_proto)
+{
+ memcpy(new_eth->h_source, old_eth->h_dest, sizeof(new_eth->h_source));
+ memcpy(new_eth->h_dest, tnl->dmac, sizeof(new_eth->h_dest));
+ new_eth->h_proto = h_proto;
+}
+
+static __always_inline int handle_ipv4(struct xdp_md *xdp, struct bpf_dynptr *xdp_ptr)
+{
+ struct bpf_dynptr new_xdp_ptr;
+ struct iptnl_info *tnl;
+ struct ethhdr *new_eth;
+ struct ethhdr *old_eth;
+ __u32 transport_hdr_sz;
+ struct iphdr *iph;
+ __u16 *next_iph;
+ __u16 payload_len;
+ struct vip vip = {};
+ int dport;
+ __u32 csum = 0;
+ int i;
+
+ if (ethhdr_sz + iphdr_sz + tcphdr_sz > xdp->data_end - xdp->data)
+ transport_hdr_sz = udphdr_sz;
+ else
+ transport_hdr_sz = tcphdr_sz;
+
+ iph = bpf_dynptr_data(xdp_ptr, ethhdr_sz, iphdr_sz + transport_hdr_sz);
+ if (!iph)
+ return XDP_DROP;
+
+ dport = get_dport(iph + 1, iph->protocol);
+ if (dport == -1)
+ return XDP_DROP;
+
+ vip.protocol = iph->protocol;
+ vip.family = AF_INET;
+ vip.daddr.v4 = iph->daddr;
+ vip.dport = dport;
+ payload_len = bpf_ntohs(iph->tot_len);
+
+ tnl = bpf_map_lookup_elem(&vip2tnl, &vip);
+ /* It only does v4-in-v4 */
+ if (!tnl || tnl->family != AF_INET)
+ return XDP_PASS;
+
+ if (bpf_xdp_adjust_head(xdp, 0 - (int)iphdr_sz))
+ return XDP_DROP;
+
+ bpf_dynptr_from_xdp(xdp, 0, &new_xdp_ptr);
+ new_eth = bpf_dynptr_data(&new_xdp_ptr, 0, ethhdr_sz + iphdr_sz + ethhdr_sz);
+ if (!new_eth)
+ return XDP_DROP;
+
+ iph = (struct iphdr *)(new_eth + 1);
+ old_eth = (struct ethhdr *)(iph + 1);
+
+ set_ethhdr(new_eth, old_eth, tnl, bpf_htons(ETH_P_IP));
+
+ iph->version = 4;
+ iph->ihl = iphdr_sz >> 2;
+ iph->frag_off = 0;
+ iph->protocol = IPPROTO_IPIP;
+ iph->check = 0;
+ iph->tos = 0;
+ iph->tot_len = bpf_htons(payload_len + iphdr_sz);
+ iph->daddr = tnl->daddr.v4;
+ iph->saddr = tnl->saddr.v4;
+ iph->ttl = 8;
+
+ next_iph = (__u16 *)iph;
+ for (i = 0; i < iphdr_sz >> 1; i++)
+ csum += *next_iph++;
+
+ iph->check = ~((csum & 0xffff) + (csum >> 16));
+
+ count_tx(vip.protocol);
+
+ return XDP_TX;
+}
+
+static __always_inline int handle_ipv6(struct xdp_md *xdp, struct bpf_dynptr *xdp_ptr)
+{
+ struct bpf_dynptr new_xdp_ptr;
+ struct iptnl_info *tnl;
+ struct ethhdr *new_eth;
+ struct ethhdr *old_eth;
+ __u32 transport_hdr_sz;
+ struct ipv6hdr *ip6h;
+ __u16 payload_len;
+ struct vip vip = {};
+ int dport;
+
+ if (ethhdr_sz + iphdr_sz + tcphdr_sz > xdp->data_end - xdp->data)
+ transport_hdr_sz = udphdr_sz;
+ else
+ transport_hdr_sz = tcphdr_sz;
+
+ ip6h = bpf_dynptr_data(xdp_ptr, ethhdr_sz, ipv6hdr_sz + transport_hdr_sz);
+ if (!ip6h)
+ return XDP_DROP;
+
+ dport = get_dport(ip6h + 1, ip6h->nexthdr);
+ if (dport == -1)
+ return XDP_DROP;
+
+ vip.protocol = ip6h->nexthdr;
+ vip.family = AF_INET6;
+ memcpy(vip.daddr.v6, ip6h->daddr.s6_addr32, sizeof(vip.daddr));
+ vip.dport = dport;
+ payload_len = ip6h->payload_len;
+
+ tnl = bpf_map_lookup_elem(&vip2tnl, &vip);
+ /* It only does v6-in-v6 */
+ if (!tnl || tnl->family != AF_INET6)
+ return XDP_PASS;
+
+ if (bpf_xdp_adjust_head(xdp, 0 - (int)ipv6hdr_sz))
+ return XDP_DROP;
+
+ bpf_dynptr_from_xdp(xdp, 0, &new_xdp_ptr);
+ new_eth = bpf_dynptr_data(&new_xdp_ptr, 0, ethhdr_sz + ipv6hdr_sz + ethhdr_sz);
+ if (!new_eth)
+ return XDP_DROP;
+
+ ip6h = (struct ipv6hdr *)(new_eth + 1);
+ old_eth = (struct ethhdr *)(ip6h + 1);
+
+ set_ethhdr(new_eth, old_eth, tnl, bpf_htons(ETH_P_IPV6));
+
+ ip6h->version = 6;
+ ip6h->priority = 0;
+ memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl));
+ ip6h->payload_len = bpf_htons(bpf_ntohs(payload_len) + ipv6hdr_sz);
+ ip6h->nexthdr = IPPROTO_IPV6;
+ ip6h->hop_limit = 8;
+ memcpy(ip6h->saddr.s6_addr32, tnl->saddr.v6, sizeof(tnl->saddr.v6));
+ memcpy(ip6h->daddr.s6_addr32, tnl->daddr.v6, sizeof(tnl->daddr.v6));
+
+ count_tx(vip.protocol);
+
+ return XDP_TX;
+}
+
+SEC("xdp")
+int _xdp_tx_iptunnel(struct xdp_md *xdp)
+{
+ struct bpf_dynptr ptr;
+ struct ethhdr *eth;
+ __u16 h_proto;
+
+ bpf_dynptr_from_xdp(xdp, 0, &ptr);
+ eth = bpf_dynptr_data(&ptr, 0, ethhdr_sz);
+ if (!eth)
+ return XDP_DROP;
+
+ h_proto = eth->h_proto;
+
+ if (h_proto == bpf_htons(ETH_P_IP))
+ return handle_ipv4(xdp, &ptr);
+ else if (h_proto == bpf_htons(ETH_P_IPV6))
+
+ return handle_ipv6(xdp, &ptr);
+ else
+ return XDP_DROP;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/test_tcp_hdr_options.h b/tools/testing/selftests/bpf/test_tcp_hdr_options.h
index 6118e3ab61fc..56c9f8a3ad3d 100644
--- a/tools/testing/selftests/bpf/test_tcp_hdr_options.h
+++ b/tools/testing/selftests/bpf/test_tcp_hdr_options.h
@@ -50,6 +50,7 @@ struct linum_err {
#define TCPOPT_EOL 0
#define TCPOPT_NOP 1
+#define TCPOPT_MSS 2
#define TCPOPT_WINDOW 3
#define TCPOPT_EXP 254
--
2.30.2
^ permalink raw reply related [flat|nested] 44+ messages in thread* Re: [PATCH v9 bpf-next 5/5] selftests/bpf: tests for using dynptrs to parse skb and xdp buffers
2023-01-27 19:17 ` [PATCH v9 bpf-next 5/5] selftests/bpf: tests for using dynptrs to parse skb and xdp buffers Joanne Koong
@ 2023-01-31 0:49 ` Andrii Nakryiko
0 siblings, 0 replies; 44+ messages in thread
From: Andrii Nakryiko @ 2023-01-31 0:49 UTC (permalink / raw)
To: Joanne Koong
Cc: bpf, daniel, andrii, martin.lau, ast, netdev, memxor, kernel-team
On Fri, Jan 27, 2023 at 11:18 AM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> Test skb and xdp dynptr functionality in the following ways:
>
> 1) progs/test_cls_redirect_dynptr.c
> * Rewrite "progs/test_cls_redirect.c" test to use dynptrs to parse
> skb data
>
> * This is a great example of how dynptrs can be used to simplify a
> lot of the parsing logic for non-statically known values.
>
> When measuring the user + system time between the original version
> vs. using dynptrs, and averaging the time for 10 runs (using
> "time ./test_progs -t cls_redirect"):
> original version: 0.092 sec
> with dynptrs: 0.078 sec
>
> 2) progs/test_xdp_dynptr.c
> * Rewrite "progs/test_xdp.c" test to use dynptrs to parse xdp data
>
> When measuring the user + system time between the original version
> vs. using dynptrs, and averaging the time for 10 runs (using
> "time ./test_progs -t xdp_attach"):
> original version: 0.118 sec
> with dynptrs: 0.094 sec
>
> 3) progs/test_l4lb_noinline_dynptr.c
> * Rewrite "progs/test_l4lb_noinline.c" test to use dynptrs to parse
> skb data
>
> When measuring the user + system time between the original version
> vs. using dynptrs, and averaging the time for 10 runs (using
> "time ./test_progs -t l4lb_all"):
> original version: 0.062 sec
> with dynptrs: 0.081 sec
>
> For number of processed verifier instructions:
> original version: 6268 insns
> with dynptrs: 2588 insns
>
> 4) progs/test_parse_tcp_hdr_opt_dynptr.c
> * Add sample code for parsing tcp hdr opt lookup using dynptrs.
> This logic is lifted from a real-world use case of packet parsing
> in katran [0], a layer 4 load balancer. The original version
> "progs/test_parse_tcp_hdr_opt.c" (not using dynptrs) is included
> here as well, for comparison.
>
> When measuring the user + system time between the original version
> vs. using dynptrs, and averaging the time for 10 runs (using
> "time ./test_progs -t parse_tcp_hdr_opt"):
> original version: 0.031 sec
> with dynptrs: 0.045 sec
>
> 5) progs/dynptr_success.c
> * Add test case "test_skb_readonly" for testing attempts at writes /
> data slices on a prog type with read-only skb ctx.
>
> 6) progs/dynptr_fail.c
> * Add test cases "skb_invalid_data_slice{1,2}" and
> "xdp_invalid_data_slice" for testing that helpers that modify the
> underlying packet buffer automatically invalidate the associated
> data slice.
> * Add test cases "skb_invalid_ctx" and "xdp_invalid_ctx" for testing
> that prog types that do not support bpf_dynptr_from_skb/xdp don't
> have access to the API.
> * Add test case "skb_invalid_write" for testing that writes to a
> read-only data slice are rejected by the verifier.
>
> [0]
> https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/pckt_parsing.h
>
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> Acked-by: Andrii Nakryiko <andrii@kernel.org>
> ---
> .../selftests/bpf/prog_tests/cls_redirect.c | 25 +
> .../testing/selftests/bpf/prog_tests/dynptr.c | 63 +-
> .../selftests/bpf/prog_tests/l4lb_all.c | 2 +
> .../bpf/prog_tests/parse_tcp_hdr_opt.c | 93 ++
> .../selftests/bpf/prog_tests/xdp_attach.c | 11 +-
> .../testing/selftests/bpf/progs/dynptr_fail.c | 124 +++
> .../selftests/bpf/progs/dynptr_success.c | 28 +
> .../bpf/progs/test_cls_redirect_dynptr.c | 973 ++++++++++++++++++
> .../bpf/progs/test_l4lb_noinline_dynptr.c | 474 +++++++++
> .../bpf/progs/test_parse_tcp_hdr_opt.c | 119 +++
> .../bpf/progs/test_parse_tcp_hdr_opt_dynptr.c | 112 ++
> .../selftests/bpf/progs/test_xdp_dynptr.c | 237 +++++
> .../selftests/bpf/test_tcp_hdr_options.h | 1 +
> 13 files changed, 2248 insertions(+), 14 deletions(-)
> create mode 100644 tools/testing/selftests/bpf/prog_tests/parse_tcp_hdr_opt.c
> create mode 100644 tools/testing/selftests/bpf/progs/test_cls_redirect_dynptr.c
> create mode 100644 tools/testing/selftests/bpf/progs/test_l4lb_noinline_dynptr.c
> create mode 100644 tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt.c
> create mode 100644 tools/testing/selftests/bpf/progs/test_parse_tcp_hdr_opt_dynptr.c
> create mode 100644 tools/testing/selftests/bpf/progs/test_xdp_dynptr.c
>
[...]
> diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
> index 062fbc8c8e5e..28c453bbb84a 100644
> --- a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
> +++ b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
> @@ -4,11 +4,10 @@
> #define IFINDEX_LO 1
> #define XDP_FLAGS_REPLACE (1U << 4)
>
> -void serial_test_xdp_attach(void)
> +static void serial_test_xdp_attach(const char *file)
> {
> __u32 duration = 0, id1, id2, id0 = 0, len;
> struct bpf_object *obj1, *obj2, *obj3;
> - const char *file = "./test_xdp.bpf.o";
> struct bpf_prog_info info = {};
> int err, fd1, fd2, fd3;
> LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
> @@ -85,3 +84,11 @@ void serial_test_xdp_attach(void)
> out_1:
> bpf_object__close(obj1);
> }
> +
> +void test_xdp_attach(void)
this test was marked as serial (it starts with "serial_test_"), so we
probably want to preserve that? Just keep this as
"serial_test_xdp_attach" and rename above serial_test_xdp_attach to
something like "subtest_xdp_attach" (this name doesn't matter to
test_progs runner).
> +{
> + if (test__start_subtest("xdp_attach"))
> + serial_test_xdp_attach("./test_xdp.bpf.o");
> + if (test__start_subtest("xdp_attach_dynptr"))
> + serial_test_xdp_attach("./test_xdp_dynptr.bpf.o");
> +}
[...]
^ permalink raw reply [flat|nested] 44+ messages in thread