* [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h.
2024-02-21 1:23 [PATCH bpf-next v3 0/5] Create shadow types for struct_ops maps in skeletons thinker.li
@ 2024-02-21 1:23 ` thinker.li
2024-02-21 11:49 ` Quentin Monnet
2024-02-21 1:23 ` [PATCH bpf-next v3 2/5] libbpf: set btf_value_type_id of struct bpf_map for struct_ops thinker.li
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: thinker.li @ 2024-02-21 1:23 UTC (permalink / raw)
To: bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng, Kui-Feng Lee
From: Kui-Feng Lee <thinker.li@gmail.com>
bpftool is going to reuse this helper function to support shadow types of
struct_ops maps.
Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
---
tools/lib/bpf/libbpf.c | 2 +-
tools/lib/bpf/libbpf_internal.h | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 01f407591a92..ef8fd20f33ca 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -2145,7 +2145,7 @@ skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
return t;
}
-static const struct btf_type *
+const struct btf_type *
resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
{
const struct btf_type *t;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index ad936ac5e639..aec6d57fe5d1 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -234,6 +234,7 @@ struct btf_type;
struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
const char *btf_kind_str(const struct btf_type *t);
const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
+const struct btf_type *resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id);
static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
{
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h.
2024-02-21 1:23 ` [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h thinker.li
@ 2024-02-21 11:49 ` Quentin Monnet
2024-02-21 16:32 ` Kui-Feng Lee
0 siblings, 1 reply; 10+ messages in thread
From: Quentin Monnet @ 2024-02-21 11:49 UTC (permalink / raw)
To: thinker.li, bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng
2024-02-21 01:23 UTC+0000 ~ thinker.li@gmail.com
> From: Kui-Feng Lee <thinker.li@gmail.com>
>
> bpftool is going to reuse this helper function to support shadow types of
> struct_ops maps.
>
> Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
> ---
> tools/lib/bpf/libbpf.c | 2 +-
> tools/lib/bpf/libbpf_internal.h | 1 +
> 2 files changed, 2 insertions(+), 1 deletion(-)
>
> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
> index 01f407591a92..ef8fd20f33ca 100644
> --- a/tools/lib/bpf/libbpf.c
> +++ b/tools/lib/bpf/libbpf.c
> @@ -2145,7 +2145,7 @@ skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
> return t;
> }
>
> -static const struct btf_type *
> +const struct btf_type *
> resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
> {
> const struct btf_type *t;
> diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
> index ad936ac5e639..aec6d57fe5d1 100644
> --- a/tools/lib/bpf/libbpf_internal.h
> +++ b/tools/lib/bpf/libbpf_internal.h
> @@ -234,6 +234,7 @@ struct btf_type;
> struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
> const char *btf_kind_str(const struct btf_type *t);
> const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
> +const struct btf_type *resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id);
If you respin, please add a comment to mention we expose it to bpftool
(see bpf_core_add_cands() in the same file), to avoid people trying to
remove it from the header file in a clean-up attempt.
Quentin
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h.
2024-02-21 11:49 ` Quentin Monnet
@ 2024-02-21 16:32 ` Kui-Feng Lee
0 siblings, 0 replies; 10+ messages in thread
From: Kui-Feng Lee @ 2024-02-21 16:32 UTC (permalink / raw)
To: Quentin Monnet, thinker.li, bpf, ast, martin.lau, song,
kernel-team, andrii
Cc: kuifeng
On 2/21/24 03:49, Quentin Monnet wrote:
> 2024-02-21 01:23 UTC+0000 ~ thinker.li@gmail.com
>> From: Kui-Feng Lee <thinker.li@gmail.com>
>>
>> bpftool is going to reuse this helper function to support shadow types of
>> struct_ops maps.
>>
>> Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
>> ---
>> tools/lib/bpf/libbpf.c | 2 +-
>> tools/lib/bpf/libbpf_internal.h | 1 +
>> 2 files changed, 2 insertions(+), 1 deletion(-)
>>
>> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
>> index 01f407591a92..ef8fd20f33ca 100644
>> --- a/tools/lib/bpf/libbpf.c
>> +++ b/tools/lib/bpf/libbpf.c
>> @@ -2145,7 +2145,7 @@ skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
>> return t;
>> }
>>
>> -static const struct btf_type *
>> +const struct btf_type *
>> resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
>> {
>> const struct btf_type *t;
>> diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
>> index ad936ac5e639..aec6d57fe5d1 100644
>> --- a/tools/lib/bpf/libbpf_internal.h
>> +++ b/tools/lib/bpf/libbpf_internal.h
>> @@ -234,6 +234,7 @@ struct btf_type;
>> struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
>> const char *btf_kind_str(const struct btf_type *t);
>> const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
>> +const struct btf_type *resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id);
>
> If you respin, please add a comment to mention we expose it to bpftool
> (see bpf_core_add_cands() in the same file), to avoid people trying to
> remove it from the header file in a clean-up attempt.
No problem
>
> Quentin
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH bpf-next v3 2/5] libbpf: set btf_value_type_id of struct bpf_map for struct_ops.
2024-02-21 1:23 [PATCH bpf-next v3 0/5] Create shadow types for struct_ops maps in skeletons thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h thinker.li
@ 2024-02-21 1:23 ` thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 3/5] libbpf: Convert st_ops->data to shadow type thinker.li
` (2 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: thinker.li @ 2024-02-21 1:23 UTC (permalink / raw)
To: bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng, Kui-Feng Lee
From: Kui-Feng Lee <thinker.li@gmail.com>
For a struct_ops map, btf_value_type_id is the type ID of it's struct
type. This value is required by bpftool to generate skeleton including
pointers of shadow types. The code generator gets the type ID from
bpf_map__btf_vaule_type_id() in order to get the type information of the
struct type of a map.
Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
---
tools/lib/bpf/libbpf.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index ef8fd20f33ca..465b50235a01 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -1229,6 +1229,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
map->name = strdup(var_name);
if (!map->name)
return -ENOMEM;
+ map->btf_value_type_id = type_id;
map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
map->def.key_size = sizeof(int);
@@ -4818,7 +4819,9 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
if (obj->btf && btf__fd(obj->btf) >= 0) {
create_attr.btf_fd = btf__fd(obj->btf);
create_attr.btf_key_type_id = map->btf_key_type_id;
- create_attr.btf_value_type_id = map->btf_value_type_id;
+ create_attr.btf_value_type_id =
+ def->type != BPF_MAP_TYPE_STRUCT_OPS ?
+ map->btf_value_type_id : 0;
}
if (bpf_map_type__is_map_in_map(def->type)) {
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH bpf-next v3 3/5] libbpf: Convert st_ops->data to shadow type.
2024-02-21 1:23 [PATCH bpf-next v3 0/5] Create shadow types for struct_ops maps in skeletons thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 1/5] libbpf: expose resolve_func_ptr() through libbpf_internal.h thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 2/5] libbpf: set btf_value_type_id of struct bpf_map for struct_ops thinker.li
@ 2024-02-21 1:23 ` thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps thinker.li
2024-02-21 1:23 ` [PATCH bpf-next v3 5/5] selftests/bpf: Test if shadow types work correctly thinker.li
4 siblings, 0 replies; 10+ messages in thread
From: thinker.li @ 2024-02-21 1:23 UTC (permalink / raw)
To: bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng, Kui-Feng Lee
From: Kui-Feng Lee <thinker.li@gmail.com>
Convert st_ops->data to the shadow type of the struct_ops map. The shadow
type of a struct_ops type is a variant of the original struct type
providing a way to access/change the values in the maps of the struct_ops
type.
bpf_map__initial_value() will return st_ops->data for struct_ops types. The
skeleton is going to use it as the pointer to the shadow type of the
original struct type.
One of the main differences between the original struct type and the shadow
type is that all function pointers of the shadow type are converted to
pointers of struct bpf_program. Users can replace these bpf_program
pointers with other BPF programs. The st_ops->progs[] will be updated
before updating the value of a map to reflect the changes made by users.
Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
---
tools/lib/bpf/libbpf.c | 53 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 53 insertions(+)
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 465b50235a01..becbb4d81012 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -1102,6 +1102,9 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
if (btf_is_ptr(mtype)) {
struct bpf_program *prog;
+ /* Update the value from the shadow type */
+ st_ops->progs[i] = *(struct bpf_program **)mdata;
+
prog = st_ops->progs[i];
if (!prog)
continue;
@@ -1172,6 +1175,36 @@ static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
return 0;
}
+/* Convert the data of a struct_ops map to shadow type.
+ *
+ * The function pointers are replaced with the pointers of bpf_program in
+ * st_ops->progs[].
+ */
+static void struct_ops_convert_shadow(struct bpf_map *map,
+ const struct btf_type *t)
+{
+ struct btf *btf = map->obj->btf;
+ struct bpf_struct_ops *st_ops = map->st_ops;
+ const struct btf_member *m;
+ const struct btf_type *mtype;
+ char *data;
+ int i;
+
+ data = st_ops->data;
+
+ for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
+ mtype = skip_mods_and_typedefs(btf, m->type, NULL);
+
+ if (btf_kind(mtype) != BTF_KIND_PTR)
+ continue;
+ if (!resolve_func_ptr(btf, m->type, NULL))
+ continue;
+
+ *((struct bpf_program **)(data + m->offset / 8)) =
+ st_ops->progs[i];
+ }
+}
+
static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
int shndx, Elf_Data *data, __u32 map_flags)
{
@@ -7531,6 +7564,19 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object
return 0;
}
+/* Convert the data to the shadow type for each struct_ops map. */
+static void bpf_object__init_shadow(struct bpf_object *obj)
+{
+ struct bpf_map *map;
+
+ bpf_object__for_each_map(map, obj) {
+ if (!bpf_map__is_struct_ops(map))
+ continue;
+
+ struct_ops_convert_shadow(map, map->st_ops->type);
+ }
+}
+
static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz,
const struct bpf_object_open_opts *opts)
{
@@ -7631,6 +7677,7 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf,
goto out;
bpf_object__elf_finish(obj);
+ bpf_object__init_shadow(obj);
return obj;
out:
@@ -9880,6 +9927,12 @@ int bpf_map__set_initial_value(struct bpf_map *map,
void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
{
+ if (bpf_map__is_struct_ops(map)) {
+ if (psize)
+ *psize = map->def.value_size;
+ return map->st_ops->data;
+ }
+
if (!map->mmaped)
return NULL;
*psize = map->def.value_size;
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps.
2024-02-21 1:23 [PATCH bpf-next v3 0/5] Create shadow types for struct_ops maps in skeletons thinker.li
` (2 preceding siblings ...)
2024-02-21 1:23 ` [PATCH bpf-next v3 3/5] libbpf: Convert st_ops->data to shadow type thinker.li
@ 2024-02-21 1:23 ` thinker.li
2024-02-21 11:49 ` Quentin Monnet
2024-02-21 1:23 ` [PATCH bpf-next v3 5/5] selftests/bpf: Test if shadow types work correctly thinker.li
4 siblings, 1 reply; 10+ messages in thread
From: thinker.li @ 2024-02-21 1:23 UTC (permalink / raw)
To: bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng, Kui-Feng Lee
From: Kui-Feng Lee <thinker.li@gmail.com>
Declares and defines a pointer of the shadow type for each struct_ops map.
The code generator will create an anonymous struct type as the shadow type
for each struct_ops map. The shadow type is translated from the original
struct type of the map. The user of the skeleton use pointers of them to
access the values of struct_ops maps.
However, shadow types only supports certain types of fields, such as scalar
types and function pointers. Any fields of unsupported types are translated
into an array of characters to occupy the space of the original
field. Function pointers are translated into pointers of the struct
bpf_program. Additionally, padding fields are generated to occupy the space
between two consecutive fields.
The pointers of shadow types of struct_osp maps are initialized when
*__open_opts() in skeletons are called. For a map called FOO, the user can
access it through the pointer at skel->struct_ops.FOO.
Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
---
tools/bpf/bpftool/gen.c | 229 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 228 insertions(+), 1 deletion(-)
diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
index a9334c57e859..20c5d5912df7 100644
--- a/tools/bpf/bpftool/gen.c
+++ b/tools/bpf/bpftool/gen.c
@@ -909,6 +909,201 @@ codegen_progs_skeleton(struct bpf_object *obj, size_t prog_cnt, bool populate_li
}
}
+static int walk_st_ops_shadow_vars(struct btf *btf,
+ const char *ident,
+ const struct bpf_map *map)
+{
+ DECLARE_LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts,
+ .indent_level = 3,
+ );
+ const struct btf_type *map_type, *member_type;
+ __u32 map_type_id, member_type_id;
+ __u32 offset, next_offset = 0;
+ const struct btf_member *m;
+ const char *member_name;
+ struct btf_dump *d = NULL;
+ int i, err = 0;
+ int size, map_size;
+
+ map_type_id = bpf_map__btf_value_type_id(map);
+ if (map_type_id == 0)
+ return -EINVAL;
+ map_type = btf__type_by_id(btf, map_type_id);
+ if (!map_type)
+ return -EINVAL;
+
+ d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL);
+ if (!d)
+ return -errno;
+
+ for (i = 0, m = btf_members(map_type);
+ i < btf_vlen(map_type);
+ i++, m++) {
+ member_type = skip_mods_and_typedefs(btf, m->type,
+ &member_type_id);
+ if (!member_type) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ member_name = btf__name_by_offset(btf, m->name_off);
+ if (!member_name) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ offset = m->offset / 8;
+ if (next_offset != offset) {
+ printf("\t\t\tchar __padding_%d[%d];\n",
+ i - 1, offset - next_offset);
+ }
+
+ switch (btf_kind(member_type)) {
+ case BTF_KIND_INT:
+ case BTF_KIND_FLOAT:
+ case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
+ /* scalar type */
+ printf("\t\t\t");
+ opts.field_name = member_name;
+ err = btf_dump__emit_type_decl(d, member_type_id,
+ &opts);
+ if (err)
+ goto out;
+ printf(";\n");
+
+ size = btf__resolve_size(btf, member_type_id);
+ if (size < 0) {
+ err = size;
+ goto out;
+ }
+
+ next_offset = offset + size;
+ break;
+
+ case BTF_KIND_PTR:
+ if (resolve_func_ptr(btf, m->type, NULL)) {
+ /* Function pointer */
+ printf("\t\t\tconst struct bpf_program *%s;\n",
+ member_name);
+
+ next_offset = offset + sizeof(void *);
+ break;
+ }
+ fallthrough;
+
+ default:
+ /* Unsupported types
+ *
+ * For unsupported types, we have to generate
+ * definitions for them in order to support
+ * them. For example, we need to generate a
+ * definition for a struct type or a union type. It
+ * may cause type conflicts without renaming since
+ * the same type may be defined for several
+ * skeletons, and the user may include these
+ * skeletons in the same compile unit.
+ */
+ if (i == btf_vlen(map_type) - 1) {
+ map_size = btf__resolve_size(btf, map_type_id);
+ if (map_size < 0)
+ return -EINVAL;
+ size = map_size - offset;
+ } else {
+ size = (m[1].offset - m->offset) / 8;
+ }
+
+ printf("\t\t\tchar __padding_%d[%d];\n", i, size);
+
+ next_offset = offset + size;
+ break;
+ }
+ }
+
+out:
+ btf_dump__free(d);
+
+ return err;
+}
+
+/* Generate the pointer of the shadow type for a struct_ops map.
+ *
+ * This function adds a pointer of the shadow type for a struct_ops map.
+ * The members of a struct_ops map can be exported through a pointer to a
+ * shadow type. The user can access these members through the pointer.
+ *
+ * A shadow type includes not all members, only members of some types.
+ * They are scalar types and function pointers. The function pointers are
+ * translated to the pointer of the struct bpf_program. The scalar types
+ * are translated to the original type without any modifiers.
+ *
+ * Unsupported types will be translated to a char array to take the same
+ * space of the original field. However, due to handling padding and
+ * alignments, the user should not access them directly.
+ */
+static int gen_st_ops_shadow_type(struct btf *btf, const char *ident,
+ const struct bpf_map *map)
+{
+ int err;
+
+ printf("\t\tstruct {\n");
+
+ err = walk_st_ops_shadow_vars(btf, ident, map);
+ if (err)
+ return err;
+
+ printf("\t\t} *%s;\n", ident);
+
+ return 0;
+}
+
+static int gen_st_ops_shadow(struct btf *btf, struct bpf_object *obj)
+{
+ struct bpf_map *map;
+ char ident[256];
+ int err;
+
+ /* Generate the pointers to shadow types of
+ * struct_ops maps.
+ */
+ printf("\tstruct {\n");
+ bpf_object__for_each_map(map, obj) {
+ if (bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
+ continue;
+ if (!get_map_ident(map, ident, sizeof(ident)))
+ continue;
+ err = gen_st_ops_shadow_type(btf, ident, map);
+ if (err)
+ return err;
+ }
+ printf("\t} struct_ops;\n");
+
+ return 0;
+}
+
+/* Generate the code to initialize the pointers of shadow types. */
+static void gen_st_ops_shadow_init(struct btf *btf, struct bpf_object *obj)
+{
+ struct bpf_map *map;
+ char ident[256];
+
+ /* Initialize the pointers to_ops shadow types of
+ * struct_ops maps.
+ */
+ bpf_object__for_each_map(map, obj) {
+ if (bpf_map__type(map) != BPF_MAP_TYPE_STRUCT_OPS)
+ continue;
+ if (!get_map_ident(map, ident, sizeof(ident)))
+ continue;
+ codegen("\
+ \n\
+ obj->struct_ops.%1$s = \n\
+ bpf_map__initial_value(obj->maps.%1$s, NULL);\n\
+ \n\
+ ", ident);
+ }
+}
+
static int do_skeleton(int argc, char **argv)
{
char header_guard[MAX_OBJ_NAME_LEN + sizeof("__SKEL_H__")];
@@ -923,6 +1118,7 @@ static int do_skeleton(int argc, char **argv)
struct bpf_map *map;
struct btf *btf;
struct stat st;
+ int st_ops_cnt = 0;
if (!REQ_ARGS(1)) {
usage();
@@ -1039,6 +1235,8 @@ static int do_skeleton(int argc, char **argv)
);
}
+ btf = bpf_object__btf(obj);
+
if (map_cnt) {
printf("\tstruct {\n");
bpf_object__for_each_map(map, obj) {
@@ -1048,8 +1246,15 @@ static int do_skeleton(int argc, char **argv)
printf("\t\tstruct bpf_map_desc %s;\n", ident);
else
printf("\t\tstruct bpf_map *%s;\n", ident);
+ if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS)
+ st_ops_cnt++;
}
printf("\t} maps;\n");
+ if (st_ops_cnt && btf) {
+ err = gen_st_ops_shadow(btf, obj);
+ if (err)
+ goto out;
+ }
}
if (prog_cnt) {
@@ -1075,7 +1280,6 @@ static int do_skeleton(int argc, char **argv)
printf("\t} links;\n");
}
- btf = bpf_object__btf(obj);
if (btf) {
err = codegen_datasecs(obj, obj_name);
if (err)
@@ -1133,6 +1337,13 @@ static int do_skeleton(int argc, char **argv)
if (err) \n\
goto err_out; \n\
\n\
+ ", obj_name);
+
+ if (st_ops_cnt && btf)
+ gen_st_ops_shadow_init(btf, obj);
+
+ codegen("\
+ \n\
return obj; \n\
err_out: \n\
%1$s__destroy(obj); \n\
@@ -1296,6 +1507,7 @@ static int do_subskeleton(int argc, char **argv)
struct btf *btf;
const struct btf_type *map_type, *var_type;
const struct btf_var_secinfo *var;
+ int st_ops_cnt = 0;
struct stat st;
if (!REQ_ARGS(1)) {
@@ -1438,10 +1650,18 @@ static int do_subskeleton(int argc, char **argv)
if (!get_map_ident(map, ident, sizeof(ident)))
continue;
printf("\t\tstruct bpf_map *%s;\n", ident);
+ if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS)
+ st_ops_cnt++;
}
printf("\t} maps;\n");
}
+ if (st_ops_cnt && btf) {
+ err = gen_st_ops_shadow(btf, obj);
+ if (err)
+ goto out;
+ }
+
if (prog_cnt) {
printf("\tstruct {\n");
bpf_object__for_each_program(prog, obj) {
@@ -1553,6 +1773,13 @@ static int do_subskeleton(int argc, char **argv)
if (err) \n\
goto err; \n\
\n\
+ ");
+
+ if (st_ops_cnt && btf)
+ gen_st_ops_shadow_init(btf, obj);
+
+ codegen("\
+ \n\
return obj; \n\
err: \n\
%1$s__destroy(obj); \n\
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread
* Re: [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps.
2024-02-21 1:23 ` [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps thinker.li
@ 2024-02-21 11:49 ` Quentin Monnet
2024-02-21 17:47 ` Kui-Feng Lee
0 siblings, 1 reply; 10+ messages in thread
From: Quentin Monnet @ 2024-02-21 11:49 UTC (permalink / raw)
To: thinker.li, bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng
2024-02-21 01:23 UTC+0000 ~ thinker.li@gmail.com
> From: Kui-Feng Lee <thinker.li@gmail.com>
>
> Declares and defines a pointer of the shadow type for each struct_ops map.
>
> The code generator will create an anonymous struct type as the shadow type
> for each struct_ops map. The shadow type is translated from the original
> struct type of the map. The user of the skeleton use pointers of them to
> access the values of struct_ops maps.
>
> However, shadow types only supports certain types of fields, such as scalar
Nit: "such as" implies the list may not be exhaustive.
> types and function pointers. Any fields of unsupported types are translated
> into an array of characters to occupy the space of the original
> field. Function pointers are translated into pointers of the struct
> bpf_program. Additionally, padding fields are generated to occupy the space
> between two consecutive fields.
>
> The pointers of shadow types of struct_osp maps are initialized when
> *__open_opts() in skeletons are called. For a map called FOO, the user can
> access it through the pointer at skel->struct_ops.FOO.
>
> Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
> ---
> tools/bpf/bpftool/gen.c | 229 +++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 228 insertions(+), 1 deletion(-)
>
> diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
> index a9334c57e859..20c5d5912df7 100644
> --- a/tools/bpf/bpftool/gen.c
> +++ b/tools/bpf/bpftool/gen.c
> @@ -909,6 +909,201 @@ codegen_progs_skeleton(struct bpf_object *obj, size_t prog_cnt, bool populate_li
> }
> }
>
> +static int walk_st_ops_shadow_vars(struct btf *btf,
> + const char *ident,
> + const struct bpf_map *map)
> +{
> + DECLARE_LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts,
> + .indent_level = 3,
> + );
> + const struct btf_type *map_type, *member_type;
> + __u32 map_type_id, member_type_id;
> + __u32 offset, next_offset = 0;
> + const struct btf_member *m;
> + const char *member_name;
> + struct btf_dump *d = NULL;
> + int i, err = 0;
> + int size, map_size;
> +
> + map_type_id = bpf_map__btf_value_type_id(map);
> + if (map_type_id == 0)
> + return -EINVAL;
> + map_type = btf__type_by_id(btf, map_type_id);
> + if (!map_type)
> + return -EINVAL;
> +
> + d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL);
> + if (!d)
> + return -errno;
> +
> + for (i = 0, m = btf_members(map_type);
> + i < btf_vlen(map_type);
> + i++, m++) {
> + member_type = skip_mods_and_typedefs(btf, m->type,
> + &member_type_id);
> + if (!member_type) {
> + err = -EINVAL;
> + goto out;
> + }
> +
> + member_name = btf__name_by_offset(btf, m->name_off);
> + if (!member_name) {
> + err = -EINVAL;
> + goto out;
> + }
> +
> + offset = m->offset / 8;
> + if (next_offset != offset) {
> + printf("\t\t\tchar __padding_%d[%d];\n",
> + i - 1, offset - next_offset);
> + }
> +
> + switch (btf_kind(member_type)) {
> + case BTF_KIND_INT:
> + case BTF_KIND_FLOAT:
> + case BTF_KIND_ENUM:
> + case BTF_KIND_ENUM64:
> + /* scalar type */
> + printf("\t\t\t");
> + opts.field_name = member_name;
> + err = btf_dump__emit_type_decl(d, member_type_id,
> + &opts);
> + if (err)
> + goto out;
> + printf(";\n");
> +
> + size = btf__resolve_size(btf, member_type_id);
> + if (size < 0) {
> + err = size;
> + goto out;
> + }
> +
> + next_offset = offset + size;
> + break;
> +
> + case BTF_KIND_PTR:
> + if (resolve_func_ptr(btf, m->type, NULL)) {
> + /* Function pointer */
> + printf("\t\t\tconst struct bpf_program *%s;\n",
> + member_name);
> +
> + next_offset = offset + sizeof(void *);
> + break;
> + }
> + fallthrough;
I wouldn't mind a comment about the "fallthrough;" to state explicitly
that only function pointers are supported for now.
> +
> + default:
> + /* Unsupported types
> + *
> + * For unsupported types, we have to generate
> + * definitions for them in order to support
> + * them. For example, we need to generate a
> + * definition for a struct type or a union type. It
> + * may cause type conflicts without renaming since
> + * the same type may be defined for several
> + * skeletons, and the user may include these
> + * skeletons in the same compile unit.
> + */
This comment could be clearer. "For unsupported types, we have to
generate definitions for them in order to support them". So do we, or do
we not support them? "It may cause type conflicts [...]" -> do we
address these?
My understanding is that this note describes the work to do if we want
to add support in the future, and this could perhaps be more explicit:
"We do not support other types yet. The reason is that ... But when we
generate definitions, we will have to take care of type conflicts
because ...". What do you think?
> + if (i == btf_vlen(map_type) - 1) {
> + map_size = btf__resolve_size(btf, map_type_id);
> + if (map_size < 0)
> + return -EINVAL;
> + size = map_size - offset;
> + } else {
> + size = (m[1].offset - m->offset) / 8;
> + }
> +
> + printf("\t\t\tchar __padding_%d[%d];\n", i, size);
> +
> + next_offset = offset + size;
> + break;
> + }
> + }
> +
> +out:
> + btf_dump__free(d);
> +
> + return err;
> +}
> +
> +/* Generate the pointer of the shadow type for a struct_ops map.
> + *
> + * This function adds a pointer of the shadow type for a struct_ops map.
> + * The members of a struct_ops map can be exported through a pointer to a
> + * shadow type. The user can access these members through the pointer.
> + *
> + * A shadow type includes not all members, only members of some types.
> + * They are scalar types and function pointers. The function pointers are
> + * translated to the pointer of the struct bpf_program. The scalar types
> + * are translated to the original type without any modifiers.
> + *
> + * Unsupported types will be translated to a char array to take the same
> + * space of the original field. However, due to handling padding and
> + * alignments, the user should not access them directly.
What's the risk, and how should users know?
> + */
[...]
Thanks for this work! The bpftool changes look good. I've got these few
observations above, but notwithstanding:
Reviewed-by: Quentin Monnet <quentin@isovalent.com>
I wonder, did you think of adding a paragraph or an example to the man
page for "bpftool gen"? Your change is for a specific use case, but
otherwise I'm not sure how users will ever know that these shadow types
are available (other than discovering them by luck in a skeleton) or how
to use them if they need to.
Thanks,
Quentin
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps.
2024-02-21 11:49 ` Quentin Monnet
@ 2024-02-21 17:47 ` Kui-Feng Lee
0 siblings, 0 replies; 10+ messages in thread
From: Kui-Feng Lee @ 2024-02-21 17:47 UTC (permalink / raw)
To: Quentin Monnet, thinker.li, bpf, ast, martin.lau, song,
kernel-team, andrii
Cc: kuifeng
On 2/21/24 03:49, Quentin Monnet wrote:
> 2024-02-21 01:23 UTC+0000 ~ thinker.li@gmail.com
>> From: Kui-Feng Lee <thinker.li@gmail.com>
>>
>> Declares and defines a pointer of the shadow type for each struct_ops map.
>>
>> The code generator will create an anonymous struct type as the shadow type
>> for each struct_ops map. The shadow type is translated from the original
>> struct type of the map. The user of the skeleton use pointers of them to
>> access the values of struct_ops maps.
>>
>> However, shadow types only supports certain types of fields, such as scalar
>
> Nit: "such as" implies the list may not be exhaustive.
>
>> types and function pointers. Any fields of unsupported types are translated
>> into an array of characters to occupy the space of the original
>> field. Function pointers are translated into pointers of the struct
>> bpf_program. Additionally, padding fields are generated to occupy the space
>> between two consecutive fields.
>>
>> The pointers of shadow types of struct_osp maps are initialized when
>> *__open_opts() in skeletons are called. For a map called FOO, the user can
>> access it through the pointer at skel->struct_ops.FOO.
>>
>> Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
>> ---
>> tools/bpf/bpftool/gen.c | 229 +++++++++++++++++++++++++++++++++++++++-
>> 1 file changed, 228 insertions(+), 1 deletion(-)
>>
>> diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
>> index a9334c57e859..20c5d5912df7 100644
>> --- a/tools/bpf/bpftool/gen.c
>> +++ b/tools/bpf/bpftool/gen.c
>> @@ -909,6 +909,201 @@ codegen_progs_skeleton(struct bpf_object *obj, size_t prog_cnt, bool populate_li
>> }
>> }
>>
>> +static int walk_st_ops_shadow_vars(struct btf *btf,
>> + const char *ident,
>> + const struct bpf_map *map)
>> +{
>> + DECLARE_LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts,
>> + .indent_level = 3,
>> + );
>> + const struct btf_type *map_type, *member_type;
>> + __u32 map_type_id, member_type_id;
>> + __u32 offset, next_offset = 0;
>> + const struct btf_member *m;
>> + const char *member_name;
>> + struct btf_dump *d = NULL;
>> + int i, err = 0;
>> + int size, map_size;
>> +
>> + map_type_id = bpf_map__btf_value_type_id(map);
>> + if (map_type_id == 0)
>> + return -EINVAL;
>> + map_type = btf__type_by_id(btf, map_type_id);
>> + if (!map_type)
>> + return -EINVAL;
>> +
>> + d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL);
>> + if (!d)
>> + return -errno;
>> +
>> + for (i = 0, m = btf_members(map_type);
>> + i < btf_vlen(map_type);
>> + i++, m++) {
>> + member_type = skip_mods_and_typedefs(btf, m->type,
>> + &member_type_id);
>> + if (!member_type) {
>> + err = -EINVAL;
>> + goto out;
>> + }
>> +
>> + member_name = btf__name_by_offset(btf, m->name_off);
>> + if (!member_name) {
>> + err = -EINVAL;
>> + goto out;
>> + }
>> +
>> + offset = m->offset / 8;
>> + if (next_offset != offset) {
>> + printf("\t\t\tchar __padding_%d[%d];\n",
>> + i - 1, offset - next_offset);
>> + }
>> +
>> + switch (btf_kind(member_type)) {
>> + case BTF_KIND_INT:
>> + case BTF_KIND_FLOAT:
>> + case BTF_KIND_ENUM:
>> + case BTF_KIND_ENUM64:
>> + /* scalar type */
>> + printf("\t\t\t");
>> + opts.field_name = member_name;
>> + err = btf_dump__emit_type_decl(d, member_type_id,
>> + &opts);
>> + if (err)
>> + goto out;
>> + printf(";\n");
>> +
>> + size = btf__resolve_size(btf, member_type_id);
>> + if (size < 0) {
>> + err = size;
>> + goto out;
>> + }
>> +
>> + next_offset = offset + size;
>> + break;
>> +
>> + case BTF_KIND_PTR:
>> + if (resolve_func_ptr(btf, m->type, NULL)) {
>> + /* Function pointer */
>> + printf("\t\t\tconst struct bpf_program *%s;\n",
>> + member_name);
>> +
>> + next_offset = offset + sizeof(void *);
>> + break;
>> + }
>> + fallthrough;
>
> I wouldn't mind a comment about the "fallthrough;" to state explicitly
> that only function pointers are supported for now.
Sure
>
>> +
>> + default:
>> + /* Unsupported types
>> + *
>> + * For unsupported types, we have to generate
>> + * definitions for them in order to support
>> + * them. For example, we need to generate a
>> + * definition for a struct type or a union type. It
>> + * may cause type conflicts without renaming since
>> + * the same type may be defined for several
>> + * skeletons, and the user may include these
>> + * skeletons in the same compile unit.
>> + */
>
> This comment could be clearer. "For unsupported types, we have to
> generate definitions for them in order to support them". So do we, or do
> we not support them? "It may cause type conflicts [...]" -> do we
> address these?
>
> My understanding is that this note describes the work to do if we want
> to add support in the future, and this could perhaps be more explicit:
> "We do not support other types yet. The reason is that ... But when we
> generate definitions, we will have to take care of type conflicts
> because ...". What do you think?
Agree! I will rephrase this comment.
>
>> + if (i == btf_vlen(map_type) - 1) {
>> + map_size = btf__resolve_size(btf, map_type_id);
>> + if (map_size < 0)
>> + return -EINVAL;
>> + size = map_size - offset;
>> + } else {
>> + size = (m[1].offset - m->offset) / 8;
>> + }
>> +
>> + printf("\t\t\tchar __padding_%d[%d];\n", i, size);
>> +
>> + next_offset = offset + size;
>> + break;
>> + }
>> + }
>> +
>> +out:
>> + btf_dump__free(d);
>> +
>> + return err;
>> +}
>> +
>> +/* Generate the pointer of the shadow type for a struct_ops map.
>> + *
>> + * This function adds a pointer of the shadow type for a struct_ops map.
>> + * The members of a struct_ops map can be exported through a pointer to a
>> + * shadow type. The user can access these members through the pointer.
>> + *
>> + * A shadow type includes not all members, only members of some types.
>> + * They are scalar types and function pointers. The function pointers are
>> + * translated to the pointer of the struct bpf_program. The scalar types
>> + * are translated to the original type without any modifiers.
>> + *
>> + * Unsupported types will be translated to a char array to take the same
>> + * space of the original field. However, due to handling padding and
>> + * alignments, the user should not access them directly.
>
> What's the risk, and how should users know?
The names of unsupported fields are replaced by "__padding_*", and their
types are "char []".
Changing names and types of fields in a struct can lead to accessing
issues, where users may inadvertently corrupt data due to padding and
field reordering in different versions.
I will include the above explanation in the next version.
>
>> + */
>
> [...]
>
> Thanks for this work! The bpftool changes look good. I've got these few
> observations above, but notwithstanding:
>
> Reviewed-by: Quentin Monnet <quentin@isovalent.com>
>
> I wonder, did you think of adding a paragraph or an example to the man
> page for "bpftool gen"? Your change is for a specific use case, but
> otherwise I'm not sure how users will ever know that these shadow types
> are available (other than discovering them by luck in a skeleton) or how
> to use them if they need to.
Sure! I will add it in the following version.
>
> Thanks,
> Quentin
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH bpf-next v3 5/5] selftests/bpf: Test if shadow types work correctly.
2024-02-21 1:23 [PATCH bpf-next v3 0/5] Create shadow types for struct_ops maps in skeletons thinker.li
` (3 preceding siblings ...)
2024-02-21 1:23 ` [PATCH bpf-next v3 4/5] bpftool: generated shadow variables for struct_ops maps thinker.li
@ 2024-02-21 1:23 ` thinker.li
4 siblings, 0 replies; 10+ messages in thread
From: thinker.li @ 2024-02-21 1:23 UTC (permalink / raw)
To: bpf, ast, martin.lau, song, kernel-team, andrii
Cc: sinquersw, kuifeng, Kui-Feng Lee
From: Kui-Feng Lee <thinker.li@gmail.com>
Change the values of fields, including scalar types and function pointers,
and check if the struct_ops map works as expected.
The test changes the field "test_2" of "testmod_1" from the pointer to
test_2() to pointer to test_3() and the field "data" to 13. The function
test_2() and test_3() both compute a new value for "test_2_result", but in
different way. By checking the value of "test_2_result", it ensures the
struct_ops map works as expected with changes through shadow types.
Signed-off-by: Kui-Feng Lee <thinker.li@gmail.com>
---
.../selftests/bpf/bpf_testmod/bpf_testmod.c | 11 ++++++++++-
.../selftests/bpf/bpf_testmod/bpf_testmod.h | 8 ++++++++
.../bpf/prog_tests/test_struct_ops_module.c | 19 +++++++++++++++----
.../selftests/bpf/progs/struct_ops_module.c | 8 ++++++++
4 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
index 66787e99ba1b..098ddd067224 100644
--- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
+++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
@@ -539,6 +539,15 @@ static int bpf_testmod_ops_init_member(const struct btf_type *t,
const struct btf_member *member,
void *kdata, const void *udata)
{
+ if (member->offset == offsetof(struct bpf_testmod_ops, data) * 8) {
+ /* For data fields, this function has to copy it and return
+ * 1 to indicate that the data has been handled by the
+ * struct_ops type, or the verifier will reject the map if
+ * the value of the data field is not zero.
+ */
+ ((struct bpf_testmod_ops *)kdata)->data = ((struct bpf_testmod_ops *)udata)->data;
+ return 1;
+ }
return 0;
}
@@ -559,7 +568,7 @@ static int bpf_dummy_reg(void *kdata)
* initialized, so we need to check for NULL.
*/
if (ops->test_2)
- ops->test_2(4, 3);
+ ops->test_2(4, ops->data);
return 0;
}
diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h
index c3b0cf788f9f..971458acfac3 100644
--- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h
+++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h
@@ -35,6 +35,14 @@ struct bpf_testmod_ops {
void (*test_2)(int a, int b);
/* Used to test nullable arguments. */
int (*test_maybe_null)(int dummy, struct task_struct *task);
+
+ /* The following fields are used to test shadow copies. */
+ char onebyte;
+ struct {
+ int a;
+ int b;
+ } unsupported;
+ int data;
};
#endif /* _BPF_TESTMOD_H */
diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c
index 8d833f0c7580..7d6facf46ebb 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c
@@ -32,17 +32,23 @@ static void check_map_info(struct bpf_map_info *info)
static void test_struct_ops_load(void)
{
- DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);
struct struct_ops_module *skel;
struct bpf_map_info info = {};
struct bpf_link *link;
int err;
u32 len;
- skel = struct_ops_module__open_opts(&opts);
+ skel = struct_ops_module__open();
if (!ASSERT_OK_PTR(skel, "struct_ops_module_open"))
return;
+ skel->struct_ops.testmod_1->data = 13;
+ skel->struct_ops.testmod_1->test_2 = skel->progs.test_3;
+ /* Since test_2() is not being used, it should be disabled from
+ * auto-loading, or it will fail to load.
+ */
+ bpf_program__set_autoload(skel->progs.test_2, false);
+
err = struct_ops_module__load(skel);
if (!ASSERT_OK(err, "struct_ops_module_load"))
goto cleanup;
@@ -56,8 +62,13 @@ static void test_struct_ops_load(void)
link = bpf_map__attach_struct_ops(skel->maps.testmod_1);
ASSERT_OK_PTR(link, "attach_test_mod_1");
- /* test_2() will be called from bpf_dummy_reg() in bpf_testmod.c */
- ASSERT_EQ(skel->bss->test_2_result, 7, "test_2_result");
+ /* test_3() will be called from bpf_dummy_reg() in bpf_testmod.c
+ *
+ * In bpf_testmod.c it will pass 4 and 13 (the value of data) to
+ * .test_2. So, the value of test_2_result should be 20 (4 + 13 +
+ * 3).
+ */
+ ASSERT_EQ(skel->bss->test_2_result, 20, "check_shadow_variables");
bpf_link__destroy(link);
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_module.c b/tools/testing/selftests/bpf/progs/struct_ops_module.c
index b78746b3cef3..25952fa09348 100644
--- a/tools/testing/selftests/bpf/progs/struct_ops_module.c
+++ b/tools/testing/selftests/bpf/progs/struct_ops_module.c
@@ -21,9 +21,17 @@ void BPF_PROG(test_2, int a, int b)
test_2_result = a + b;
}
+SEC("struct_ops/test_3")
+int BPF_PROG(test_3, int a, int b)
+{
+ test_2_result = a + b + 3;
+ return a + b + 3;
+}
+
SEC(".struct_ops.link")
struct bpf_testmod_ops testmod_1 = {
.test_1 = (void *)test_1,
.test_2 = (void *)test_2,
+ .data = 0x1,
};
--
2.34.1
^ permalink raw reply related [flat|nested] 10+ messages in thread