From: Jiri Olsa <jolsa@redhat.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: netdev@vger.kernel.org, bpf@vger.kernel.org,
Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>
Subject: [PATCH bpf-next 12/29] bpf: Add struct bpf_tramp_node layer
Date: Thu, 18 Nov 2021 12:24:38 +0100 [thread overview]
Message-ID: <20211118112455.475349-13-jolsa@kernel.org> (raw)
In-Reply-To: <20211118112455.475349-1-jolsa@kernel.org>
Currently each trampoline holds a list of programs that
are attached to it. With multi func attach support we need
a way for a single program to be connected to multiple
trampolines.
Adding struct bpf_tramp_node object that holds bpf_prog
pointer, so it can be resolved directly. We can now
have multiple struct bpf_tramp_node being attached to
different trampolines pointing to single bpf_prog.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 15 ++++++++++-----
kernel/bpf/core.c | 1 +
kernel/bpf/syscall.c | 4 ++--
kernel/bpf/trampoline.c | 22 ++++++++++++----------
4 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index c57141a76e7b..21f8dbcf3f48 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -677,6 +677,11 @@ struct bpf_tramp_id {
void *addr;
};
+struct bpf_tramp_node {
+ struct hlist_node hlist_tramp;
+ struct bpf_prog *prog;
+};
+
struct bpf_trampoline {
/* hlist for trampoline_table */
struct hlist_node hlist;
@@ -744,8 +749,8 @@ int bpf_tramp_id_is_equal(struct bpf_tramp_id *a, struct bpf_tramp_id *b);
void bpf_tramp_id_init(struct bpf_tramp_id *id,
const struct bpf_prog *tgt_prog,
struct btf *btf, u32 btf_id);
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr);
struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -794,12 +799,12 @@ void bpf_ksym_del(struct bpf_ksym *ksym);
int bpf_jit_charge_modmem(u32 pages);
void bpf_jit_uncharge_modmem(u32 pages);
#else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr)
{
return -ENOTSUPP;
}
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
struct bpf_trampoline *tr)
{
return -ENOTSUPP;
@@ -894,7 +899,7 @@ struct bpf_prog_aux {
bool sleepable;
bool tail_call_reachable;
bool multi_func;
- struct hlist_node tramp_hlist;
+ struct bpf_tramp_node tramp_node;
/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
const struct btf_type *attach_func_proto;
/* function name for valid attach_btf_id */
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index b52dc845ecea..2eed91153a3f 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -105,6 +105,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
fp->aux = aux;
fp->aux->prog = fp;
fp->jit_requested = ebpf_jit_enabled();
+ fp->aux->tramp_node.prog = fp;
INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);
mutex_init(&fp->aux->used_maps_mutex);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index f99ea3237f9c..0d916e3b7676 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2665,7 +2665,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
struct bpf_trampoline *tr = link_trampoline(tr_link);
struct bpf_prog *prog = link->prog;
- WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog, tr));
+ WARN_ON_ONCE(bpf_trampoline_unlink_prog(&link->prog->aux->tramp_node, tr));
if (prog->type != BPF_PROG_TYPE_EXT)
prog->aux->trampoline = NULL;
@@ -2874,7 +2874,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
if (err)
goto out_unlock;
- err = bpf_trampoline_link_prog(prog, tr);
+ err = bpf_trampoline_link_prog(&prog->aux->tramp_node, tr);
if (err) {
bpf_link_cleanup(&link_primer);
link = NULL;
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index e19c5112be67..b6af3e0982d7 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -216,8 +216,8 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
static struct bpf_tramp_progs *
bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
{
- const struct bpf_prog_aux *aux;
struct bpf_tramp_progs *tprogs;
+ struct bpf_tramp_node *node;
struct bpf_prog **progs;
int kind;
@@ -231,9 +231,9 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a
*total += tr->progs_cnt[kind];
progs = tprogs[kind].progs;
- hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {
- *ip_arg |= aux->prog->call_get_func_ip;
- *progs++ = aux->prog;
+ hlist_for_each_entry(node, &tr->progs_hlist[kind], hlist_tramp) {
+ *ip_arg |= node->prog->call_get_func_ip;
+ *progs++ = node->prog;
}
}
return tprogs;
@@ -455,8 +455,9 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
}
}
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr)
{
+ struct bpf_prog *prog = node->prog;
enum bpf_tramp_prog_type kind;
int err = 0;
int cnt;
@@ -486,16 +487,16 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
err = -E2BIG;
goto out;
}
- if (!hlist_unhashed(&prog->aux->tramp_hlist)) {
+ if (!hlist_unhashed(&node->hlist_tramp)) {
/* prog already linked */
err = -EBUSY;
goto out;
}
- hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
+ hlist_add_head(&node->hlist_tramp, &tr->progs_hlist[kind]);
tr->progs_cnt[kind]++;
err = bpf_trampoline_update(tr);
if (err) {
- hlist_del_init(&prog->aux->tramp_hlist);
+ hlist_del_init(&node->hlist_tramp);
tr->progs_cnt[kind]--;
}
out:
@@ -504,8 +505,9 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
}
/* bpf_trampoline_unlink_prog() should never fail. */
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr)
{
+ struct bpf_prog *prog = node->prog;
enum bpf_tramp_prog_type kind;
int err;
@@ -518,7 +520,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
tr->extension_prog = NULL;
goto out;
}
- hlist_del_init(&prog->aux->tramp_hlist);
+ hlist_del_init(&node->hlist_tramp);
tr->progs_cnt[kind]--;
err = bpf_trampoline_update(tr);
out:
--
2.31.1
next prev parent reply other threads:[~2021-11-18 11:28 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-18 11:24 [RFC bpf-next v5 00/29] bpf: Add batch support for attaching trampolines Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 01/29] ftrace: Use direct_ops hash in unregister_ftrace_direct Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 02/29] ftrace: Add cleanup to unregister_ftrace_direct_multi Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 03/29] ftrace: Add ftrace_set_filter_ips function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 04/29] bpf: Factor bpf_check_attach_target function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 05/29] bpf: Add bpf_check_attach_model function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 06/29] bpf: Add bpf_arg/bpf_ret_value helpers for tracing programs Jiri Olsa
2021-11-24 21:43 ` Andrii Nakryiko
2021-11-25 16:14 ` Alexei Starovoitov
2021-11-28 18:07 ` Jiri Olsa
2021-11-28 18:06 ` Jiri Olsa
2021-12-01 7:13 ` Andrii Nakryiko
2021-12-01 17:37 ` Alexei Starovoitov
2021-12-01 17:59 ` Andrii Nakryiko
2021-12-01 20:36 ` Alexei Starovoitov
2021-12-01 21:16 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 07/29] bpf, x64: Allow to use caller address from stack Jiri Olsa
2021-11-19 4:14 ` Alexei Starovoitov
2021-11-19 21:46 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 08/29] bpf: Keep active attached trampoline in bpf_prog Jiri Olsa
2021-11-24 21:48 ` Andrii Nakryiko
2021-11-28 17:24 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 09/29] bpf: Add support to load multi func tracing program Jiri Olsa
2021-11-19 4:11 ` Alexei Starovoitov
2021-11-22 20:15 ` Jiri Olsa
2021-11-24 21:51 ` Andrii Nakryiko
2021-11-28 17:41 ` Jiri Olsa
2021-12-01 7:17 ` Andrii Nakryiko
2021-12-01 21:20 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 10/29] bpf: Add bpf_trampoline_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 11/29] bpf: Add addr to " Jiri Olsa
2021-11-18 11:24 ` Jiri Olsa [this message]
2021-11-18 11:24 ` [PATCH bpf-next 13/29] bpf: Add bpf_tramp_attach layer for trampoline attachment Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 14/29] bpf: Add support to store multiple ids in bpf_tramp_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 15/29] bpf: Add support to store multiple addrs " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 16/29] bpf: Add bpf_tramp_id_single function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 17/29] bpf: Resolve id in bpf_tramp_id_single Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 18/29] bpf: Add refcount_t to struct bpf_tramp_id Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 19/29] bpf: Add support to attach trampolines with multiple IDs Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 20/29] bpf: Add support for tracing multi link Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 21/29] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 22/29] libbpf: Add support to link multi func tracing program Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 23/29] selftests/bpf: Add bpf_arg/bpf_ret_value test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 24/29] selftests/bpf: Add fentry multi func test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 25/29] selftests/bpf: Add fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 26/29] selftests/bpf: Add fentry/fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 27/29] selftests/bpf: Add mixed " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 28/29] selftests/bpf: Add ret_mod " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 29/29] selftests/bpf: Add attach " Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211118112455.475349-13-jolsa@kernel.org \
--to=jolsa@redhat.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=netdev@vger.kernel.org \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).