All of lore.kernel.org
 help / color / mirror / Atom feed
From: Jiri Olsa <jolsa@redhat.com>
To: Alexei Starovoitov <ast@kernel.org>,
	Daniel Borkmann <daniel@iogearbox.net>,
	Andrii Nakryiko <andriin@fb.com>,
	"Steven Rostedt (VMware)" <rostedt@goodmis.org>
Cc: netdev@vger.kernel.org, bpf@vger.kernel.org,
	Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
	Yonghong Song <yhs@fb.com>,
	John Fastabend <john.fastabend@gmail.com>,
	KP Singh <kpsingh@chromium.org>, Daniel Xu <dxu@dxuuu.xyz>,
	Viktor Malik <vmalik@redhat.com>
Subject: [PATCH bpf-next v4 10/27] bpf: Add struct bpf_tramp_node layer
Date: Thu, 26 Aug 2021 21:39:05 +0200	[thread overview]
Message-ID: <20210826193922.66204-11-jolsa@kernel.org> (raw)
In-Reply-To: <20210826193922.66204-1-jolsa@kernel.org>

Currently each trampoline holds a list of programs that
are attached to it. With multi func attach support we need
a way for a single program to be connected to multiple
trampolines.

Adding struct bpf_tramp_node object that holds bpf_prog
pointer, so it can be resolved directly. We can now
have multiple struct bpf_tramp_node being attached to
different trampolines pointing to single bpf_prog.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
 include/linux/bpf.h     | 15 ++++++++++-----
 kernel/bpf/core.c       |  1 +
 kernel/bpf/syscall.c    |  4 ++--
 kernel/bpf/trampoline.c | 22 ++++++++++++----------
 4 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index dc9838d741ac..f0f548f8f391 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -656,6 +656,11 @@ struct bpf_tramp_image {
 	};
 };
 
+struct bpf_tramp_node {
+	struct hlist_node hlist;
+	struct bpf_prog *prog;
+};
+
 struct bpf_trampoline {
 	/* hlist for trampoline_table */
 	struct hlist_node hlist;
@@ -717,8 +722,8 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
 	return bpf_func(ctx, insnsi);
 }
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
 					  struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -767,12 +772,12 @@ void bpf_ksym_del(struct bpf_ksym *ksym);
 int bpf_jit_charge_modmem(u32 pages);
 void bpf_jit_uncharge_modmem(u32 pages);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
 					   struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
 					     struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
@@ -866,7 +871,7 @@ struct bpf_prog_aux {
 	bool sleepable;
 	bool tail_call_reachable;
 	bool multi_func;
-	struct hlist_node tramp_hlist;
+	struct bpf_tramp_node tramp_node;
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
 	const struct btf_type *attach_func_proto;
 	/* function name for valid attach_btf_id */
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 9f4636d021b1..bad03dde97a2 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -104,6 +104,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
 	fp->aux = aux;
 	fp->aux->prog = fp;
 	fp->jit_requested = ebpf_jit_enabled();
+	fp->aux->tramp_node.prog = fp;
 
 	INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);
 	mutex_init(&fp->aux->used_maps_mutex);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index fa3f93c423d8..e667d392cc33 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2625,7 +2625,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
 	struct bpf_tracing_link *tr_link =
 		container_of(link, struct bpf_tracing_link, link);
 
-	WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
+	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&link->prog->aux->tramp_node,
 						tr_link->trampoline));
 
 	bpf_trampoline_put(tr_link->trampoline);
@@ -2813,7 +2813,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
 	if (err)
 		goto out_unlock;
 
-	err = bpf_trampoline_link_prog(prog, tr);
+	err = bpf_trampoline_link_prog(&prog->aux->tramp_node, tr);
 	if (err) {
 		bpf_link_cleanup(&link_primer);
 		link = NULL;
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index fe1e857324e6..525fa74c2f62 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -174,8 +174,8 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
 static struct bpf_tramp_progs *
 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
 {
-	const struct bpf_prog_aux *aux;
 	struct bpf_tramp_progs *tprogs;
+	struct bpf_tramp_node *node;
 	struct bpf_prog **progs;
 	int kind;
 
@@ -189,9 +189,9 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a
 		*total += tr->progs_cnt[kind];
 		progs = tprogs[kind].progs;
 
-		hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {
-			*ip_arg |= aux->prog->call_get_func_ip;
-			*progs++ = aux->prog;
+		hlist_for_each_entry(node, &tr->progs_hlist[kind], hlist) {
+			*ip_arg |= node->prog->call_get_func_ip;
+			*progs++ = node->prog;
 		}
 	}
 	return tprogs;
@@ -410,8 +410,9 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
 	}
 }
 
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr)
 {
+	struct bpf_prog *prog = node->prog;
 	enum bpf_tramp_prog_type kind;
 	int err = 0;
 	int cnt;
@@ -441,16 +442,16 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
 		err = -E2BIG;
 		goto out;
 	}
-	if (!hlist_unhashed(&prog->aux->tramp_hlist)) {
+	if (!hlist_unhashed(&node->hlist)) {
 		/* prog already linked */
 		err = -EBUSY;
 		goto out;
 	}
-	hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
+	hlist_add_head(&node->hlist, &tr->progs_hlist[kind]);
 	tr->progs_cnt[kind]++;
 	err = bpf_trampoline_update(tr);
 	if (err) {
-		hlist_del_init(&prog->aux->tramp_hlist);
+		hlist_del_init(&node->hlist);
 		tr->progs_cnt[kind]--;
 	}
 out:
@@ -459,8 +460,9 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
 }
 
 /* bpf_trampoline_unlink_prog() should never fail. */
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr)
 {
+	struct bpf_prog *prog = node->prog;
 	enum bpf_tramp_prog_type kind;
 	int err;
 
@@ -473,7 +475,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
 		tr->extension_prog = NULL;
 		goto out;
 	}
-	hlist_del_init(&prog->aux->tramp_hlist);
+	hlist_del_init(&node->hlist);
 	tr->progs_cnt[kind]--;
 	err = bpf_trampoline_update(tr);
 out:
-- 
2.31.1


  parent reply	other threads:[~2021-08-26 19:40 UTC|newest]

Thread overview: 44+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-08-26 19:38 [PATCH bpf-next v4 00/27] x86/ftrace/bpf: Add batch support for direct/tracing attach Jiri Olsa
2021-08-26 19:38 ` [PATCH bpf-next v4 01/27] x86/ftrace: Remove extra orig rax move Jiri Olsa
2021-08-26 19:38 ` [PATCH bpf-next v4 02/27] x86/ftrace: Remove fault protection code in prepare_ftrace_return Jiri Olsa
2021-08-26 19:38 ` [PATCH bpf-next v4 03/27] x86/ftrace: Make function graph use ftrace directly Jiri Olsa
2021-08-26 19:38 ` [PATCH bpf-next v4 04/27] tracing: Add trampoline/graph selftest Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 05/27] ftrace: Add ftrace_add_rec_direct function Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 06/27] ftrace: Add multi direct register/unregister interface Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 07/27] ftrace: Add multi direct modify interface Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 08/27] ftrace/samples: Add multi direct interface test module Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 09/27] bpf: Add support to load multi func tracing program Jiri Olsa
2021-08-31 23:17   ` Andrii Nakryiko
2021-09-01 11:32     ` Jiri Olsa
2021-08-26 19:39 ` Jiri Olsa [this message]
2021-08-26 19:39 ` [PATCH bpf-next v4 11/27] bpf: Factor out bpf_trampoline_init function Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 12/27] bpf: Factor out __bpf_trampoline_lookup function Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 13/27] bpf: Factor out __bpf_trampoline_put function Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 14/27] bpf: Change bpf_trampoline_get to return error pointer Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 15/27] bpf, x64: Allow to use caller address from stack Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 16/27] bpf: Add bpf_trampoline_multi_get/put functions Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 17/27] bpf: Add multi trampoline attach support Jiri Olsa
2021-08-31 23:36   ` Andrii Nakryiko
2021-09-01  0:02     ` Andrii Nakryiko
2021-09-01 11:39     ` Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 18/27] bpf, x64: Store properly return value for trampoline with multi func programs Jiri Olsa
2021-08-31 23:51   ` Andrii Nakryiko
2021-09-01 15:15     ` Jiri Olsa
2021-09-02  3:56       ` Andrii Nakryiko
2021-09-02 12:57         ` Jiri Olsa
2021-09-02 16:54           ` Andrii Nakryiko
2021-09-02 21:55           ` Alexei Starovoitov
2021-09-03  9:50             ` Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 19/27] bpf: Attach multi trampoline with ftrace_ops Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 20/27] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2021-09-01  0:10   ` Andrii Nakryiko
2021-09-01 11:33     ` Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 21/27] libbpf: Add support to link multi func tracing program Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 22/27] selftests/bpf: Add fentry multi func test Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 23/27] selftests/bpf: Add fexit " Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 24/27] selftests/bpf: Add fentry/fexit " Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 25/27] selftests/bpf: Add mixed " Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 26/27] selftests/bpf: Add attach " Jiri Olsa
2021-08-26 19:39 ` [PATCH bpf-next v4 27/27] selftests/bpf: Add ret_mod " Jiri Olsa
2021-08-29 17:04 ` [PATCH bpf-next v4 00/27] x86/ftrace/bpf: Add batch support for direct/tracing attach Alexei Starovoitov
2021-08-30  8:02   ` Jiri Olsa

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210826193922.66204-11-jolsa@kernel.org \
    --to=jolsa@redhat.com \
    --cc=andriin@fb.com \
    --cc=ast@kernel.org \
    --cc=bpf@vger.kernel.org \
    --cc=daniel@iogearbox.net \
    --cc=dxu@dxuuu.xyz \
    --cc=john.fastabend@gmail.com \
    --cc=kafai@fb.com \
    --cc=kpsingh@chromium.org \
    --cc=netdev@vger.kernel.org \
    --cc=rostedt@goodmis.org \
    --cc=songliubraving@fb.com \
    --cc=vmalik@redhat.com \
    --cc=yhs@fb.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.