From: Jiri Olsa <jolsa@redhat.com>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: netdev@vger.kernel.org, bpf@vger.kernel.org,
Martin KaFai Lau <kafai@fb.com>, Song Liu <songliubraving@fb.com>,
Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>
Subject: [PATCH bpf-next 10/29] bpf: Add bpf_trampoline_id object
Date: Thu, 18 Nov 2021 12:24:36 +0100 [thread overview]
Message-ID: <20211118112455.475349-11-jolsa@kernel.org> (raw)
In-Reply-To: <20211118112455.475349-1-jolsa@kernel.org>
Replacing the bpf_trampoline's key with struct bpf_tramp_id object,
that currently holds only obj_id/btf_id, so same data as key.
Having the key in the struct will allow us to add more ids (functions)
to single trampoline in following patches.
No functional change is intended.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 18 +++++++++--
include/linux/bpf_verifier.h | 19 ------------
kernel/bpf/syscall.c | 30 +++++++++++++-----
kernel/bpf/trampoline.c | 59 +++++++++++++++++++++++++++++++-----
kernel/bpf/verifier.c | 14 ++++++---
5 files changed, 99 insertions(+), 41 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 35f484f323f3..2ce8b1c49af7 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -671,13 +671,18 @@ struct bpf_tramp_image {
};
};
+struct bpf_tramp_id {
+ u32 obj_id;
+ u32 btf_id;
+};
+
struct bpf_trampoline {
/* hlist for trampoline_table */
struct hlist_node hlist;
/* serializes access to fields of this trampoline */
struct mutex mutex;
refcount_t refcnt;
- u64 key;
+ struct bpf_tramp_id *id;
struct {
struct btf_func_model model;
void *addr;
@@ -732,9 +737,16 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
return bpf_func(ctx, insnsi);
}
#ifdef CONFIG_BPF_JIT
+struct bpf_tramp_id *bpf_tramp_id_alloc(void);
+void bpf_tramp_id_free(struct bpf_tramp_id *id);
+bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id);
+int bpf_tramp_id_is_equal(struct bpf_tramp_id *a, struct bpf_tramp_id *b);
+void bpf_tramp_id_init(struct bpf_tramp_id *id,
+ const struct bpf_prog *tgt_prog,
+ struct btf *btf, u32 btf_id);
int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-struct bpf_trampoline *bpf_trampoline_get(u64 key,
+struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info);
void bpf_trampoline_put(struct bpf_trampoline *tr);
#define BPF_DISPATCHER_INIT(_name) { \
@@ -792,7 +804,7 @@ static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
{
return -ENOTSUPP;
}
-static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
+static inline struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info)
{
return ERR_PTR(-EOPNOTSUPP);
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b561c0b08e68..0c01a50180a0 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -503,25 +503,6 @@ int check_ctx_reg(struct bpf_verifier_env *env,
int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
u32 regno, u32 mem_size);
-/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
-static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
- struct btf *btf, u32 btf_id)
-{
- if (tgt_prog)
- return ((u64)tgt_prog->aux->id << 32) | btf_id;
- else
- return ((u64)btf_obj_id(btf) << 32) | 0x80000000 | btf_id;
-}
-
-/* unpack the IDs from the key as constructed above */
-static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id)
-{
- if (obj_id)
- *obj_id = key >> 32;
- if (btf_id)
- *btf_id = key & 0x7FFFFFFF;
-}
-
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 648155f0a4b1..f99ea3237f9c 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2703,9 +2703,8 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
struct bpf_trampoline *tr = link_trampoline(tr_link);
info->tracing.attach_type = tr_link->attach_type;
- bpf_trampoline_unpack_key(tr->key,
- &info->tracing.target_obj_id,
- &info->tracing.target_btf_id);
+ info->tracing.target_obj_id = tr->id->obj_id;
+ info->tracing.target_btf_id = tr->id->btf_id;
return 0;
}
@@ -2726,7 +2725,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
struct bpf_prog *tgt_prog = NULL;
struct bpf_trampoline *tr = NULL;
struct bpf_tracing_link *link;
- u64 key = 0;
+ struct bpf_tramp_id *id = NULL;
int err;
switch (prog->type) {
@@ -2767,6 +2766,12 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
+ id = bpf_tramp_id_alloc();
+ if (!id) {
+ err = -ENOMEM;
+ goto out_put_prog;
+ }
+
tgt_prog = bpf_prog_get(tgt_prog_fd);
if (IS_ERR(tgt_prog)) {
err = PTR_ERR(tgt_prog);
@@ -2774,7 +2779,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
- key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
+ bpf_tramp_id_init(id, tgt_prog, NULL, btf_id);
}
link = kzalloc(sizeof(*link), GFP_USER);
@@ -2823,12 +2828,20 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
err = -EINVAL;
goto out_unlock;
}
+
+ id = bpf_tramp_id_alloc();
+ if (!id) {
+ err = -ENOMEM;
+ goto out_unlock;
+ }
+
btf_id = prog->aux->attach_btf_id;
- key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
+ bpf_tramp_id_init(id, NULL, prog->aux->attach_btf, btf_id);
}
if (!prog->aux->dst_trampoline ||
- (key && key != prog->aux->dst_trampoline->key)) {
+ (!bpf_tramp_id_is_empty(id) &&
+ bpf_tramp_id_is_equal(id, prog->aux->dst_trampoline->id))) {
/* If there is no saved target, or the specified target is
* different from the destination specified at load time, we
* need a new trampoline and a check for compatibility
@@ -2840,7 +2853,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
if (err)
goto out_unlock;
- tr = bpf_trampoline_get(key, &tgt_info);
+ tr = bpf_trampoline_get(id, &tgt_info);
if (!tr) {
err = -ENOMEM;
goto out_unlock;
@@ -2900,6 +2913,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
out_put_prog:
if (tgt_prog_fd && tgt_prog)
bpf_prog_put(tgt_prog);
+ bpf_tramp_id_free(id);
return err;
}
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index e98de5e73ba5..ae2573c36653 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -59,16 +59,57 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym)
PAGE_SIZE, true, ksym->name);
}
-static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
+static u64 bpf_tramp_id_key(struct bpf_tramp_id *id)
+{
+ return ((u64) id->obj_id << 32) | id->btf_id;
+}
+
+bool bpf_tramp_id_is_empty(struct bpf_tramp_id *id)
+{
+ return !id || (!id->obj_id && !id->btf_id);
+}
+
+int bpf_tramp_id_is_equal(struct bpf_tramp_id *a,
+ struct bpf_tramp_id *b)
+{
+ return !memcmp(a, b, sizeof(*a));
+}
+
+struct bpf_tramp_id *bpf_tramp_id_alloc(void)
+{
+ struct bpf_tramp_id *id;
+
+ return kzalloc(sizeof(*id), GFP_KERNEL);
+}
+
+void bpf_tramp_id_init(struct bpf_tramp_id *id,
+ const struct bpf_prog *tgt_prog,
+ struct btf *btf, u32 btf_id)
+{
+ if (tgt_prog)
+ id->obj_id = tgt_prog->aux->id;
+ else
+ id->obj_id = btf_obj_id(btf);
+ id->btf_id = btf_id;
+}
+
+void bpf_tramp_id_free(struct bpf_tramp_id *id)
+{
+ kfree(id);
+}
+
+static struct bpf_trampoline *bpf_trampoline_lookup(struct bpf_tramp_id *id)
{
struct bpf_trampoline *tr;
struct hlist_head *head;
+ u64 key;
int i;
+ key = bpf_tramp_id_key(id);
mutex_lock(&trampoline_mutex);
head = &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)];
hlist_for_each_entry(tr, head, hlist) {
- if (tr->key == key) {
+ if (bpf_tramp_id_is_equal(tr->id, id)) {
refcount_inc(&tr->refcnt);
goto out;
}
@@ -77,7 +118,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
if (!tr)
goto out;
- tr->key = key;
+ tr->id = id;
INIT_HLIST_NODE(&tr->hlist);
hlist_add_head(&tr->hlist, head);
refcount_set(&tr->refcnt, 1);
@@ -291,12 +332,14 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
}
-static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
+static struct bpf_tramp_image*
+bpf_tramp_image_alloc(struct bpf_tramp_id *id, u32 idx)
{
struct bpf_tramp_image *im;
struct bpf_ksym *ksym;
void *image;
int err = -ENOMEM;
+ u64 key;
im = kzalloc(sizeof(*im), GFP_KERNEL);
if (!im)
@@ -317,6 +360,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
ksym = &im->ksym;
INIT_LIST_HEAD_RCU(&ksym->lnode);
+ key = bpf_tramp_id_key(id);
snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu_%u", key, idx);
bpf_image_ksym_add(image, ksym);
return im;
@@ -351,7 +395,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
goto out;
}
- im = bpf_tramp_image_alloc(tr->key, tr->selector);
+ im = bpf_tramp_image_alloc(tr->id, tr->selector);
if (IS_ERR(im)) {
err = PTR_ERR(im);
goto out;
@@ -482,12 +526,12 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
return err;
}
-struct bpf_trampoline *bpf_trampoline_get(u64 key,
+struct bpf_trampoline *bpf_trampoline_get(struct bpf_tramp_id *id,
struct bpf_attach_target_info *tgt_info)
{
struct bpf_trampoline *tr;
- tr = bpf_trampoline_lookup(key);
+ tr = bpf_trampoline_lookup(id);
if (!tr)
return NULL;
@@ -521,6 +565,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
* multiple rcu callbacks.
*/
hlist_del(&tr->hlist);
+ bpf_tramp_id_free(tr->id);
kfree(tr);
out:
mutex_unlock(&trampoline_mutex);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index af6a39bbb0dc..a1e4389b0e9e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13929,8 +13929,8 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
struct bpf_attach_target_info tgt_info = {};
u32 btf_id = prog->aux->attach_btf_id;
struct bpf_trampoline *tr;
+ struct bpf_tramp_id *id;
int ret;
- u64 key;
if (prog->type == BPF_PROG_TYPE_SYSCALL) {
if (prog->aux->sleepable)
@@ -13995,11 +13995,17 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
return -EINVAL;
}
- key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
- tr = bpf_trampoline_get(key, &tgt_info);
- if (!tr)
+ id = bpf_tramp_id_alloc();
+ if (!id)
return -ENOMEM;
+ bpf_tramp_id_init(id, tgt_prog, prog->aux->attach_btf, btf_id);
+ tr = bpf_trampoline_get(id, &tgt_info);
+ if (!tr) {
+ bpf_tramp_id_free(id);
+ return -ENOMEM;
+ }
+
prog->aux->dst_trampoline = tr;
return 0;
}
--
2.31.1
next prev parent reply other threads:[~2021-11-18 11:27 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-18 11:24 [RFC bpf-next v5 00/29] bpf: Add batch support for attaching trampolines Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 01/29] ftrace: Use direct_ops hash in unregister_ftrace_direct Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 02/29] ftrace: Add cleanup to unregister_ftrace_direct_multi Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 03/29] ftrace: Add ftrace_set_filter_ips function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 04/29] bpf: Factor bpf_check_attach_target function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 05/29] bpf: Add bpf_check_attach_model function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 06/29] bpf: Add bpf_arg/bpf_ret_value helpers for tracing programs Jiri Olsa
2021-11-24 21:43 ` Andrii Nakryiko
2021-11-25 16:14 ` Alexei Starovoitov
2021-11-28 18:07 ` Jiri Olsa
2021-11-28 18:06 ` Jiri Olsa
2021-12-01 7:13 ` Andrii Nakryiko
2021-12-01 17:37 ` Alexei Starovoitov
2021-12-01 17:59 ` Andrii Nakryiko
2021-12-01 20:36 ` Alexei Starovoitov
2021-12-01 21:16 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 07/29] bpf, x64: Allow to use caller address from stack Jiri Olsa
2021-11-19 4:14 ` Alexei Starovoitov
2021-11-19 21:46 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 08/29] bpf: Keep active attached trampoline in bpf_prog Jiri Olsa
2021-11-24 21:48 ` Andrii Nakryiko
2021-11-28 17:24 ` Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 09/29] bpf: Add support to load multi func tracing program Jiri Olsa
2021-11-19 4:11 ` Alexei Starovoitov
2021-11-22 20:15 ` Jiri Olsa
2021-11-24 21:51 ` Andrii Nakryiko
2021-11-28 17:41 ` Jiri Olsa
2021-12-01 7:17 ` Andrii Nakryiko
2021-12-01 21:20 ` Jiri Olsa
2021-11-18 11:24 ` Jiri Olsa [this message]
2021-11-18 11:24 ` [PATCH bpf-next 11/29] bpf: Add addr to bpf_trampoline_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 12/29] bpf: Add struct bpf_tramp_node layer Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 13/29] bpf: Add bpf_tramp_attach layer for trampoline attachment Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 14/29] bpf: Add support to store multiple ids in bpf_tramp_id object Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 15/29] bpf: Add support to store multiple addrs " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 16/29] bpf: Add bpf_tramp_id_single function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 17/29] bpf: Resolve id in bpf_tramp_id_single Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 18/29] bpf: Add refcount_t to struct bpf_tramp_id Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 19/29] bpf: Add support to attach trampolines with multiple IDs Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 20/29] bpf: Add support for tracing multi link Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 21/29] libbpf: Add btf__find_by_glob_kind function Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 22/29] libbpf: Add support to link multi func tracing program Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 23/29] selftests/bpf: Add bpf_arg/bpf_ret_value test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 24/29] selftests/bpf: Add fentry multi func test Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 25/29] selftests/bpf: Add fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 26/29] selftests/bpf: Add fentry/fexit " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 27/29] selftests/bpf: Add mixed " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 28/29] selftests/bpf: Add ret_mod " Jiri Olsa
2021-11-18 11:24 ` [PATCH bpf-next 29/29] selftests/bpf: Add attach " Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211118112455.475349-11-jolsa@kernel.org \
--to=jolsa@redhat.com \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=netdev@vger.kernel.org \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).