* [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array
@ 2019-05-28 18:29 Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 2/4] bpf: media: properly use bpf_prog_array api Stanislav Fomichev
` (3 more replies)
0 siblings, 4 replies; 9+ messages in thread
From: Stanislav Fomichev @ 2019-05-28 18:29 UTC (permalink / raw)
To: netdev, bpf; +Cc: davem, ast, daniel, Stanislav Fomichev, Roman Gushchin
Drop __rcu annotations and rcu read sections from bpf_prog_array
helper functions. They are not needed since all existing callers
call those helpers from the rcu update side while holding a mutex.
This guarantees that use-after-free could not happen.
In the next patches I'll fix the callers with missing
rcu_dereference_protected to make sparse/lockdep happy, the proper
way to use these helpers is:
struct bpf_prog_array __rcu *progs = ...;
struct bpf_prog_array *p;
mutex_lock(&mtx);
p = rcu_dereference_protected(progs, lockdep_is_held(&mtx));
bpf_prog_array_length(p);
bpf_prog_array_copy_to_user(p, ...);
bpf_prog_array_delete_safe(p, ...);
bpf_prog_array_copy_info(p, ...);
bpf_prog_array_copy(p, ...);
bpf_prog_array_free(p);
mutex_unlock(&mtx);
No functional changes! rcu_dereference_protected with lockdep_is_held
should catch any cases where we update prog array without a mutex
(I've looked at existing call sites and I think we hold a mutex
everywhere).
Motivation is to fix sparse warnings:
kernel/bpf/core.c:1803:9: warning: incorrect type in argument 1 (different address spaces)
kernel/bpf/core.c:1803:9: expected struct callback_head *head
kernel/bpf/core.c:1803:9: got struct callback_head [noderef] <asn:4> *
kernel/bpf/core.c:1877:44: warning: incorrect type in initializer (different address spaces)
kernel/bpf/core.c:1877:44: expected struct bpf_prog_array_item *item
kernel/bpf/core.c:1877:44: got struct bpf_prog_array_item [noderef] <asn:4> *
kernel/bpf/core.c:1901:26: warning: incorrect type in assignment (different address spaces)
kernel/bpf/core.c:1901:26: expected struct bpf_prog_array_item *existing
kernel/bpf/core.c:1901:26: got struct bpf_prog_array_item [noderef] <asn:4> *
kernel/bpf/core.c:1935:26: warning: incorrect type in assignment (different address spaces)
kernel/bpf/core.c:1935:26: expected struct bpf_prog_array_item *[assigned] existing
kernel/bpf/core.c:1935:26: got struct bpf_prog_array_item [noderef] <asn:4> *
v2:
* remove comment about potential race; that can't happen
because all callers are in rcu-update section
Cc: Roman Gushchin <guro@fb.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
include/linux/bpf.h | 12 ++++++------
kernel/bpf/core.c | 37 +++++++++++++------------------------
2 files changed, 19 insertions(+), 30 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index d98141edb74b..ff3e00ff84d2 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -514,17 +514,17 @@ struct bpf_prog_array {
};
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
-void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
-int bpf_prog_array_length(struct bpf_prog_array __rcu *progs);
-int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
+void bpf_prog_array_free(struct bpf_prog_array *progs);
+int bpf_prog_array_length(struct bpf_prog_array *progs);
+int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
__u32 __user *prog_ids, u32 cnt);
-void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
+void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
struct bpf_prog *old_prog);
-int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
+int bpf_prog_array_copy_info(struct bpf_prog_array *array,
u32 *prog_ids, u32 request_cnt,
u32 *prog_cnt);
-int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
+int bpf_prog_array_copy(struct bpf_prog_array *old_array,
struct bpf_prog *exclude_prog,
struct bpf_prog *include_prog,
struct bpf_prog_array **new_array);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 3675b19ecb90..33fb292f2e30 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1795,38 +1795,33 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
return &empty_prog_array.hdr;
}
-void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
+void bpf_prog_array_free(struct bpf_prog_array *progs)
{
- if (!progs ||
- progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
+ if (!progs || progs == &empty_prog_array.hdr)
return;
kfree_rcu(progs, rcu);
}
-int bpf_prog_array_length(struct bpf_prog_array __rcu *array)
+int bpf_prog_array_length(struct bpf_prog_array *array)
{
struct bpf_prog_array_item *item;
u32 cnt = 0;
- rcu_read_lock();
- item = rcu_dereference(array)->items;
- for (; item->prog; item++)
+ for (item = array->items; item->prog; item++)
if (item->prog != &dummy_bpf_prog.prog)
cnt++;
- rcu_read_unlock();
return cnt;
}
-static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
+static bool bpf_prog_array_copy_core(struct bpf_prog_array *array,
u32 *prog_ids,
u32 request_cnt)
{
struct bpf_prog_array_item *item;
int i = 0;
- item = rcu_dereference_check(array, 1)->items;
- for (; item->prog; item++) {
+ for (item = array->items; item->prog; item++) {
if (item->prog == &dummy_bpf_prog.prog)
continue;
prog_ids[i] = item->prog->aux->id;
@@ -1839,7 +1834,7 @@ static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
return !!(item->prog);
}
-int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
+int bpf_prog_array_copy_to_user(struct bpf_prog_array *array,
__u32 __user *prog_ids, u32 cnt)
{
unsigned long err = 0;
@@ -1850,18 +1845,12 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
* cnt = bpf_prog_array_length();
* if (cnt > 0)
* bpf_prog_array_copy_to_user(..., cnt);
- * so below kcalloc doesn't need extra cnt > 0 check, but
- * bpf_prog_array_length() releases rcu lock and
- * prog array could have been swapped with empty or larger array,
- * so always copy 'cnt' prog_ids to the user.
- * In a rare race the user will see zero prog_ids
+ * so below kcalloc doesn't need extra cnt > 0 check.
*/
ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN);
if (!ids)
return -ENOMEM;
- rcu_read_lock();
nospc = bpf_prog_array_copy_core(array, ids, cnt);
- rcu_read_unlock();
err = copy_to_user(prog_ids, ids, cnt * sizeof(u32));
kfree(ids);
if (err)
@@ -1871,19 +1860,19 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
return 0;
}
-void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *array,
+void bpf_prog_array_delete_safe(struct bpf_prog_array *array,
struct bpf_prog *old_prog)
{
- struct bpf_prog_array_item *item = array->items;
+ struct bpf_prog_array_item *item;
- for (; item->prog; item++)
+ for (item = array->items; item->prog; item++)
if (item->prog == old_prog) {
WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);
break;
}
}
-int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
+int bpf_prog_array_copy(struct bpf_prog_array *old_array,
struct bpf_prog *exclude_prog,
struct bpf_prog *include_prog,
struct bpf_prog_array **new_array)
@@ -1947,7 +1936,7 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
return 0;
}
-int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
+int bpf_prog_array_copy_info(struct bpf_prog_array *array,
u32 *prog_ids, u32 request_cnt,
u32 *prog_cnt)
{
--
2.22.0.rc1.257.g3120a18244-goog
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH bpf-next v3 2/4] bpf: media: properly use bpf_prog_array api
2019-05-28 18:29 [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Stanislav Fomichev
@ 2019-05-28 18:29 ` Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 3/4] bpf: cgroup: " Stanislav Fomichev
` (2 subsequent siblings)
3 siblings, 0 replies; 9+ messages in thread
From: Stanislav Fomichev @ 2019-05-28 18:29 UTC (permalink / raw)
To: netdev, bpf
Cc: davem, ast, daniel, Stanislav Fomichev, linux-media,
Mauro Carvalho Chehab, Sean Young
Now that we don't have __rcu markers on the bpf_prog_array helpers,
let's use proper rcu_dereference_protected to obtain array pointer
under mutex.
Cc: linux-media@vger.kernel.org
Cc: Mauro Carvalho Chehab <mchehab@kernel.org>
Cc: Sean Young <sean@mess.org>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
drivers/media/rc/bpf-lirc.c | 30 +++++++++++++++++-------------
1 file changed, 17 insertions(+), 13 deletions(-)
diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c
index ee657003c1a1..0a0ce620e4a2 100644
--- a/drivers/media/rc/bpf-lirc.c
+++ b/drivers/media/rc/bpf-lirc.c
@@ -8,6 +8,9 @@
#include <linux/bpf_lirc.h>
#include "rc-core-priv.h"
+#define lirc_rcu_dereference(p) \
+ rcu_dereference_protected(p, lockdep_is_held(&ir_raw_handler_lock))
+
/*
* BPF interface for raw IR
*/
@@ -136,7 +139,7 @@ const struct bpf_verifier_ops lirc_mode2_verifier_ops = {
static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
{
- struct bpf_prog_array __rcu *old_array;
+ struct bpf_prog_array *old_array;
struct bpf_prog_array *new_array;
struct ir_raw_event_ctrl *raw;
int ret;
@@ -154,12 +157,12 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
goto unlock;
}
- if (raw->progs && bpf_prog_array_length(raw->progs) >= BPF_MAX_PROGS) {
+ old_array = lirc_rcu_dereference(raw->progs);
+ if (old_array && bpf_prog_array_length(old_array) >= BPF_MAX_PROGS) {
ret = -E2BIG;
goto unlock;
}
- old_array = raw->progs;
ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
if (ret < 0)
goto unlock;
@@ -174,7 +177,7 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog)
{
- struct bpf_prog_array __rcu *old_array;
+ struct bpf_prog_array *old_array;
struct bpf_prog_array *new_array;
struct ir_raw_event_ctrl *raw;
int ret;
@@ -192,7 +195,7 @@ static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog)
goto unlock;
}
- old_array = raw->progs;
+ old_array = lirc_rcu_dereference(raw->progs);
ret = bpf_prog_array_copy(old_array, prog, NULL, &new_array);
/*
* Do not use bpf_prog_array_delete_safe() as we would end up
@@ -223,21 +226,22 @@ void lirc_bpf_run(struct rc_dev *rcdev, u32 sample)
/*
* This should be called once the rc thread has been stopped, so there can be
* no concurrent bpf execution.
+ *
+ * Should be called with the ir_raw_handler_lock held.
*/
void lirc_bpf_free(struct rc_dev *rcdev)
{
struct bpf_prog_array_item *item;
+ struct bpf_prog_array *array;
- if (!rcdev->raw->progs)
+ array = lirc_rcu_dereference(rcdev->raw->progs);
+ if (!array)
return;
- item = rcu_dereference(rcdev->raw->progs)->items;
- while (item->prog) {
+ for (item = array->items; item->prog; item++)
bpf_prog_put(item->prog);
- item++;
- }
- bpf_prog_array_free(rcdev->raw->progs);
+ bpf_prog_array_free(array);
}
int lirc_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
@@ -290,7 +294,7 @@ int lirc_prog_detach(const union bpf_attr *attr)
int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
{
__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
- struct bpf_prog_array __rcu *progs;
+ struct bpf_prog_array *progs;
struct rc_dev *rcdev;
u32 cnt, flags = 0;
int ret;
@@ -311,7 +315,7 @@ int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
if (ret)
goto put;
- progs = rcdev->raw->progs;
+ progs = lirc_rcu_dereference(rcdev->raw->progs);
cnt = progs ? bpf_prog_array_length(progs) : 0;
if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt))) {
--
2.22.0.rc1.257.g3120a18244-goog
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH bpf-next v3 3/4] bpf: cgroup: properly use bpf_prog_array api
2019-05-28 18:29 [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 2/4] bpf: media: properly use bpf_prog_array api Stanislav Fomichev
@ 2019-05-28 18:29 ` Stanislav Fomichev
2019-05-28 18:57 ` Roman Gushchin
2019-05-28 19:43 ` Roman Gushchin
2019-05-28 18:29 ` [PATCH bpf-next v3 4/4] bpf: tracing: " Stanislav Fomichev
2019-05-28 18:56 ` [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Roman Gushchin
3 siblings, 2 replies; 9+ messages in thread
From: Stanislav Fomichev @ 2019-05-28 18:29 UTC (permalink / raw)
To: netdev, bpf; +Cc: davem, ast, daniel, Stanislav Fomichev, Roman Gushchin
Now that we don't have __rcu markers on the bpf_prog_array helpers,
let's use proper rcu_dereference_protected to obtain array pointer
under mutex.
We also don't need __rcu annotations on cgroup_bpf.inactive since
it's not read/updated concurrently.
v3:
* amend cgroup_rcu_dereference to include percpu_ref_is_dying;
cgroup_bpf is now reference counted and we don't hold cgroup_mutex
anymore in cgroup_bpf_release
v2:
* replace xchg with rcu_swap_protected
Cc: Roman Gushchin <guro@fb.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
include/linux/bpf-cgroup.h | 2 +-
kernel/bpf/cgroup.c | 32 +++++++++++++++++++++-----------
2 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index 9f100fc422c3..b631ee75762d 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -72,7 +72,7 @@ struct cgroup_bpf {
u32 flags[MAX_BPF_ATTACH_TYPE];
/* temp storage for effective prog array used by prog_attach/detach */
- struct bpf_prog_array __rcu *inactive;
+ struct bpf_prog_array *inactive;
/* reference counter used to detach bpf programs after cgroup removal */
struct percpu_ref refcnt;
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index d995edbe816d..118b70175dd9 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -22,6 +22,13 @@
DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
EXPORT_SYMBOL(cgroup_bpf_enabled_key);
+#define cgroup_rcu_dereference(cgrp, p) \
+ rcu_dereference_protected(p, lockdep_is_held(&cgroup_mutex) || \
+ percpu_ref_is_dying(&cgrp->bpf.refcnt))
+
+#define cgroup_rcu_swap(rcu_ptr, ptr) \
+ rcu_swap_protected(rcu_ptr, ptr, lockdep_is_held(&cgroup_mutex))
+
void cgroup_bpf_offline(struct cgroup *cgrp)
{
cgroup_get(cgrp);
@@ -38,6 +45,7 @@ static void cgroup_bpf_release(struct work_struct *work)
struct cgroup *cgrp = container_of(work, struct cgroup,
bpf.release_work);
enum bpf_cgroup_storage_type stype;
+ struct bpf_prog_array *old_array;
unsigned int type;
for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
@@ -54,7 +62,9 @@ static void cgroup_bpf_release(struct work_struct *work)
kfree(pl);
static_branch_dec(&cgroup_bpf_enabled_key);
}
- bpf_prog_array_free(cgrp->bpf.effective[type]);
+ old_array = cgroup_rcu_dereference(cgrp,
+ cgrp->bpf.effective[type]);
+ bpf_prog_array_free(old_array);
}
percpu_ref_exit(&cgrp->bpf.refcnt);
@@ -126,7 +136,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp,
*/
static int compute_effective_progs(struct cgroup *cgrp,
enum bpf_attach_type type,
- struct bpf_prog_array __rcu **array)
+ struct bpf_prog_array **array)
{
enum bpf_cgroup_storage_type stype;
struct bpf_prog_array *progs;
@@ -164,17 +174,15 @@ static int compute_effective_progs(struct cgroup *cgrp,
}
} while ((p = cgroup_parent(p)));
- rcu_assign_pointer(*array, progs);
+ *array = progs;
return 0;
}
static void activate_effective_progs(struct cgroup *cgrp,
enum bpf_attach_type type,
- struct bpf_prog_array __rcu *array)
+ struct bpf_prog_array *old_array)
{
- struct bpf_prog_array __rcu *old_array;
-
- old_array = xchg(&cgrp->bpf.effective[type], array);
+ cgroup_rcu_swap(cgrp->bpf.effective[type], old_array);
/* free prog array after grace period, since __cgroup_bpf_run_*()
* might be still walking the array
*/
@@ -191,7 +199,7 @@ int cgroup_bpf_inherit(struct cgroup *cgrp)
* that array below is variable length
*/
#define NR ARRAY_SIZE(cgrp->bpf.effective)
- struct bpf_prog_array __rcu *arrays[NR] = {};
+ struct bpf_prog_array *arrays[NR] = {};
int ret, i;
ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
@@ -477,10 +485,13 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
enum bpf_attach_type type = attr->query.attach_type;
struct list_head *progs = &cgrp->bpf.progs[type];
u32 flags = cgrp->bpf.flags[type];
+ struct bpf_prog_array *effective;
int cnt, ret = 0, i;
+ effective = cgroup_rcu_dereference(cgrp, cgrp->bpf.effective[type]);
+
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
- cnt = bpf_prog_array_length(cgrp->bpf.effective[type]);
+ cnt = bpf_prog_array_length(effective);
else
cnt = prog_list_length(progs);
@@ -497,8 +508,7 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
}
if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
- return bpf_prog_array_copy_to_user(cgrp->bpf.effective[type],
- prog_ids, cnt);
+ return bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
} else {
struct bpf_prog_list *pl;
u32 id;
--
2.22.0.rc1.257.g3120a18244-goog
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH bpf-next v3 4/4] bpf: tracing: properly use bpf_prog_array api
2019-05-28 18:29 [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 2/4] bpf: media: properly use bpf_prog_array api Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 3/4] bpf: cgroup: " Stanislav Fomichev
@ 2019-05-28 18:29 ` Stanislav Fomichev
2019-05-28 18:56 ` [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Roman Gushchin
3 siblings, 0 replies; 9+ messages in thread
From: Stanislav Fomichev @ 2019-05-28 18:29 UTC (permalink / raw)
To: netdev, bpf
Cc: davem, ast, daniel, Stanislav Fomichev, Steven Rostedt, Ingo Molnar
Now that we don't have __rcu markers on the bpf_prog_array helpers,
let's use proper rcu_dereference_protected to obtain array pointer
under mutex.
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ingo Molnar <mingo@redhat.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
---
kernel/trace/bpf_trace.c | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index fe73926a07cd..3994a231eb92 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -19,6 +19,9 @@
#include "trace_probe.h"
#include "trace.h"
+#define bpf_event_rcu_dereference(p) \
+ rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex))
+
#ifdef CONFIG_MODULES
struct bpf_trace_module {
struct module *module;
@@ -1099,7 +1102,7 @@ static DEFINE_MUTEX(bpf_event_mutex);
int perf_event_attach_bpf_prog(struct perf_event *event,
struct bpf_prog *prog)
{
- struct bpf_prog_array __rcu *old_array;
+ struct bpf_prog_array *old_array;
struct bpf_prog_array *new_array;
int ret = -EEXIST;
@@ -1117,7 +1120,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event,
if (event->prog)
goto unlock;
- old_array = event->tp_event->prog_array;
+ old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
if (old_array &&
bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) {
ret = -E2BIG;
@@ -1140,7 +1143,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event,
void perf_event_detach_bpf_prog(struct perf_event *event)
{
- struct bpf_prog_array __rcu *old_array;
+ struct bpf_prog_array *old_array;
struct bpf_prog_array *new_array;
int ret;
@@ -1149,7 +1152,7 @@ void perf_event_detach_bpf_prog(struct perf_event *event)
if (!event->prog)
goto unlock;
- old_array = event->tp_event->prog_array;
+ old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array);
if (ret == -ENOENT)
goto unlock;
@@ -1171,6 +1174,7 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info)
{
struct perf_event_query_bpf __user *uquery = info;
struct perf_event_query_bpf query = {};
+ struct bpf_prog_array *progs;
u32 *ids, prog_cnt, ids_len;
int ret;
@@ -1195,10 +1199,8 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info)
*/
mutex_lock(&bpf_event_mutex);
- ret = bpf_prog_array_copy_info(event->tp_event->prog_array,
- ids,
- ids_len,
- &prog_cnt);
+ progs = bpf_event_rcu_dereference(event->tp_event->prog_array);
+ ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt);
mutex_unlock(&bpf_event_mutex);
if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
--
2.22.0.rc1.257.g3120a18244-goog
^ permalink raw reply related [flat|nested] 9+ messages in thread
* Re: [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array
2019-05-28 18:29 [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Stanislav Fomichev
` (2 preceding siblings ...)
2019-05-28 18:29 ` [PATCH bpf-next v3 4/4] bpf: tracing: " Stanislav Fomichev
@ 2019-05-28 18:56 ` Roman Gushchin
3 siblings, 0 replies; 9+ messages in thread
From: Roman Gushchin @ 2019-05-28 18:56 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, bpf, davem, ast, daniel
On Tue, May 28, 2019 at 11:29:43AM -0700, Stanislav Fomichev wrote:
> Drop __rcu annotations and rcu read sections from bpf_prog_array
> helper functions. They are not needed since all existing callers
> call those helpers from the rcu update side while holding a mutex.
> This guarantees that use-after-free could not happen.
>
> In the next patches I'll fix the callers with missing
> rcu_dereference_protected to make sparse/lockdep happy, the proper
> way to use these helpers is:
>
> struct bpf_prog_array __rcu *progs = ...;
> struct bpf_prog_array *p;
>
> mutex_lock(&mtx);
> p = rcu_dereference_protected(progs, lockdep_is_held(&mtx));
> bpf_prog_array_length(p);
> bpf_prog_array_copy_to_user(p, ...);
> bpf_prog_array_delete_safe(p, ...);
> bpf_prog_array_copy_info(p, ...);
> bpf_prog_array_copy(p, ...);
> bpf_prog_array_free(p);
> mutex_unlock(&mtx);
>
> No functional changes! rcu_dereference_protected with lockdep_is_held
> should catch any cases where we update prog array without a mutex
> (I've looked at existing call sites and I think we hold a mutex
> everywhere).
>
> Motivation is to fix sparse warnings:
> kernel/bpf/core.c:1803:9: warning: incorrect type in argument 1 (different address spaces)
> kernel/bpf/core.c:1803:9: expected struct callback_head *head
> kernel/bpf/core.c:1803:9: got struct callback_head [noderef] <asn:4> *
> kernel/bpf/core.c:1877:44: warning: incorrect type in initializer (different address spaces)
> kernel/bpf/core.c:1877:44: expected struct bpf_prog_array_item *item
> kernel/bpf/core.c:1877:44: got struct bpf_prog_array_item [noderef] <asn:4> *
> kernel/bpf/core.c:1901:26: warning: incorrect type in assignment (different address spaces)
> kernel/bpf/core.c:1901:26: expected struct bpf_prog_array_item *existing
> kernel/bpf/core.c:1901:26: got struct bpf_prog_array_item [noderef] <asn:4> *
> kernel/bpf/core.c:1935:26: warning: incorrect type in assignment (different address spaces)
> kernel/bpf/core.c:1935:26: expected struct bpf_prog_array_item *[assigned] existing
> kernel/bpf/core.c:1935:26: got struct bpf_prog_array_item [noderef] <asn:4> *
>
> v2:
> * remove comment about potential race; that can't happen
> because all callers are in rcu-update section
>
> Cc: Roman Gushchin <guro@fb.com>
> Signed-off-by: Stanislav Fomichev <sdf@google.com>
Acked-by: Roman Gushchin <guro@fb.com>
Thanks!
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH bpf-next v3 3/4] bpf: cgroup: properly use bpf_prog_array api
2019-05-28 18:29 ` [PATCH bpf-next v3 3/4] bpf: cgroup: " Stanislav Fomichev
@ 2019-05-28 18:57 ` Roman Gushchin
2019-05-28 19:43 ` Roman Gushchin
1 sibling, 0 replies; 9+ messages in thread
From: Roman Gushchin @ 2019-05-28 18:57 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, bpf, davem, ast, daniel
On Tue, May 28, 2019 at 11:29:45AM -0700, Stanislav Fomichev wrote:
> Now that we don't have __rcu markers on the bpf_prog_array helpers,
> let's use proper rcu_dereference_protected to obtain array pointer
> under mutex.
>
> We also don't need __rcu annotations on cgroup_bpf.inactive since
> it's not read/updated concurrently.
>
> v3:
> * amend cgroup_rcu_dereference to include percpu_ref_is_dying;
> cgroup_bpf is now reference counted and we don't hold cgroup_mutex
> anymore in cgroup_bpf_release
>
> v2:
> * replace xchg with rcu_swap_protected
>
> Cc: Roman Gushchin <guro@fb.com>
> Signed-off-by: Stanislav Fomichev <sdf@google.com>
> ---
> include/linux/bpf-cgroup.h | 2 +-
> kernel/bpf/cgroup.c | 32 +++++++++++++++++++++-----------
> 2 files changed, 22 insertions(+), 12 deletions(-)
>
> diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
> index 9f100fc422c3..b631ee75762d 100644
> --- a/include/linux/bpf-cgroup.h
> +++ b/include/linux/bpf-cgroup.h
> @@ -72,7 +72,7 @@ struct cgroup_bpf {
> u32 flags[MAX_BPF_ATTACH_TYPE];
>
> /* temp storage for effective prog array used by prog_attach/detach */
> - struct bpf_prog_array __rcu *inactive;
> + struct bpf_prog_array *inactive;
>
> /* reference counter used to detach bpf programs after cgroup removal */
> struct percpu_ref refcnt;
> diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
> index d995edbe816d..118b70175dd9 100644
> --- a/kernel/bpf/cgroup.c
> +++ b/kernel/bpf/cgroup.c
> @@ -22,6 +22,13 @@
> DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
> EXPORT_SYMBOL(cgroup_bpf_enabled_key);
>
> +#define cgroup_rcu_dereference(cgrp, p) \
> + rcu_dereference_protected(p, lockdep_is_held(&cgroup_mutex) || \
> + percpu_ref_is_dying(&cgrp->bpf.refcnt))
> +
> +#define cgroup_rcu_swap(rcu_ptr, ptr) \
> + rcu_swap_protected(rcu_ptr, ptr, lockdep_is_held(&cgroup_mutex))
> +
> void cgroup_bpf_offline(struct cgroup *cgrp)
> {
> cgroup_get(cgrp);
> @@ -38,6 +45,7 @@ static void cgroup_bpf_release(struct work_struct *work)
> struct cgroup *cgrp = container_of(work, struct cgroup,
> bpf.release_work);
> enum bpf_cgroup_storage_type stype;
> + struct bpf_prog_array *old_array;
> unsigned int type;
>
> for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
> @@ -54,7 +62,9 @@ static void cgroup_bpf_release(struct work_struct *work)
> kfree(pl);
> static_branch_dec(&cgroup_bpf_enabled_key);
> }
> - bpf_prog_array_free(cgrp->bpf.effective[type]);
> + old_array = cgroup_rcu_dereference(cgrp,
> + cgrp->bpf.effective[type]);
> + bpf_prog_array_free(old_array);
> }
>
> percpu_ref_exit(&cgrp->bpf.refcnt);
> @@ -126,7 +136,7 @@ static bool hierarchy_allows_attach(struct cgroup *cgrp,
> */
> static int compute_effective_progs(struct cgroup *cgrp,
> enum bpf_attach_type type,
> - struct bpf_prog_array __rcu **array)
> + struct bpf_prog_array **array)
> {
> enum bpf_cgroup_storage_type stype;
> struct bpf_prog_array *progs;
> @@ -164,17 +174,15 @@ static int compute_effective_progs(struct cgroup *cgrp,
> }
> } while ((p = cgroup_parent(p)));
>
> - rcu_assign_pointer(*array, progs);
> + *array = progs;
> return 0;
> }
>
> static void activate_effective_progs(struct cgroup *cgrp,
> enum bpf_attach_type type,
> - struct bpf_prog_array __rcu *array)
> + struct bpf_prog_array *old_array)
> {
> - struct bpf_prog_array __rcu *old_array;
> -
> - old_array = xchg(&cgrp->bpf.effective[type], array);
> + cgroup_rcu_swap(cgrp->bpf.effective[type], old_array);
> /* free prog array after grace period, since __cgroup_bpf_run_*()
> * might be still walking the array
> */
> @@ -191,7 +199,7 @@ int cgroup_bpf_inherit(struct cgroup *cgrp)
> * that array below is variable length
> */
> #define NR ARRAY_SIZE(cgrp->bpf.effective)
> - struct bpf_prog_array __rcu *arrays[NR] = {};
> + struct bpf_prog_array *arrays[NR] = {};
> int ret, i;
>
> ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
> @@ -477,10 +485,13 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
> enum bpf_attach_type type = attr->query.attach_type;
> struct list_head *progs = &cgrp->bpf.progs[type];
> u32 flags = cgrp->bpf.flags[type];
> + struct bpf_prog_array *effective;
> int cnt, ret = 0, i;
>
> + effective = cgroup_rcu_dereference(cgrp, cgrp->bpf.effective[type]);
> +
> if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE)
> - cnt = bpf_prog_array_length(cgrp->bpf.effective[type]);
> + cnt = bpf_prog_array_length(effective);
> else
> cnt = prog_list_length(progs);
>
> @@ -497,8 +508,7 @@ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
> }
>
> if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
> - return bpf_prog_array_copy_to_user(cgrp->bpf.effective[type],
> - prog_ids, cnt);
> + return bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
> } else {
> struct bpf_prog_list *pl;
> u32 id;
> --
> 2.22.0.rc1.257.g3120a18244-goog
>
Acked-by: Roman Gushchin <guro@fb.com>
Thanks!
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH bpf-next v3 3/4] bpf: cgroup: properly use bpf_prog_array api
2019-05-28 18:29 ` [PATCH bpf-next v3 3/4] bpf: cgroup: " Stanislav Fomichev
2019-05-28 18:57 ` Roman Gushchin
@ 2019-05-28 19:43 ` Roman Gushchin
2019-05-28 20:16 ` Stanislav Fomichev
1 sibling, 1 reply; 9+ messages in thread
From: Roman Gushchin @ 2019-05-28 19:43 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: netdev, bpf, davem, ast, daniel
On Tue, May 28, 2019 at 11:29:45AM -0700, Stanislav Fomichev wrote:
> Now that we don't have __rcu markers on the bpf_prog_array helpers,
> let's use proper rcu_dereference_protected to obtain array pointer
> under mutex.
>
> We also don't need __rcu annotations on cgroup_bpf.inactive since
> it's not read/updated concurrently.
>
> v3:
> * amend cgroup_rcu_dereference to include percpu_ref_is_dying;
> cgroup_bpf is now reference counted and we don't hold cgroup_mutex
> anymore in cgroup_bpf_release
>
> v2:
> * replace xchg with rcu_swap_protected
>
> Cc: Roman Gushchin <guro@fb.com>
> Signed-off-by: Stanislav Fomichev <sdf@google.com>
> ---
> include/linux/bpf-cgroup.h | 2 +-
> kernel/bpf/cgroup.c | 32 +++++++++++++++++++++-----------
> 2 files changed, 22 insertions(+), 12 deletions(-)
>
> diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
> index 9f100fc422c3..b631ee75762d 100644
> --- a/include/linux/bpf-cgroup.h
> +++ b/include/linux/bpf-cgroup.h
> @@ -72,7 +72,7 @@ struct cgroup_bpf {
> u32 flags[MAX_BPF_ATTACH_TYPE];
>
> /* temp storage for effective prog array used by prog_attach/detach */
> - struct bpf_prog_array __rcu *inactive;
> + struct bpf_prog_array *inactive;
>
> /* reference counter used to detach bpf programs after cgroup removal */
> struct percpu_ref refcnt;
> diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
> index d995edbe816d..118b70175dd9 100644
> --- a/kernel/bpf/cgroup.c
> +++ b/kernel/bpf/cgroup.c
> @@ -22,6 +22,13 @@
> DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
> EXPORT_SYMBOL(cgroup_bpf_enabled_key);
>
> +#define cgroup_rcu_dereference(cgrp, p) \
> + rcu_dereference_protected(p, lockdep_is_held(&cgroup_mutex) || \
> + percpu_ref_is_dying(&cgrp->bpf.refcnt))
Some comments why percpu_ref_is_dying(&cgrp->bpf.refcnt) is enough here will
be appreciated.
Thanks!
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH bpf-next v3 3/4] bpf: cgroup: properly use bpf_prog_array api
2019-05-28 19:43 ` Roman Gushchin
@ 2019-05-28 20:16 ` Stanislav Fomichev
2019-05-28 20:53 ` Roman Gushchin
0 siblings, 1 reply; 9+ messages in thread
From: Stanislav Fomichev @ 2019-05-28 20:16 UTC (permalink / raw)
To: Roman Gushchin; +Cc: Stanislav Fomichev, netdev, bpf, davem, ast, daniel
On 05/28, Roman Gushchin wrote:
> On Tue, May 28, 2019 at 11:29:45AM -0700, Stanislav Fomichev wrote:
> > Now that we don't have __rcu markers on the bpf_prog_array helpers,
> > let's use proper rcu_dereference_protected to obtain array pointer
> > under mutex.
> >
> > We also don't need __rcu annotations on cgroup_bpf.inactive since
> > it's not read/updated concurrently.
> >
> > v3:
> > * amend cgroup_rcu_dereference to include percpu_ref_is_dying;
> > cgroup_bpf is now reference counted and we don't hold cgroup_mutex
> > anymore in cgroup_bpf_release
> >
> > v2:
> > * replace xchg with rcu_swap_protected
> >
> > Cc: Roman Gushchin <guro@fb.com>
> > Signed-off-by: Stanislav Fomichev <sdf@google.com>
> > ---
> > include/linux/bpf-cgroup.h | 2 +-
> > kernel/bpf/cgroup.c | 32 +++++++++++++++++++++-----------
> > 2 files changed, 22 insertions(+), 12 deletions(-)
> >
> > diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
> > index 9f100fc422c3..b631ee75762d 100644
> > --- a/include/linux/bpf-cgroup.h
> > +++ b/include/linux/bpf-cgroup.h
> > @@ -72,7 +72,7 @@ struct cgroup_bpf {
> > u32 flags[MAX_BPF_ATTACH_TYPE];
> >
> > /* temp storage for effective prog array used by prog_attach/detach */
> > - struct bpf_prog_array __rcu *inactive;
> > + struct bpf_prog_array *inactive;
> >
> > /* reference counter used to detach bpf programs after cgroup removal */
> > struct percpu_ref refcnt;
> > diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
> > index d995edbe816d..118b70175dd9 100644
> > --- a/kernel/bpf/cgroup.c
> > +++ b/kernel/bpf/cgroup.c
> > @@ -22,6 +22,13 @@
> > DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
> > EXPORT_SYMBOL(cgroup_bpf_enabled_key);
> >
> > +#define cgroup_rcu_dereference(cgrp, p) \
> > + rcu_dereference_protected(p, lockdep_is_held(&cgroup_mutex) || \
> > + percpu_ref_is_dying(&cgrp->bpf.refcnt))
>
> Some comments why percpu_ref_is_dying(&cgrp->bpf.refcnt) is enough here will
> be appreciated.
I was actually debating whether to just use raw
rcu_dereference_protected(p, lockdep_is_held()) in __cgroup_bpf_query and
rcu_dereference_protected(p, percpu_ref_is_dying()) in cgroup_bpf_release
instead of having a cgroup_rcu_dereference which covers both cases.
Maybe that should make it more clear (and doesn't require any comment)?
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH bpf-next v3 3/4] bpf: cgroup: properly use bpf_prog_array api
2019-05-28 20:16 ` Stanislav Fomichev
@ 2019-05-28 20:53 ` Roman Gushchin
0 siblings, 0 replies; 9+ messages in thread
From: Roman Gushchin @ 2019-05-28 20:53 UTC (permalink / raw)
To: Stanislav Fomichev; +Cc: Stanislav Fomichev, netdev, bpf, davem, ast, daniel
On Tue, May 28, 2019 at 01:16:46PM -0700, Stanislav Fomichev wrote:
> On 05/28, Roman Gushchin wrote:
> > On Tue, May 28, 2019 at 11:29:45AM -0700, Stanislav Fomichev wrote:
> > > Now that we don't have __rcu markers on the bpf_prog_array helpers,
> > > let's use proper rcu_dereference_protected to obtain array pointer
> > > under mutex.
> > >
> > > We also don't need __rcu annotations on cgroup_bpf.inactive since
> > > it's not read/updated concurrently.
> > >
> > > v3:
> > > * amend cgroup_rcu_dereference to include percpu_ref_is_dying;
> > > cgroup_bpf is now reference counted and we don't hold cgroup_mutex
> > > anymore in cgroup_bpf_release
> > >
> > > v2:
> > > * replace xchg with rcu_swap_protected
> > >
> > > Cc: Roman Gushchin <guro@fb.com>
> > > Signed-off-by: Stanislav Fomichev <sdf@google.com>
> > > ---
> > > include/linux/bpf-cgroup.h | 2 +-
> > > kernel/bpf/cgroup.c | 32 +++++++++++++++++++++-----------
> > > 2 files changed, 22 insertions(+), 12 deletions(-)
> > >
> > > diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
> > > index 9f100fc422c3..b631ee75762d 100644
> > > --- a/include/linux/bpf-cgroup.h
> > > +++ b/include/linux/bpf-cgroup.h
> > > @@ -72,7 +72,7 @@ struct cgroup_bpf {
> > > u32 flags[MAX_BPF_ATTACH_TYPE];
> > >
> > > /* temp storage for effective prog array used by prog_attach/detach */
> > > - struct bpf_prog_array __rcu *inactive;
> > > + struct bpf_prog_array *inactive;
> > >
> > > /* reference counter used to detach bpf programs after cgroup removal */
> > > struct percpu_ref refcnt;
> > > diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
> > > index d995edbe816d..118b70175dd9 100644
> > > --- a/kernel/bpf/cgroup.c
> > > +++ b/kernel/bpf/cgroup.c
> > > @@ -22,6 +22,13 @@
> > > DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
> > > EXPORT_SYMBOL(cgroup_bpf_enabled_key);
> > >
> > > +#define cgroup_rcu_dereference(cgrp, p) \
> > > + rcu_dereference_protected(p, lockdep_is_held(&cgroup_mutex) || \
> > > + percpu_ref_is_dying(&cgrp->bpf.refcnt))
> >
> > Some comments why percpu_ref_is_dying(&cgrp->bpf.refcnt) is enough here will
> > be appreciated.
> I was actually debating whether to just use raw
> rcu_dereference_protected(p, lockdep_is_held()) in __cgroup_bpf_query and
> rcu_dereference_protected(p, percpu_ref_is_dying()) in cgroup_bpf_release
> instead of having a cgroup_rcu_dereference which covers both cases.
>
> Maybe that should make it more clear (and doesn't require any comment)?
Yeah, this makes total sense to me.
^ permalink raw reply [flat|nested] 9+ messages in thread
end of thread, other threads:[~2019-05-28 20:54 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-05-28 18:29 [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 2/4] bpf: media: properly use bpf_prog_array api Stanislav Fomichev
2019-05-28 18:29 ` [PATCH bpf-next v3 3/4] bpf: cgroup: " Stanislav Fomichev
2019-05-28 18:57 ` Roman Gushchin
2019-05-28 19:43 ` Roman Gushchin
2019-05-28 20:16 ` Stanislav Fomichev
2019-05-28 20:53 ` Roman Gushchin
2019-05-28 18:29 ` [PATCH bpf-next v3 4/4] bpf: tracing: " Stanislav Fomichev
2019-05-28 18:56 ` [PATCH bpf-next v3 1/4] bpf: remove __rcu annotations from bpf_prog_array Roman Gushchin
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.