From: Jiri Olsa <jolsa@kernel.org>
To: Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Andrii Nakryiko <andrii@kernel.org>
Cc: bpf@vger.kernel.org, Martin KaFai Lau <kafai@fb.com>,
Song Liu <songliubraving@fb.com>, Yonghong Song <yhs@fb.com>,
John Fastabend <john.fastabend@gmail.com>,
KP Singh <kpsingh@chromium.org>,
Stanislav Fomichev <sdf@google.com>, Hao Luo <haoluo@google.com>,
Hou Tao <houtao1@huawei.com>, Daniel Xu <dxu@dxuuu.xyz>
Subject: [PATCH bpf-next 01/12] bpf: Move update_prog_stats to syscall object
Date: Mon, 28 Aug 2023 09:55:26 +0200 [thread overview]
Message-ID: <20230828075537.194192-2-jolsa@kernel.org> (raw)
In-Reply-To: <20230828075537.194192-1-jolsa@kernel.org>
Moving update_prog_stats function to syscall object and making it
global together with NO_START_TIME macro and adding 'bpf_' prefix
for both.
It will be used by other program types in following changes.
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
include/linux/bpf.h | 7 +++++++
kernel/bpf/syscall.c | 22 ++++++++++++++++++++++
kernel/bpf/trampoline.c | 37 +++++++------------------------------
3 files changed, 36 insertions(+), 30 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 12596af59c00..05eece17a989 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1982,6 +1982,8 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
return ret;
}
+#define BPF_PROG_NO_START_TIME 1
+
#ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active);
extern struct mutex bpf_stats_enabled_mutex;
@@ -2456,6 +2458,7 @@ static inline bool has_current_bpf_ctx(void)
}
void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
+void notrace bpf_prog_update_prog_stats(struct bpf_prog *prog, u64 start);
void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
enum bpf_dynptr_type type, u32 offset, u32 size);
@@ -2695,6 +2698,10 @@ static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
{
}
+static void bpf_prog_update_prog_stats(struct bpf_prog *prog, u64 start)
+{
+}
+
static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ebeb0695305a..5d39d98f5eb1 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2188,6 +2188,28 @@ void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
+void notrace bpf_prog_update_prog_stats(struct bpf_prog *prog,
+ u64 start)
+{
+ struct bpf_prog_stats *stats;
+
+ if (static_branch_unlikely(&bpf_stats_enabled_key) &&
+ /* static_key could be enabled in __bpf_prog_enter*
+ * and disabled in __bpf_prog_exit*.
+ * And vice versa.
+ * Hence check that 'start' is valid.
+ */
+ start > BPF_PROG_NO_START_TIME) {
+ unsigned long flags;
+
+ stats = this_cpu_ptr(prog->stats);
+ flags = u64_stats_update_begin_irqsave(&stats->syncp);
+ u64_stats_inc(&stats->cnt);
+ u64_stats_add(&stats->nsecs, sched_clock() - start);
+ u64_stats_update_end_irqrestore(&stats->syncp, flags);
+ }
+}
+
static void bpf_prog_get_stats(const struct bpf_prog *prog,
struct bpf_prog_kstats *stats)
{
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 78acf28d4873..a6528e847fae 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -819,15 +819,14 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
mutex_unlock(&trampoline_mutex);
}
-#define NO_START_TIME 1
static __always_inline u64 notrace bpf_prog_start_time(void)
{
- u64 start = NO_START_TIME;
+ u64 start = BPF_PROG_NO_START_TIME;
if (static_branch_unlikely(&bpf_stats_enabled_key)) {
start = sched_clock();
if (unlikely(!start))
- start = NO_START_TIME;
+ start = BPF_PROG_NO_START_TIME;
}
return start;
}
@@ -860,35 +859,13 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
return bpf_prog_start_time();
}
-static void notrace update_prog_stats(struct bpf_prog *prog,
- u64 start)
-{
- struct bpf_prog_stats *stats;
-
- if (static_branch_unlikely(&bpf_stats_enabled_key) &&
- /* static_key could be enabled in __bpf_prog_enter*
- * and disabled in __bpf_prog_exit*.
- * And vice versa.
- * Hence check that 'start' is valid.
- */
- start > NO_START_TIME) {
- unsigned long flags;
-
- stats = this_cpu_ptr(prog->stats);
- flags = u64_stats_update_begin_irqsave(&stats->syncp);
- u64_stats_inc(&stats->cnt);
- u64_stats_add(&stats->nsecs, sched_clock() - start);
- u64_stats_update_end_irqrestore(&stats->syncp, flags);
- }
-}
-
static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
struct bpf_tramp_run_ctx *run_ctx)
__releases(RCU)
{
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
- update_prog_stats(prog, start);
+ bpf_prog_update_prog_stats(prog, start);
this_cpu_dec(*(prog->active));
migrate_enable();
rcu_read_unlock();
@@ -906,7 +883,7 @@ static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
- return NO_START_TIME;
+ return BPF_PROG_NO_START_TIME;
}
static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
@@ -941,7 +918,7 @@ void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
{
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
- update_prog_stats(prog, start);
+ bpf_prog_update_prog_stats(prog, start);
this_cpu_dec(*(prog->active));
migrate_enable();
rcu_read_unlock_trace();
@@ -964,7 +941,7 @@ static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
{
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
- update_prog_stats(prog, start);
+ bpf_prog_update_prog_stats(prog, start);
migrate_enable();
rcu_read_unlock_trace();
}
@@ -987,7 +964,7 @@ static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
{
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
- update_prog_stats(prog, start);
+ bpf_prog_update_prog_stats(prog, start);
migrate_enable();
rcu_read_unlock();
}
--
2.41.0
next prev parent reply other threads:[~2023-08-28 7:55 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-08-28 7:55 [PATCH bpf-next 00/12] bpf: Add missed stats for kprobes Jiri Olsa
2023-08-28 7:55 ` Jiri Olsa [this message]
2023-08-28 17:25 ` [PATCH bpf-next 01/12] bpf: Move update_prog_stats to syscall object Alexei Starovoitov
2023-08-29 8:00 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 02/12] bpf: Move bpf_prog_start_time to linux/filter.h Jiri Olsa
2023-09-04 13:12 ` Hou Tao
2023-08-28 7:55 ` [PATCH bpf-next 03/12] bpf: Count stats for kprobe_multi programs Jiri Olsa
2023-09-04 13:30 ` Hou Tao
2023-09-05 6:15 ` Hou Tao
2023-09-05 7:19 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 04/12] bpf: Add missed value to kprobe_multi link info Jiri Olsa
2023-09-04 13:33 ` Hou Tao
2023-08-28 7:55 ` [PATCH bpf-next 05/12] bpf: Add missed value to kprobe perf " Jiri Olsa
2023-09-05 2:23 ` Hou Tao
2023-09-05 7:19 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 06/12] bpf: Count missed stats in trace_call_bpf Jiri Olsa
2023-08-28 17:32 ` Alexei Starovoitov
2023-08-29 8:04 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 07/12] bpf: Move bpf_prog_run_array down in the header file Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 08/12] bpf: Count run stats in bpf_prog_run_array Jiri Olsa
2023-09-05 2:40 ` Hou Tao
2023-09-05 7:19 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 09/12] bpftool: Display missed count for kprobe_multi link Jiri Olsa
2023-08-29 16:40 ` Quentin Monnet
2023-08-30 10:46 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 10/12] bpftool: Display missed count for kprobe perf link Jiri Olsa
2023-08-29 16:42 ` Quentin Monnet
2023-08-30 16:01 ` Jiri Olsa
2023-08-28 7:55 ` [PATCH bpf-next 11/12] selftests/bpf: Add test missed counts of perf event link kprobe Jiri Olsa
2023-09-05 3:14 ` Hou Tao
2023-08-28 7:55 ` [PATCH bpf-next 12/12] selftests/bpf: Add test recursion stats " Jiri Olsa
2023-09-05 3:23 ` Hou Tao
2023-09-05 3:30 ` [PATCH bpf-next 00/12] bpf: Add missed stats for kprobes Hou Tao
2023-09-05 7:20 ` Jiri Olsa
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230828075537.194192-2-jolsa@kernel.org \
--to=jolsa@kernel.org \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=dxu@dxuuu.xyz \
--cc=haoluo@google.com \
--cc=houtao1@huawei.com \
--cc=john.fastabend@gmail.com \
--cc=kafai@fb.com \
--cc=kpsingh@chromium.org \
--cc=sdf@google.com \
--cc=songliubraving@fb.com \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).