From: Namhyung Kim <namhyung@kernel.org>
To: Peter Zijlstra <peterz@infradead.org>, Ingo Molnar <mingo@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>,
Alexander Shishkin <alexander.shishkin@linux.intel.com>,
Arnaldo Carvalho de Melo <acme@kernel.org>,
Jiri Olsa <jolsa@kernel.org>, LKML <linux-kernel@vger.kernel.org>,
Song Liu <song@kernel.org>,
bpf@vger.kernel.org
Subject: [PATCH 5/8] perf/core: Set data->sample_flags in perf_prepare_sample()
Date: Tue, 17 Jan 2023 22:05:56 -0800 [thread overview]
Message-ID: <20230118060559.615653-6-namhyung@kernel.org> (raw)
In-Reply-To: <20230118060559.615653-1-namhyung@kernel.org>
The perf_prepare_sample() sets the perf_sample_data according to the
attr->sample_type before copying it to the ring buffer. But BPF also
wants to access the sample data so it needs to prepare the sample even
before the regular path.
That means the perf_prepare_sample() can be called more than once. Set
the data->sample_flags consistently so that it can indicate which fields
are set already and skip them if sets.
Also update the filtered_sample_type to have the dependent flags to
reduce the number of branches.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Jiri Olsa <jolsa@kernel.org>
Tested-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
kernel/events/core.c | 85 +++++++++++++++++++++++++++++++++-----------
1 file changed, 65 insertions(+), 20 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 0218b6ffaf36..1384137a90f7 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7053,12 +7053,21 @@ static void perf_aux_sample_output(struct perf_event *event,
ring_buffer_put(rb);
}
+/*
+ * A set of common sample data types saved even for non-sample records
+ * when event->attr.sample_id_all is set.
+ */
+#define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID | PERF_SAMPLE_TIME | \
+ PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \
+ PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER)
+
static void __perf_event_header__init_id(struct perf_event_header *header,
struct perf_sample_data *data,
struct perf_event *event,
u64 sample_type)
{
data->type = event->attr.sample_type;
+ data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL;
header->size += event->id_header_size;
if (sample_type & PERF_SAMPLE_TID) {
@@ -7561,6 +7570,11 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
return callchain ?: &__empty_callchain;
}
+static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
+{
+ return d * !!(flags & s);
+}
+
void perf_prepare_sample(struct perf_event_header *header,
struct perf_sample_data *data,
struct perf_event *event,
@@ -7576,14 +7590,24 @@ void perf_prepare_sample(struct perf_event_header *header,
header->misc |= perf_misc_flags(regs);
/*
- * Clear the sample flags that have already been done by the
- * PMU driver.
+ * Add the sample flags that are dependent to others. And clear the
+ * sample flags that have already been done by the PMU driver.
*/
- filtered_sample_type = sample_type & ~data->sample_flags;
+ filtered_sample_type = sample_type;
+ filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE,
+ PERF_SAMPLE_IP);
+ filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE |
+ PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR);
+ filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER,
+ PERF_SAMPLE_REGS_USER);
+ filtered_sample_type &= ~data->sample_flags;
+
__perf_event_header__init_id(header, data, event, filtered_sample_type);
- if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE))
+ if (filtered_sample_type & PERF_SAMPLE_IP) {
data->ip = perf_instruction_pointer(regs);
+ data->sample_flags |= PERF_SAMPLE_IP;
+ }
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN)
perf_sample_save_callchain(data, event, regs);
@@ -7600,10 +7624,15 @@ void perf_prepare_sample(struct perf_event_header *header,
data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;
}
- if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
+ if (filtered_sample_type & PERF_SAMPLE_REGS_USER)
perf_sample_regs_user(&data->regs_user, regs);
- if (sample_type & PERF_SAMPLE_REGS_USER) {
+ /*
+ * It cannot use the filtered_sample_type here as REGS_USER can be set
+ * by STACK_USER (using __cond_set() above) and we don't want to update
+ * the dyn_size if it's not requested by users.
+ */
+ if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) {
/* regs dump ABI info */
int size = sizeof(u64);
@@ -7613,9 +7642,10 @@ void perf_prepare_sample(struct perf_event_header *header,
}
data->dyn_size += size;
+ data->sample_flags |= PERF_SAMPLE_REGS_USER;
}
- if (sample_type & PERF_SAMPLE_STACK_USER) {
+ if (filtered_sample_type & PERF_SAMPLE_STACK_USER) {
/*
* Either we need PERF_SAMPLE_STACK_USER bit to be always
* processed as the last one or have additional check added
@@ -7638,23 +7668,30 @@ void perf_prepare_sample(struct perf_event_header *header,
data->stack_user_size = stack_size;
data->dyn_size += size;
+ data->sample_flags |= PERF_SAMPLE_STACK_USER;
}
- if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE)
+ if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
data->weight.full = 0;
+ data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
+ }
- if (filtered_sample_type & PERF_SAMPLE_DATA_SRC)
+ if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) {
data->data_src.val = PERF_MEM_NA;
+ data->sample_flags |= PERF_SAMPLE_DATA_SRC;
+ }
- if (filtered_sample_type & PERF_SAMPLE_TRANSACTION)
+ if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) {
data->txn = 0;
+ data->sample_flags |= PERF_SAMPLE_TRANSACTION;
+ }
- if (sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_DATA_PAGE_SIZE)) {
- if (filtered_sample_type & PERF_SAMPLE_ADDR)
- data->addr = 0;
+ if (filtered_sample_type & PERF_SAMPLE_ADDR) {
+ data->addr = 0;
+ data->sample_flags |= PERF_SAMPLE_ADDR;
}
- if (sample_type & PERF_SAMPLE_REGS_INTR) {
+ if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) {
/* regs dump ABI info */
int size = sizeof(u64);
@@ -7667,19 +7704,22 @@ void perf_prepare_sample(struct perf_event_header *header,
}
data->dyn_size += size;
+ data->sample_flags |= PERF_SAMPLE_REGS_INTR;
}
- if (sample_type & PERF_SAMPLE_PHYS_ADDR &&
- filtered_sample_type & PERF_SAMPLE_PHYS_ADDR)
+ if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) {
data->phys_addr = perf_virt_to_phys(data->addr);
+ data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;
+ }
#ifdef CONFIG_CGROUP_PERF
- if (sample_type & PERF_SAMPLE_CGROUP) {
+ if (filtered_sample_type & PERF_SAMPLE_CGROUP) {
struct cgroup *cgrp;
/* protected by RCU */
cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
data->cgroup = cgroup_id(cgrp);
+ data->sample_flags |= PERF_SAMPLE_CGROUP;
}
#endif
@@ -7688,13 +7728,17 @@ void perf_prepare_sample(struct perf_event_header *header,
* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr,
* but the value will not dump to the userspace.
*/
- if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
+ if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) {
data->data_page_size = perf_get_page_size(data->addr);
+ data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;
+ }
- if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
+ if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
data->code_page_size = perf_get_page_size(data->ip);
+ data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;
+ }
- if (sample_type & PERF_SAMPLE_AUX) {
+ if (filtered_sample_type & PERF_SAMPLE_AUX) {
u64 size;
header->size += sizeof(u64); /* size */
@@ -7712,6 +7756,7 @@ void perf_prepare_sample(struct perf_event_header *header,
WARN_ON_ONCE(size + header->size > U16_MAX);
data->dyn_size += size + sizeof(u64); /* size above */
+ data->sample_flags |= PERF_SAMPLE_AUX;
}
header->size += data->dyn_size;
--
2.39.0.314.g84b9a713c41-goog
next prev parent reply other threads:[~2023-01-18 6:23 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-01-18 6:05 [PATCHSET 0/8] perf/core: Prepare sample data for BPF (v3) Namhyung Kim
2023-01-18 6:05 ` [PATCH 1/8] perf/core: Save the dynamic parts of sample data size Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 2/8] perf/core: Add perf_sample_save_callchain() helper Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 3/8] perf/core: Add perf_sample_save_raw_data() helper Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 4/8] perf/core: Add perf_sample_save_brstack() helper Namhyung Kim
2023-01-18 6:05 ` Namhyung Kim
2023-01-18 9:06 ` Athira Rajeev
2023-01-18 9:06 ` Athira Rajeev
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` Namhyung Kim [this message]
2023-01-18 11:37 ` [tip: perf/core] perf/core: Set data->sample_flags in perf_prepare_sample() tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 6/8] perf/core: Do not pass header for sample id init Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] perf/core: Do not pass header for sample ID init tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 7/8] perf/core: Introduce perf_prepare_header() Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 6:05 ` [PATCH 8/8] perf/core: Call perf_prepare_sample() before running BPF Namhyung Kim
2023-01-18 11:37 ` [tip: perf/core] " tip-bot2 for Namhyung Kim
2023-01-18 10:50 ` [PATCHSET 0/8] perf/core: Prepare sample data for BPF (v3) Peter Zijlstra
2023-01-18 19:42 ` Alexei Starovoitov
2023-01-18 22:08 ` Namhyung Kim
-- strict thread matches above, loose matches on Subject: below --
2023-01-12 21:40 [PATCHSET 0/8] perf/core: Prepare sample data for BPF Namhyung Kim
2023-01-12 21:40 ` [PATCH 5/8] perf/core: Set data->sample_flags in perf_prepare_sample() Namhyung Kim
2023-01-13 11:20 ` Peter Zijlstra
2023-01-13 19:31 ` Namhyung Kim
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230118060559.615653-6-namhyung@kernel.org \
--to=namhyung@kernel.org \
--cc=acme@kernel.org \
--cc=alexander.shishkin@linux.intel.com \
--cc=bpf@vger.kernel.org \
--cc=jolsa@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=song@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.