From 422d2a95eff344407ec425f0de55b264841d1757 Mon Sep 17 00:00:00 2001 From: Milian Wolff Date: Wed, 14 Nov 2018 14:10:47 +0100 Subject: [PATCH 1/2] [WIP] perf: make it possible to collect both, iregs and uregs Previously, only one set of registers was stored in the perf data for both, user and interrupt registers. Now, two distinct sets can be sampled. Signed-off-by: Milian Wolff Cc: Arnaldo Carvalho de Melo Cc: Andi Kleen Cc: Jiri Olsa --- arch/x86/events/amd/ibs.c | 2 +- arch/x86/events/core.c | 2 +- arch/x86/events/intel/core.c | 2 +- arch/x86/events/intel/ds.c | 7 +++---- arch/x86/events/intel/knc.c | 2 +- arch/x86/events/intel/p4.c | 2 +- arch/x86/kernel/ptrace.c | 2 +- arch/x86/kvm/pmu.c | 4 ++-- drivers/oprofile/nmi_timer_int.c | 2 +- include/linux/perf_event.h | 18 +++++++++++------ kernel/events/core.c | 34 ++++++++++++++++---------------- kernel/trace/bpf_trace.c | 2 +- kernel/watchdog_hld.c | 2 +- 13 files changed, 43 insertions(+), 38 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index d50bb4dc0650..567db8878511 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -670,7 +670,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) data.raw = &raw; } - throttle = perf_event_overflow(event, &data, ®s); + throttle = perf_event_overflow(event, &data, ®s, iregs); out: if (throttle) perf_ibs_stop(event, 0); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 106911b603bd..acdcafa57ca0 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1493,7 +1493,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) if (!x86_perf_event_set_period(event)) continue; - if (perf_event_overflow(event, &data, regs)) + if (perf_event_overflow(event, &data, regs, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 273c62e81546..2156620b3d9e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2299,7 +2299,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (has_branch_stack(event)) data.br_stack = &cpuc->lbr_stack; - if (perf_event_overflow(event, &data, regs)) + if (perf_event_overflow(event, &data, regs, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index b7b01d762d32..018fc0649033 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -639,7 +639,7 @@ int intel_pmu_drain_bts_buffer(void) * the sample. */ rcu_read_lock(); - perf_prepare_sample(&header, &data, event, ®s); + perf_prepare_sample(&header, &data, event, ®s, ®s); if (perf_output_begin(&handle, event, header.size * (top - base - skip))) @@ -1273,7 +1273,6 @@ static void setup_pebs_sample_data(struct perf_event *event, set_linear_ip(regs, pebs->ip); } - if ((sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR)) && x86_pmu.intel_cap.pebs_format >= 1) data->addr = pebs->dla; @@ -1430,7 +1429,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, while (count > 1) { setup_pebs_sample_data(event, iregs, at, &data, ®s); - perf_event_output(event, &data, ®s); + perf_event_output(event, &data, ®s, iregs); at += x86_pmu.pebs_record_size; at = get_next_pebs_record_by_bit(at, top, bit); count--; @@ -1442,7 +1441,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, * All but the last records are processed. * The last one is left to be able to call the overflow handler. */ - if (perf_event_overflow(event, &data, ®s)) { + if (perf_event_overflow(event, &data, ®s, iregs)) { x86_pmu_stop(event, 0); return; } diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index 618001c208e8..9ea5a13af83f 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -252,7 +252,7 @@ static int knc_pmu_handle_irq(struct pt_regs *regs) perf_sample_data_init(&data, 0, event->hw.last_period); - if (perf_event_overflow(event, &data, regs)) + if (perf_event_overflow(event, &data, regs, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index d32c0eed38ca..704457b5f49a 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -1037,7 +1037,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) continue; - if (perf_event_overflow(event, &data, regs)) + if (perf_event_overflow(event, &data, regs, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index ffae9b9740fd..13b2230e5e9b 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -499,7 +499,7 @@ static int genregs_set(struct task_struct *target, static void ptrace_triggered(struct perf_event *bp, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { int i; struct thread_struct *thread = &(current->thread); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 58ead7db71a3..b556b2d467e1 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -57,7 +57,7 @@ static void kvm_pmi_trigger_fn(struct irq_work *irq_work) static void kvm_perf_overflow(struct perf_event *perf_event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { struct kvm_pmc *pmc = perf_event->overflow_handler_context; struct kvm_pmu *pmu = pmc_to_pmu(pmc); @@ -71,7 +71,7 @@ static void kvm_perf_overflow(struct perf_event *perf_event, static void kvm_perf_overflow_intr(struct perf_event *perf_event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { struct kvm_pmc *pmc = perf_event->overflow_handler_context; struct kvm_pmu *pmu = pmc_to_pmu(pmc); diff --git a/drivers/oprofile/nmi_timer_int.c b/drivers/oprofile/nmi_timer_int.c index f343bd96609a..110dfef21420 100644 --- a/drivers/oprofile/nmi_timer_int.c +++ b/drivers/oprofile/nmi_timer_int.c @@ -28,7 +28,7 @@ static struct perf_event_attr nmi_timer_attr = { static void nmi_timer_callback(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { event->hw.interrupts = 0; /* don't throttle interrupts */ oprofile_add_sample(regs, 0); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 53c500f0ca79..3a989c64c2c7 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -506,7 +506,8 @@ struct perf_sample_data; typedef void (*perf_overflow_handler_t)(struct perf_event *, struct perf_sample_data *, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); /* * Event capabilities. For event_caps and groups caps. @@ -966,21 +967,26 @@ extern void perf_output_sample(struct perf_output_handle *handle, extern void perf_prepare_sample(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); extern int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); extern void perf_event_output_forward(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); extern void perf_event_output_backward(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); extern void perf_event_output(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs); + struct pt_regs *regs, + struct pt_regs *iregs); static inline bool is_default_overflow_handler(struct perf_event *event) diff --git a/kernel/events/core.c b/kernel/events/core.c index 84530ab358c3..1b57602dc6d8 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6369,7 +6369,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) void perf_prepare_sample(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { u64 sample_type = event->attr.sample_type; @@ -6474,7 +6474,7 @@ void perf_prepare_sample(struct perf_event_header *header, /* regs dump ABI info */ int size = sizeof(u64); - perf_sample_regs_intr(&data->regs_intr, regs); + perf_sample_regs_intr(&data->regs_intr, iregs); if (data->regs_intr.regs) { u64 mask = event->attr.sample_regs_intr; @@ -6492,7 +6492,7 @@ void perf_prepare_sample(struct perf_event_header *header, static __always_inline void __perf_event_output(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs, + struct pt_regs *regs, struct pt_regs *iregs, int (*output_begin)(struct perf_output_handle *, struct perf_event *, unsigned int)) @@ -6503,7 +6503,7 @@ __perf_event_output(struct perf_event *event, /* protect the callchain buffers */ rcu_read_lock(); - perf_prepare_sample(&header, data, event, regs); + perf_prepare_sample(&header, data, event, regs, iregs); if (output_begin(&handle, event, header.size)) goto exit; @@ -6519,25 +6519,25 @@ __perf_event_output(struct perf_event *event, void perf_event_output_forward(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { - __perf_event_output(event, data, regs, perf_output_begin_forward); + __perf_event_output(event, data, regs, iregs, perf_output_begin_forward); } void perf_event_output_backward(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { - __perf_event_output(event, data, regs, perf_output_begin_backward); + __perf_event_output(event, data, regs, iregs, perf_output_begin_backward); } void perf_event_output(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { - __perf_event_output(event, data, regs, perf_output_begin); + __perf_event_output(event, data, regs, iregs, perf_output_begin); } /* @@ -7738,7 +7738,7 @@ int perf_event_account_interrupt(struct perf_event *event) static int __perf_event_overflow(struct perf_event *event, int throttle, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { int events = atomic_read(&event->event_limit); int ret = 0; @@ -7765,7 +7765,7 @@ static int __perf_event_overflow(struct perf_event *event, perf_event_disable_inatomic(event); } - READ_ONCE(event->overflow_handler)(event, data, regs); + READ_ONCE(event->overflow_handler)(event, data, regs, iregs); if (*perf_event_fasync(event) && event->pending_kill) { event->pending_wakeup = 1; @@ -7777,9 +7777,9 @@ static int __perf_event_overflow(struct perf_event *event, int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { - return __perf_event_overflow(event, 1, data, regs); + return __perf_event_overflow(event, 1, data, regs, iregs); } /* @@ -7842,7 +7842,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow, for (; overflow; overflow--) { if (__perf_event_overflow(event, throttle, - data, regs)) { + data, regs, regs)) { /* * We inhibit the overflow from happening when * hwc->interrupts == MAX_INTERRUPTS. @@ -8550,7 +8550,7 @@ static void bpf_overflow_handler(struct perf_event *event, if (!ret) return; - event->orig_overflow_handler(event, data, regs); + event->orig_overflow_handler(event, data, regs, regs); } static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd) @@ -9152,7 +9152,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) if (regs && !perf_exclude_event(event, regs)) { if (!(event->attr.exclude_idle && is_idle_task(current))) - if (__perf_event_overflow(event, 1, &data, regs)) + if (__perf_event_overflow(event, 1, &data, regs, regs)) ret = HRTIMER_NORESTART; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 08fcfe440c63..6faf12fd6114 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -392,7 +392,7 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, if (unlikely(event->oncpu != cpu)) return -EOPNOTSUPP; - perf_event_output(event, sd, regs); + perf_event_output(event, sd, regs, regs); return 0; } diff --git a/kernel/watchdog_hld.c b/kernel/watchdog_hld.c index 71381168dede..5f4e18d003bb 100644 --- a/kernel/watchdog_hld.c +++ b/kernel/watchdog_hld.c @@ -109,7 +109,7 @@ static struct perf_event_attr wd_hw_attr = { /* Callback function for perf event subsystem */ static void watchdog_overflow_callback(struct perf_event *event, struct perf_sample_data *data, - struct pt_regs *regs) + struct pt_regs *regs, struct pt_regs *iregs) { /* Ensure the watchdog never gets throttled */ event->hw.interrupts = 0; -- 2.19.1