* [RFC][PATCH 01/19] perf: Fix CPU hotplug
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 02/19] perf: deconstify struct pmu Peter Zijlstra
` (18 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-x86-fix-cpuhotplug.patch --]
[-- Type: text/plain, Size: 789 bytes --]
Since we have UP_PREPARE, we should also have UP_CANCELED.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -5743,15 +5743,15 @@ perf_cpu_notify(struct notifier_block *s
{
unsigned int cpu = (long)hcpu;
- switch (action) {
+ switch (action & ~CPU_TASKS_FROZEN) {
case CPU_UP_PREPARE:
- case CPU_UP_PREPARE_FROZEN:
+ case CPU_DOWN_FAILED:
perf_event_init_cpu(cpu);
break;
+ case CPU_UP_CANCELED:
case CPU_DOWN_PREPARE:
- case CPU_DOWN_PREPARE_FROZEN:
perf_event_exit_cpu(cpu);
break;
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 02/19] perf: deconstify struct pmu
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 01/19] perf: Fix CPU hotplug Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 03/19] perf: register pmu implementations Peter Zijlstra
` (17 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-deconst-pmu.patch --]
[-- Type: text/plain, Size: 13126 bytes --]
sed -ie 's/const struct pmu\>/struct pmu/g' `git grep -l "const struct pmu\>"`
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/alpha/kernel/perf_event.c | 4 ++--
arch/arm/kernel/perf_event.c | 2 +-
arch/powerpc/kernel/perf_event.c | 8 ++++----
arch/powerpc/kernel/perf_event_fsl_emb.c | 2 +-
arch/sh/kernel/perf_event.c | 4 ++--
arch/sparc/kernel/perf_event.c | 10 +++++-----
arch/x86/kernel/cpu/perf_event.c | 14 +++++++-------
include/linux/perf_event.h | 10 +++++-----
kernel/perf_event.c | 26 +++++++++++++-------------
9 files changed, 40 insertions(+), 40 deletions(-)
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -491,7 +491,7 @@ __hw_perf_event_init(struct perf_event *
return err;
}
-const struct pmu *
+struct pmu *
hw_perf_event_init(struct perf_event *event)
{
int err = 0;
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event.c
+++ linux-2.6/arch/powerpc/kernel/perf_event.c
@@ -857,7 +857,7 @@ static void power_pmu_unthrottle(struct
* Set the flag to make pmu::enable() not perform the
* schedulability test, it will be performed at commit time
*/
-void power_pmu_start_txn(const struct pmu *pmu)
+void power_pmu_start_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
@@ -870,7 +870,7 @@ void power_pmu_start_txn(const struct pm
* Clear the flag and pmu::enable() will perform the
* schedulability test.
*/
-void power_pmu_cancel_txn(const struct pmu *pmu)
+void power_pmu_cancel_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
@@ -882,7 +882,7 @@ void power_pmu_cancel_txn(const struct p
* Perform the group schedulability test as a whole
* Return 0 if success
*/
-int power_pmu_commit_txn(const struct pmu *pmu)
+int power_pmu_commit_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
long i, n;
@@ -1014,7 +1014,7 @@ static int hw_perf_cache_event(u64 confi
return 0;
}
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
u64 ev;
unsigned long flags;
Index: linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -428,7 +428,7 @@ static int hw_perf_cache_event(u64 confi
return 0;
}
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
u64 ev;
struct perf_event *events[MAX_HWEVENTS];
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sh/kernel/perf_event.c
+++ linux-2.6/arch/sh/kernel/perf_event.c
@@ -257,13 +257,13 @@ static void sh_pmu_read(struct perf_even
sh_perf_event_update(event, &event->hw, event->hw.idx);
}
-static const struct pmu pmu = {
+static struct pmu pmu = {
.enable = sh_pmu_enable,
.disable = sh_pmu_disable,
.read = sh_pmu_read,
};
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
int err = __hw_perf_event_init(event);
if (unlikely(err)) {
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -1099,7 +1099,7 @@ static int __hw_perf_event_init(struct p
* Set the flag to make pmu::enable() not perform the
* schedulability test, it will be performed at commit time
*/
-static void sparc_pmu_start_txn(const struct pmu *pmu)
+static void sparc_pmu_start_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
@@ -1111,7 +1111,7 @@ static void sparc_pmu_start_txn(const st
* Clear the flag and pmu::enable() will perform the
* schedulability test.
*/
-static void sparc_pmu_cancel_txn(const struct pmu *pmu)
+static void sparc_pmu_cancel_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
@@ -1123,7 +1123,7 @@ static void sparc_pmu_cancel_txn(const s
* Perform the group schedulability test as a whole
* Return 0 if success
*/
-static int sparc_pmu_commit_txn(const struct pmu *pmu)
+static int sparc_pmu_commit_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int n;
@@ -1142,7 +1142,7 @@ static int sparc_pmu_commit_txn(const st
return 0;
}
-static const struct pmu pmu = {
+static struct pmu pmu = {
.enable = sparc_pmu_enable,
.disable = sparc_pmu_disable,
.read = sparc_pmu_read,
@@ -1152,7 +1152,7 @@ static const struct pmu pmu = {
.commit_txn = sparc_pmu_commit_txn,
};
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
int err = __hw_perf_event_init(event);
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -618,7 +618,7 @@ static void x86_pmu_enable_all(int added
}
}
-static const struct pmu pmu;
+static struct pmu pmu;
static inline int is_x86_event(struct perf_event *event)
{
@@ -1394,7 +1394,7 @@ static inline void x86_pmu_read(struct p
* Set the flag to make pmu::enable() not perform the
* schedulability test, it will be performed at commit time
*/
-static void x86_pmu_start_txn(const struct pmu *pmu)
+static void x86_pmu_start_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -1407,7 +1407,7 @@ static void x86_pmu_start_txn(const stru
* Clear the flag and pmu::enable() will perform the
* schedulability test.
*/
-static void x86_pmu_cancel_txn(const struct pmu *pmu)
+static void x86_pmu_cancel_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -1424,7 +1424,7 @@ static void x86_pmu_cancel_txn(const str
* Perform the group schedulability test as a whole
* Return 0 if success
*/
-static int x86_pmu_commit_txn(const struct pmu *pmu)
+static int x86_pmu_commit_txn(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int assign[X86_PMC_IDX_MAX];
@@ -1450,7 +1450,7 @@ static int x86_pmu_commit_txn(const stru
return 0;
}
-static const struct pmu pmu = {
+static struct pmu pmu = {
.enable = x86_pmu_enable,
.disable = x86_pmu_disable,
.start = x86_pmu_start,
@@ -1536,9 +1536,9 @@ out:
return ret;
}
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
- const struct pmu *tmp;
+ struct pmu *tmp;
int err;
err = __hw_perf_event_init(event);
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -578,19 +578,19 @@ struct pmu {
* Start the transaction, after this ->enable() doesn't need
* to do schedulability tests.
*/
- void (*start_txn) (const struct pmu *pmu);
+ void (*start_txn) (struct pmu *pmu);
/*
* If ->start_txn() disabled the ->enable() schedulability test
* then ->commit_txn() is required to perform one. On success
* the transaction is closed. On error the transaction is kept
* open until ->cancel_txn() is called.
*/
- int (*commit_txn) (const struct pmu *pmu);
+ int (*commit_txn) (struct pmu *pmu);
/*
* Will cancel the transaction, assumes ->disable() is called for
* each successfull ->enable() during the transaction.
*/
- void (*cancel_txn) (const struct pmu *pmu);
+ void (*cancel_txn) (struct pmu *pmu);
};
/**
@@ -669,7 +669,7 @@ struct perf_event {
int nr_siblings;
int group_flags;
struct perf_event *group_leader;
- const struct pmu *pmu;
+ struct pmu *pmu;
enum perf_event_active_state state;
unsigned int attach_state;
@@ -847,7 +847,7 @@ struct perf_output_handle {
*/
extern int perf_max_events;
-extern const struct pmu *hw_perf_event_init(struct perf_event *event);
+extern struct pmu *hw_perf_event_init(struct perf_event *event);
extern void perf_event_task_sched_in(struct task_struct *task);
extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -75,7 +75,7 @@ static DEFINE_SPINLOCK(perf_resource_loc
/*
* Architecture provided APIs - weak aliases:
*/
-extern __weak const struct pmu *hw_perf_event_init(struct perf_event *event)
+extern __weak struct pmu *hw_perf_event_init(struct perf_event *event)
{
return NULL;
}
@@ -673,7 +673,7 @@ group_sched_in(struct perf_event *group_
struct perf_event_context *ctx)
{
struct perf_event *event, *partial_group = NULL;
- const struct pmu *pmu = group_event->pmu;
+ struct pmu *pmu = group_event->pmu;
bool txn = false;
if (group_event->state == PERF_EVENT_STATE_OFF)
@@ -4291,7 +4291,7 @@ static int perf_swevent_int(struct perf_
return 0;
}
-static const struct pmu perf_ops_generic = {
+static struct pmu perf_ops_generic = {
.enable = perf_swevent_enable,
.disable = perf_swevent_disable,
.start = perf_swevent_int,
@@ -4404,7 +4404,7 @@ static void cpu_clock_perf_event_read(st
cpu_clock_perf_event_update(event);
}
-static const struct pmu perf_ops_cpu_clock = {
+static struct pmu perf_ops_cpu_clock = {
.enable = cpu_clock_perf_event_enable,
.disable = cpu_clock_perf_event_disable,
.read = cpu_clock_perf_event_read,
@@ -4461,7 +4461,7 @@ static void task_clock_perf_event_read(s
task_clock_perf_event_update(event, time);
}
-static const struct pmu perf_ops_task_clock = {
+static struct pmu perf_ops_task_clock = {
.enable = task_clock_perf_event_enable,
.disable = task_clock_perf_event_disable,
.read = task_clock_perf_event_read,
@@ -4575,7 +4575,7 @@ static int swevent_hlist_get(struct perf
#ifdef CONFIG_EVENT_TRACING
-static const struct pmu perf_ops_tracepoint = {
+static struct pmu perf_ops_tracepoint = {
.enable = perf_trace_enable,
.disable = perf_trace_disable,
.start = perf_swevent_int,
@@ -4639,7 +4639,7 @@ static void tp_perf_event_destroy(struct
perf_trace_destroy(event);
}
-static const struct pmu *tp_perf_event_init(struct perf_event *event)
+static struct pmu *tp_perf_event_init(struct perf_event *event)
{
int err;
@@ -4686,7 +4686,7 @@ static void perf_event_free_filter(struc
#else
-static const struct pmu *tp_perf_event_init(struct perf_event *event)
+static struct pmu *tp_perf_event_init(struct perf_event *event)
{
return NULL;
}
@@ -4708,7 +4708,7 @@ static void bp_perf_event_destroy(struct
release_bp_slot(event);
}
-static const struct pmu *bp_perf_event_init(struct perf_event *bp)
+static struct pmu *bp_perf_event_init(struct perf_event *bp)
{
int err;
@@ -4732,7 +4732,7 @@ void perf_bp_event(struct perf_event *bp
perf_swevent_add(bp, 1, 1, &sample, regs);
}
#else
-static const struct pmu *bp_perf_event_init(struct perf_event *bp)
+static struct pmu *bp_perf_event_init(struct perf_event *bp)
{
return NULL;
}
@@ -4754,9 +4754,9 @@ static void sw_perf_event_destroy(struct
swevent_hlist_put(event);
}
-static const struct pmu *sw_perf_event_init(struct perf_event *event)
+static struct pmu *sw_perf_event_init(struct perf_event *event)
{
- const struct pmu *pmu = NULL;
+ struct pmu *pmu = NULL;
u64 event_id = event->attr.config;
/*
@@ -4818,7 +4818,7 @@ perf_event_alloc(struct perf_event_attr
perf_overflow_handler_t overflow_handler,
gfp_t gfpflags)
{
- const struct pmu *pmu;
+ struct pmu *pmu;
struct perf_event *event;
struct hw_perf_event *hwc;
long err;
Index: linux-2.6/arch/alpha/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/perf_event.c
+++ linux-2.6/arch/alpha/kernel/perf_event.c
@@ -642,7 +642,7 @@ static int __hw_perf_event_init(struct p
return 0;
}
-static const struct pmu pmu = {
+static struct pmu pmu = {
.enable = alpha_pmu_enable,
.disable = alpha_pmu_disable,
.read = alpha_pmu_read,
@@ -653,7 +653,7 @@ static const struct pmu pmu = {
/*
* Main entry point to initialise a HW performance event.
*/
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+struct pmu *hw_perf_event_init(struct perf_event *event)
{
int err;
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 03/19] perf: register pmu implementations
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 01/19] perf: Fix CPU hotplug Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 02/19] perf: deconstify struct pmu Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 04/19] perf: Unindent labels Peter Zijlstra
` (16 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-register-pmu.patch --]
[-- Type: text/plain, Size: 35889 bytes --]
Simple registration interface for struct pmu, this provides the
infrastructure for removing all the weak functions.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/alpha/kernel/perf_event.c | 37 +
arch/arm/kernel/perf_event.c | 38 +-
arch/powerpc/kernel/perf_event.c | 46 +-
arch/powerpc/kernel/perf_event_fsl_emb.c | 37 +
arch/sh/kernel/perf_event.c | 35 +
arch/sparc/kernel/perf_event.c | 29 -
arch/x86/kernel/cpu/perf_event.c | 45 +-
include/linux/perf_event.h | 10
kernel/hw_breakpoint.c | 35 +
kernel/perf_event.c | 588 +++++++++++++++----------------
10 files changed, 488 insertions(+), 412 deletions(-)
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -306,12 +306,7 @@ out:
return err;
}
-static struct pmu pmu = {
- .enable = armpmu_enable,
- .disable = armpmu_disable,
- .unthrottle = armpmu_unthrottle,
- .read = armpmu_read,
-};
+static struct pmu pmu;
static int
validate_event(struct cpu_hw_events *cpuc,
@@ -491,20 +486,29 @@ __hw_perf_event_init(struct perf_event *
return err;
}
-struct pmu *
-hw_perf_event_init(struct perf_event *event)
+static int armpmu_event_init(struct perf_event *event)
{
int err = 0;
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
+ break;
+
+ default:
+ return -ENOENT;
+ }
+
if (!armpmu)
- return ERR_PTR(-ENODEV);
+ return -ENODEV;
event->destroy = hw_perf_event_destroy;
if (!atomic_inc_not_zero(&active_events)) {
if (atomic_read(&active_events) > perf_max_events) {
atomic_dec(&active_events);
- return ERR_PTR(-ENOSPC);
+ return -ENOSPC;
}
mutex_lock(&pmu_reserve_mutex);
@@ -518,15 +522,23 @@ hw_perf_event_init(struct perf_event *ev
}
if (err)
- return ERR_PTR(err);
+ return err;
err = __hw_perf_event_init(event);
if (err)
hw_perf_event_destroy(event);
- return err ? ERR_PTR(err) : &pmu;
+ return err;
}
+static struct pmu pmu = {
+ .event_init = armpmu_event_init,
+ .enable = armpmu_enable,
+ .disable = armpmu_disable,
+ .unthrottle = armpmu_unthrottle,
+ .read = armpmu_read,
+};
+
void
hw_perf_enable(void)
{
@@ -2994,6 +3006,8 @@ init_hw_perf_events(void)
perf_max_events = -1;
}
+ perf_pmu_register(&pmu);
+
return 0;
}
arch_initcall(init_hw_perf_events);
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event.c
+++ linux-2.6/arch/powerpc/kernel/perf_event.c
@@ -904,16 +904,6 @@ int power_pmu_commit_txn(struct pmu *pmu
return 0;
}
-struct pmu power_pmu = {
- .enable = power_pmu_enable,
- .disable = power_pmu_disable,
- .read = power_pmu_read,
- .unthrottle = power_pmu_unthrottle,
- .start_txn = power_pmu_start_txn,
- .cancel_txn = power_pmu_cancel_txn,
- .commit_txn = power_pmu_commit_txn,
-};
-
/*
* Return 1 if we might be able to put event on a limited PMC,
* or 0 if not.
@@ -1014,7 +1004,7 @@ static int hw_perf_cache_event(u64 confi
return 0;
}
-struct pmu *hw_perf_event_init(struct perf_event *event)
+static int power_pmu_event_init(struct perf_event *event)
{
u64 ev;
unsigned long flags;
@@ -1026,25 +1016,27 @@ struct pmu *hw_perf_event_init(struct pe
struct cpu_hw_events *cpuhw;
if (!ppmu)
- return ERR_PTR(-ENXIO);
+ return -ENOENT;
+
switch (event->attr.type) {
case PERF_TYPE_HARDWARE:
ev = event->attr.config;
if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
- return ERR_PTR(-EOPNOTSUPP);
+ return -EOPNOTSUPP;
ev = ppmu->generic_events[ev];
break;
case PERF_TYPE_HW_CACHE:
err = hw_perf_cache_event(event->attr.config, &ev);
if (err)
- return ERR_PTR(err);
+ return err;
break;
case PERF_TYPE_RAW:
ev = event->attr.config;
break;
default:
- return ERR_PTR(-EINVAL);
+ return -ENOENT;
}
+
event->hw.config_base = ev;
event->hw.idx = 0;
@@ -1081,7 +1073,7 @@ struct pmu *hw_perf_event_init(struct pe
*/
ev = normal_pmc_alternative(ev, flags);
if (!ev)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
}
@@ -1095,19 +1087,19 @@ struct pmu *hw_perf_event_init(struct pe
n = collect_events(event->group_leader, ppmu->n_counter - 1,
ctrs, events, cflags);
if (n < 0)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
events[n] = ev;
ctrs[n] = event;
cflags[n] = flags;
if (check_excludes(ctrs, cflags, n, 1))
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
cpuhw = &get_cpu_var(cpu_hw_events);
err = power_check_constraints(cpuhw, events, cflags, n + 1);
put_cpu_var(cpu_hw_events);
if (err)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
event->hw.config = events[n];
event->hw.event_base = cflags[n];
@@ -1132,11 +1124,20 @@ struct pmu *hw_perf_event_init(struct pe
}
event->destroy = hw_perf_event_destroy;
- if (err)
- return ERR_PTR(err);
- return &power_pmu;
+ return err;
}
+struct pmu power_pmu = {
+ .event_init = power_pmu_event_init,
+ .enable = power_pmu_enable,
+ .disable = power_pmu_disable,
+ .read = power_pmu_read,
+ .unthrottle = power_pmu_unthrottle,
+ .start_txn = power_pmu_start_txn,
+ .cancel_txn = power_pmu_cancel_txn,
+ .commit_txn = power_pmu_commit_txn,
+};
+
/*
* A counter has overflowed; update its count and record
* things if requested. Note that interrupts are hard-disabled
@@ -1342,6 +1343,7 @@ int register_power_pmu(struct power_pmu
freeze_events_kernel = MMCR0_FCHV;
#endif /* CONFIG_PPC64 */
+ perf_pmu_register(&power_pmu);
perf_cpu_notifier(power_pmu_notifier);
return 0;
Index: linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -378,13 +378,6 @@ static void fsl_emb_pmu_unthrottle(struc
local_irq_restore(flags);
}
-static struct pmu fsl_emb_pmu = {
- .enable = fsl_emb_pmu_enable,
- .disable = fsl_emb_pmu_disable,
- .read = fsl_emb_pmu_read,
- .unthrottle = fsl_emb_pmu_unthrottle,
-};
-
/*
* Release the PMU if this is the last perf_event.
*/
@@ -428,7 +421,7 @@ static int hw_perf_cache_event(u64 confi
return 0;
}
-struct pmu *hw_perf_event_init(struct perf_event *event)
+static int fsl_emb_pmu_event_init(struct perf_event *event)
{
u64 ev;
struct perf_event *events[MAX_HWEVENTS];
@@ -441,14 +434,14 @@ struct pmu *hw_perf_event_init(struct pe
case PERF_TYPE_HARDWARE:
ev = event->attr.config;
if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
- return ERR_PTR(-EOPNOTSUPP);
+ return -EOPNOTSUPP;
ev = ppmu->generic_events[ev];
break;
case PERF_TYPE_HW_CACHE:
err = hw_perf_cache_event(event->attr.config, &ev);
if (err)
- return ERR_PTR(err);
+ return err;
break;
case PERF_TYPE_RAW:
@@ -456,12 +449,12 @@ struct pmu *hw_perf_event_init(struct pe
break;
default:
- return ERR_PTR(-EINVAL);
+ return -ENOENT;
}
event->hw.config = ppmu->xlate_event(ev);
if (!(event->hw.config & FSL_EMB_EVENT_VALID))
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
/*
* If this is in a group, check if it can go on with all the
@@ -473,7 +466,7 @@ struct pmu *hw_perf_event_init(struct pe
n = collect_events(event->group_leader,
ppmu->n_counter - 1, events);
if (n < 0)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
@@ -484,7 +477,7 @@ struct pmu *hw_perf_event_init(struct pe
}
if (num_restricted >= ppmu->n_restricted)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
event->hw.idx = -1;
@@ -497,7 +490,7 @@ struct pmu *hw_perf_event_init(struct pe
if (event->attr.exclude_kernel)
event->hw.config_base |= PMLCA_FCS;
if (event->attr.exclude_idle)
- return ERR_PTR(-ENOTSUPP);
+ return -ENOTSUPP;
event->hw.last_period = event->hw.sample_period;
local64_set(&event->hw.period_left, event->hw.last_period);
@@ -523,11 +516,17 @@ struct pmu *hw_perf_event_init(struct pe
}
event->destroy = hw_perf_event_destroy;
- if (err)
- return ERR_PTR(err);
- return &fsl_emb_pmu;
+ return err;
}
+static struct pmu fsl_emb_pmu = {
+ .event_init = fsl_emb_pmu_event_init,
+ .enable = fsl_emb_pmu_enable,
+ .disable = fsl_emb_pmu_disable,
+ .read = fsl_emb_pmu_read,
+ .unthrottle = fsl_emb_pmu_unthrottle,
+};
+
/*
* A counter has overflowed; update its count and record
* things if requested. Note that interrupts are hard-disabled
@@ -651,5 +650,7 @@ int register_fsl_emb_pmu(struct fsl_emb_
pr_info("%s performance monitor hardware support registered\n",
pmu->name);
+ perf_pmu_register(&fsl_emb_pmu);
+
return 0;
}
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sh/kernel/perf_event.c
+++ linux-2.6/arch/sh/kernel/perf_event.c
@@ -257,26 +257,38 @@ static void sh_pmu_read(struct perf_even
sh_perf_event_update(event, &event->hw, event->hw.idx);
}
-static struct pmu pmu = {
- .enable = sh_pmu_enable,
- .disable = sh_pmu_disable,
- .read = sh_pmu_read,
-};
-
-struct pmu *hw_perf_event_init(struct perf_event *event)
+static int sh_pmu_event_init(struct perf_event *event)
{
- int err = __hw_perf_event_init(event);
+ int err;
+
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HW_CACHE:
+ case PERF_TYPE_HARDWARE:
+ err = __hw_perf_event_init(event);
+ break;
+
+ default:
+ return -ENOENT;
+ }
+
if (unlikely(err)) {
if (event->destroy)
event->destroy(event);
- return ERR_PTR(err);
}
- return &pmu;
+ return err;
}
+static struct pmu pmu = {
+ .event_init = sh_pmu_event_init,
+ .enable = sh_pmu_enable,
+ .disable = sh_pmu_disable,
+ .read = sh_pmu_read,
+};
+
static void sh_pmu_setup(int cpu)
-{
+
struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
memset(cpuhw, 0, sizeof(struct cpu_hw_events));
@@ -325,6 +337,7 @@ int __cpuinit register_sh_pmu(struct sh_
WARN_ON(pmu->num_events > MAX_HWEVENTS);
+ perf_pmu_register(&pmu);
perf_cpu_notifier(sh_pmu_notifier);
return 0;
}
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -1025,7 +1025,7 @@ out:
return ret;
}
-static int __hw_perf_event_init(struct perf_event *event)
+static int sparc_pmu_event_init(struct perf_event *event)
{
struct perf_event_attr *attr = &event->attr;
struct perf_event *evts[MAX_HWEVENTS];
@@ -1038,17 +1038,27 @@ static int __hw_perf_event_init(struct p
if (atomic_read(&nmi_active) < 0)
return -ENODEV;
- if (attr->type == PERF_TYPE_HARDWARE) {
+ switch (attr->type) {
+ case PERF_TYPE_HARDWARE:
if (attr->config >= sparc_pmu->max_events)
return -EINVAL;
pmap = sparc_pmu->event_map(attr->config);
- } else if (attr->type == PERF_TYPE_HW_CACHE) {
+ break;
+
+ case PERF_TYPE_HW_CACHE:
pmap = sparc_map_cache_event(attr->config);
if (IS_ERR(pmap))
return PTR_ERR(pmap);
- } else
+ break;
+
+ case PERF_TYPE_RAW:
return -EOPNOTSUPP;
+ default:
+ return -ENOENT;
+
+ }
+
/* We save the enable bits in the config_base. */
hwc->config_base = sparc_pmu->irq_bit;
if (!attr->exclude_user)
@@ -1143,6 +1153,7 @@ static int sparc_pmu_commit_txn(struct p
}
static struct pmu pmu = {
+ .event_init = sparc_pmu_event_init,
.enable = sparc_pmu_enable,
.disable = sparc_pmu_disable,
.read = sparc_pmu_read,
@@ -1152,15 +1163,6 @@ static struct pmu pmu = {
.commit_txn = sparc_pmu_commit_txn,
};
-struct pmu *hw_perf_event_init(struct perf_event *event)
-{
- int err = __hw_perf_event_init(event);
-
- if (err)
- return ERR_PTR(err);
- return &pmu;
-}
-
void perf_event_print_debug(void)
{
unsigned long flags;
@@ -1280,6 +1282,7 @@ void __init init_hw_perf_events(void)
/* All sparc64 PMUs currently have 2 events. */
perf_max_events = 2;
+ perf_pmu_register(&pmu);
register_die_notifier(&perf_event_nmi_notifier);
}
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -530,7 +530,7 @@ static int x86_pmu_hw_config(struct perf
/*
* Setup the hardware configuration for a given attr_type
*/
-static int __hw_perf_event_init(struct perf_event *event)
+static int __x86_pmu_event_init(struct perf_event *event)
{
int err;
@@ -1381,6 +1381,7 @@ void __init init_hw_perf_events(void)
pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
+ perf_pmu_register(&pmu);
perf_cpu_notifier(x86_pmu_notifier);
}
@@ -1450,18 +1451,6 @@ static int x86_pmu_commit_txn(struct pmu
return 0;
}
-static struct pmu pmu = {
- .enable = x86_pmu_enable,
- .disable = x86_pmu_disable,
- .start = x86_pmu_start,
- .stop = x86_pmu_stop,
- .read = x86_pmu_read,
- .unthrottle = x86_pmu_unthrottle,
- .start_txn = x86_pmu_start_txn,
- .cancel_txn = x86_pmu_cancel_txn,
- .commit_txn = x86_pmu_commit_txn,
-};
-
/*
* validate that we can schedule this event
*/
@@ -1536,12 +1525,22 @@ out:
return ret;
}
-struct pmu *hw_perf_event_init(struct perf_event *event)
+int x86_pmu_event_init(struct perf_event *event)
{
struct pmu *tmp;
int err;
- err = __hw_perf_event_init(event);
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
+ break;
+
+ default:
+ return -ENOENT;
+ }
+
+ err = __x86_pmu_event_init(event);
if (!err) {
/*
* we temporarily connect event to its pmu
@@ -1561,12 +1560,24 @@ struct pmu *hw_perf_event_init(struct pe
if (err) {
if (event->destroy)
event->destroy(event);
- return ERR_PTR(err);
}
- return &pmu;
+ return err;
}
+static struct pmu pmu = {
+ .event_init = x86_pmu_event_init,
+ .enable = x86_pmu_enable,
+ .disable = x86_pmu_disable,
+ .start = x86_pmu_start,
+ .stop = x86_pmu_stop,
+ .read = x86_pmu_read,
+ .unthrottle = x86_pmu_unthrottle,
+ .start_txn = x86_pmu_start_txn,
+ .cancel_txn = x86_pmu_cancel_txn,
+ .commit_txn = x86_pmu_commit_txn,
+};
+
/*
* callchain support
*/
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -561,6 +561,13 @@ struct perf_event;
* struct pmu - generic performance monitoring unit
*/
struct pmu {
+ struct list_head entry;
+
+ /*
+ * Should return -ENOENT when the @event doesn't match this pmu
+ */
+ int (*event_init) (struct perf_event *event);
+
int (*enable) (struct perf_event *event);
void (*disable) (struct perf_event *event);
int (*start) (struct perf_event *event);
@@ -847,7 +854,8 @@ struct perf_output_handle {
*/
extern int perf_max_events;
-extern struct pmu *hw_perf_event_init(struct perf_event *event);
+extern int perf_pmu_register(struct pmu *pmu);
+extern void perf_pmu_unregister(struct pmu *pmu);
extern void perf_event_task_sched_in(struct task_struct *task);
extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
Index: linux-2.6/kernel/hw_breakpoint.c
===================================================================
--- linux-2.6.orig/kernel/hw_breakpoint.c
+++ linux-2.6/kernel/hw_breakpoint.c
@@ -565,6 +565,34 @@ static struct notifier_block hw_breakpoi
.priority = 0x7fffffff
};
+static void bp_perf_event_destroy(struct perf_event *event)
+{
+ release_bp_slot(event);
+}
+
+static int hw_breakpoint_event_init(struct perf_event *bp)
+{
+ int err;
+
+ if (bp->attr.type != PERF_TYPE_BREAKPOINT)
+ return -ENOENT;
+
+ err = register_perf_hw_breakpoint(bp);
+ if (err)
+ return err;
+
+ bp->destroy = bp_perf_event_destroy;
+
+ return 0;
+}
+
+static struct pmu perf_breakpoint = {
+ .event_init = hw_breakpoint_event_init,
+ .enable = arch_install_hw_breakpoint,
+ .disable = arch_uninstall_hw_breakpoint,
+ .read = hw_breakpoint_pmu_read,
+};
+
static int __init init_hw_breakpoint(void)
{
unsigned int **task_bp_pinned;
@@ -586,6 +614,8 @@ static int __init init_hw_breakpoint(voi
constraints_initialized = 1;
+ perf_pmu_register(&perf_breakpoint);
+
return register_die_notifier(&hw_breakpoint_exceptions_nb);
err_alloc:
@@ -601,8 +631,3 @@ static int __init init_hw_breakpoint(voi
core_initcall(init_hw_breakpoint);
-struct pmu perf_ops_bp = {
- .enable = arch_install_hw_breakpoint,
- .disable = arch_uninstall_hw_breakpoint,
- .read = hw_breakpoint_pmu_read,
-};
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -31,7 +31,6 @@
#include <linux/kernel_stat.h>
#include <linux/perf_event.h>
#include <linux/ftrace_event.h>
-#include <linux/hw_breakpoint.h>
#include <asm/irq_regs.h>
@@ -72,14 +71,6 @@ static atomic64_t perf_event_id;
*/
static DEFINE_SPINLOCK(perf_resource_lock);
-/*
- * Architecture provided APIs - weak aliases:
- */
-extern __weak struct pmu *hw_perf_event_init(struct perf_event *event)
-{
- return NULL;
-}
-
void __weak hw_perf_disable(void) { barrier(); }
void __weak hw_perf_enable(void) { barrier(); }
@@ -4291,182 +4282,6 @@ static int perf_swevent_int(struct perf_
return 0;
}
-static struct pmu perf_ops_generic = {
- .enable = perf_swevent_enable,
- .disable = perf_swevent_disable,
- .start = perf_swevent_int,
- .stop = perf_swevent_void,
- .read = perf_swevent_read,
- .unthrottle = perf_swevent_void, /* hwc->interrupts already reset */
-};
-
-/*
- * hrtimer based swevent callback
- */
-
-static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
-{
- enum hrtimer_restart ret = HRTIMER_RESTART;
- struct perf_sample_data data;
- struct pt_regs *regs;
- struct perf_event *event;
- u64 period;
-
- event = container_of(hrtimer, struct perf_event, hw.hrtimer);
- event->pmu->read(event);
-
- perf_sample_data_init(&data, 0);
- data.period = event->hw.last_period;
- regs = get_irq_regs();
-
- if (regs && !perf_exclude_event(event, regs)) {
- if (!(event->attr.exclude_idle && current->pid == 0))
- if (perf_event_overflow(event, 0, &data, regs))
- ret = HRTIMER_NORESTART;
- }
-
- period = max_t(u64, 10000, event->hw.sample_period);
- hrtimer_forward_now(hrtimer, ns_to_ktime(period));
-
- return ret;
-}
-
-static void perf_swevent_start_hrtimer(struct perf_event *event)
-{
- struct hw_perf_event *hwc = &event->hw;
-
- hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- hwc->hrtimer.function = perf_swevent_hrtimer;
- if (hwc->sample_period) {
- u64 period;
-
- if (hwc->remaining) {
- if (hwc->remaining < 0)
- period = 10000;
- else
- period = hwc->remaining;
- hwc->remaining = 0;
- } else {
- period = max_t(u64, 10000, hwc->sample_period);
- }
- __hrtimer_start_range_ns(&hwc->hrtimer,
- ns_to_ktime(period), 0,
- HRTIMER_MODE_REL, 0);
- }
-}
-
-static void perf_swevent_cancel_hrtimer(struct perf_event *event)
-{
- struct hw_perf_event *hwc = &event->hw;
-
- if (hwc->sample_period) {
- ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
- hwc->remaining = ktime_to_ns(remaining);
-
- hrtimer_cancel(&hwc->hrtimer);
- }
-}
-
-/*
- * Software event: cpu wall time clock
- */
-
-static void cpu_clock_perf_event_update(struct perf_event *event)
-{
- int cpu = raw_smp_processor_id();
- s64 prev;
- u64 now;
-
- now = cpu_clock(cpu);
- prev = local64_xchg(&event->hw.prev_count, now);
- local64_add(now - prev, &event->count);
-}
-
-static int cpu_clock_perf_event_enable(struct perf_event *event)
-{
- struct hw_perf_event *hwc = &event->hw;
- int cpu = raw_smp_processor_id();
-
- local64_set(&hwc->prev_count, cpu_clock(cpu));
- perf_swevent_start_hrtimer(event);
-
- return 0;
-}
-
-static void cpu_clock_perf_event_disable(struct perf_event *event)
-{
- perf_swevent_cancel_hrtimer(event);
- cpu_clock_perf_event_update(event);
-}
-
-static void cpu_clock_perf_event_read(struct perf_event *event)
-{
- cpu_clock_perf_event_update(event);
-}
-
-static struct pmu perf_ops_cpu_clock = {
- .enable = cpu_clock_perf_event_enable,
- .disable = cpu_clock_perf_event_disable,
- .read = cpu_clock_perf_event_read,
-};
-
-/*
- * Software event: task time clock
- */
-
-static void task_clock_perf_event_update(struct perf_event *event, u64 now)
-{
- u64 prev;
- s64 delta;
-
- prev = local64_xchg(&event->hw.prev_count, now);
- delta = now - prev;
- local64_add(delta, &event->count);
-}
-
-static int task_clock_perf_event_enable(struct perf_event *event)
-{
- struct hw_perf_event *hwc = &event->hw;
- u64 now;
-
- now = event->ctx->time;
-
- local64_set(&hwc->prev_count, now);
-
- perf_swevent_start_hrtimer(event);
-
- return 0;
-}
-
-static void task_clock_perf_event_disable(struct perf_event *event)
-{
- perf_swevent_cancel_hrtimer(event);
- task_clock_perf_event_update(event, event->ctx->time);
-
-}
-
-static void task_clock_perf_event_read(struct perf_event *event)
-{
- u64 time;
-
- if (!in_nmi()) {
- update_context_time(event->ctx);
- time = event->ctx->time;
- } else {
- u64 now = perf_clock();
- u64 delta = now - event->ctx->timestamp;
- time = event->ctx->time + delta;
- }
-
- task_clock_perf_event_update(event, time);
-}
-
-static struct pmu perf_ops_task_clock = {
- .enable = task_clock_perf_event_enable,
- .disable = task_clock_perf_event_disable,
- .read = task_clock_perf_event_read,
-};
-
/* Deref the hlist from the update side */
static inline struct swevent_hlist *
swevent_hlist_deref(struct perf_cpu_context *cpuctx)
@@ -4573,17 +4388,63 @@ static int swevent_hlist_get(struct perf
return err;
}
-#ifdef CONFIG_EVENT_TRACING
+atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
-static struct pmu perf_ops_tracepoint = {
- .enable = perf_trace_enable,
- .disable = perf_trace_disable,
+static void sw_perf_event_destroy(struct perf_event *event)
+{
+ u64 event_id = event->attr.config;
+
+ WARN_ON(event->parent);
+
+ atomic_dec(&perf_swevent_enabled[event_id]);
+ swevent_hlist_put(event);
+}
+
+static int perf_swevent_init(struct perf_event *event)
+{
+ int event_id = event->attr.config;
+
+ if (event->attr.type != PERF_TYPE_SOFTWARE)
+ return -ENOENT;
+
+ switch (event_id) {
+ case PERF_COUNT_SW_CPU_CLOCK:
+ case PERF_COUNT_SW_TASK_CLOCK:
+ return -ENOENT;
+
+ default:
+ break;
+ }
+
+ if (event_id > PERF_COUNT_SW_MAX)
+ return -ENOENT;
+
+ if (!event->parent) {
+ int err;
+
+ err = swevent_hlist_get(event);
+ if (err)
+ return err;
+
+ atomic_inc(&perf_swevent_enabled[event_id]);
+ event->destroy = sw_perf_event_destroy;
+ }
+
+ return 0;
+}
+
+static struct pmu perf_swevent = {
+ .event_init = perf_swevent_init,
+ .enable = perf_swevent_enable,
+ .disable = perf_swevent_disable,
.start = perf_swevent_int,
.stop = perf_swevent_void,
.read = perf_swevent_read,
- .unthrottle = perf_swevent_void,
+ .unthrottle = perf_swevent_void, /* hwc->interrupts already reset */
};
+#ifdef CONFIG_EVENT_TRACING
+
static int perf_tp_filter_match(struct perf_event *event,
struct perf_sample_data *data)
{
@@ -4639,10 +4500,13 @@ static void tp_perf_event_destroy(struct
perf_trace_destroy(event);
}
-static struct pmu *tp_perf_event_init(struct perf_event *event)
+static int perf_tp_event_init(struct perf_event *event)
{
int err;
+ if (event->attr.type != PERF_TYPE_TRACEPOINT)
+ return -ENOENT;
+
/*
* Raw tracepoint data is a severe data leak, only allow root to
* have these.
@@ -4650,15 +4514,30 @@ static struct pmu *tp_perf_event_init(st
if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
perf_paranoid_tracepoint_raw() &&
!capable(CAP_SYS_ADMIN))
- return ERR_PTR(-EPERM);
+ return -EPERM;
err = perf_trace_init(event);
if (err)
- return NULL;
+ return err;
event->destroy = tp_perf_event_destroy;
- return &perf_ops_tracepoint;
+ return 0;
+}
+
+static struct pmu perf_tracepoint = {
+ .event_init = perf_tp_event_init,
+ .enable = perf_trace_enable,
+ .disable = perf_trace_disable,
+ .start = perf_swevent_int,
+ .stop = perf_swevent_void,
+ .read = perf_swevent_read,
+ .unthrottle = perf_swevent_void,
+};
+
+static inline void perf_tp_register(void)
+{
+ perf_pmu_register(&perf_tracepoint);
}
static int perf_event_set_filter(struct perf_event *event, void __user *arg)
@@ -4686,9 +4565,8 @@ static void perf_event_free_filter(struc
#else
-static struct pmu *tp_perf_event_init(struct perf_event *event)
+static inline void perf_tp_register(void)
{
- return NULL;
}
static int perf_event_set_filter(struct perf_event *event, void __user *arg)
@@ -4703,105 +4581,247 @@ static void perf_event_free_filter(struc
#endif /* CONFIG_EVENT_TRACING */
#ifdef CONFIG_HAVE_HW_BREAKPOINT
-static void bp_perf_event_destroy(struct perf_event *event)
+void perf_bp_event(struct perf_event *bp, void *data)
{
- release_bp_slot(event);
+ struct perf_sample_data sample;
+ struct pt_regs *regs = data;
+
+ perf_sample_data_init(&sample, bp->attr.bp_addr);
+
+ if (!perf_exclude_event(bp, regs))
+ perf_swevent_add(bp, 1, 1, &sample, regs);
}
+#endif
-static struct pmu *bp_perf_event_init(struct perf_event *bp)
+/*
+ * hrtimer based swevent callback
+ */
+
+static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
{
- int err;
+ enum hrtimer_restart ret = HRTIMER_RESTART;
+ struct perf_sample_data data;
+ struct pt_regs *regs;
+ struct perf_event *event;
+ u64 period;
- err = register_perf_hw_breakpoint(bp);
- if (err)
- return ERR_PTR(err);
+ event = container_of(hrtimer, struct perf_event, hw.hrtimer);
+ event->pmu->read(event);
+
+ perf_sample_data_init(&data, 0);
+ data.period = event->hw.last_period;
+ regs = get_irq_regs();
+
+ if (regs && !perf_exclude_event(event, regs)) {
+ if (!(event->attr.exclude_idle && current->pid == 0))
+ if (perf_event_overflow(event, 0, &data, regs))
+ ret = HRTIMER_NORESTART;
+ }
- bp->destroy = bp_perf_event_destroy;
+ period = max_t(u64, 10000, event->hw.sample_period);
+ hrtimer_forward_now(hrtimer, ns_to_ktime(period));
- return &perf_ops_bp;
+ return ret;
}
-void perf_bp_event(struct perf_event *bp, void *data)
+static void perf_swevent_start_hrtimer(struct perf_event *event)
{
- struct perf_sample_data sample;
- struct pt_regs *regs = data;
+ struct hw_perf_event *hwc = &event->hw;
- perf_sample_data_init(&sample, bp->attr.bp_addr);
+ hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ hwc->hrtimer.function = perf_swevent_hrtimer;
+ if (hwc->sample_period) {
+ u64 period;
- if (!perf_exclude_event(bp, regs))
- perf_swevent_add(bp, 1, 1, &sample, regs);
+ if (hwc->remaining) {
+ if (hwc->remaining < 0)
+ period = 10000;
+ else
+ period = hwc->remaining;
+ hwc->remaining = 0;
+ } else {
+ period = max_t(u64, 10000, hwc->sample_period);
+ }
+ __hrtimer_start_range_ns(&hwc->hrtimer,
+ ns_to_ktime(period), 0,
+ HRTIMER_MODE_REL, 0);
+ }
}
-#else
-static struct pmu *bp_perf_event_init(struct perf_event *bp)
+
+static void perf_swevent_cancel_hrtimer(struct perf_event *event)
{
- return NULL;
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (hwc->sample_period) {
+ ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
+ hwc->remaining = ktime_to_ns(remaining);
+
+ hrtimer_cancel(&hwc->hrtimer);
+ }
}
-void perf_bp_event(struct perf_event *bp, void *regs)
+/*
+ * Software event: cpu wall time clock
+ */
+
+static void cpu_clock_event_update(struct perf_event *event)
{
+ int cpu = raw_smp_processor_id();
+ s64 prev;
+ u64 now;
+
+ now = cpu_clock(cpu);
+ prev = local64_xchg(&event->hw.prev_count, now);
+ local64_add(now - prev, &event->count);
}
-#endif
-atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
+static int cpu_clock_event_enable(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ int cpu = raw_smp_processor_id();
-static void sw_perf_event_destroy(struct perf_event *event)
+ local64_set(&hwc->prev_count, cpu_clock(cpu));
+ perf_swevent_start_hrtimer(event);
+
+ return 0;
+}
+
+static void cpu_clock_event_disable(struct perf_event *event)
{
- u64 event_id = event->attr.config;
+ perf_swevent_cancel_hrtimer(event);
+ cpu_clock_event_update(event);
+}
- WARN_ON(event->parent);
+static void cpu_clock_event_read(struct perf_event *event)
+{
+ cpu_clock_event_update(event);
+}
- atomic_dec(&perf_swevent_enabled[event_id]);
- swevent_hlist_put(event);
+static int cpu_clock_event_init(struct perf_event *event)
+{
+ if (event->attr.type != PERF_TYPE_SOFTWARE)
+ return -ENOENT;
+
+ if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
+ return -ENOENT;
+
+ return 0;
}
-static struct pmu *sw_perf_event_init(struct perf_event *event)
+static struct pmu perf_cpu_clock = {
+ .event_init = cpu_clock_event_init,
+ .enable = cpu_clock_event_enable,
+ .disable = cpu_clock_event_disable,
+ .read = cpu_clock_event_read,
+};
+
+/*
+ * Software event: task time clock
+ */
+
+static void task_clock_event_update(struct perf_event *event, u64 now)
{
- struct pmu *pmu = NULL;
- u64 event_id = event->attr.config;
+ u64 prev;
+ s64 delta;
- /*
- * Software events (currently) can't in general distinguish
- * between user, kernel and hypervisor events.
- * However, context switches and cpu migrations are considered
- * to be kernel events, and page faults are never hypervisor
- * events.
- */
- switch (event_id) {
- case PERF_COUNT_SW_CPU_CLOCK:
- pmu = &perf_ops_cpu_clock;
+ prev = local64_xchg(&event->hw.prev_count, now);
+ delta = now - prev;
+ local64_add(delta, &event->count);
+}
- break;
- case PERF_COUNT_SW_TASK_CLOCK:
- /*
- * If the user instantiates this as a per-cpu event,
- * use the cpu_clock event instead.
- */
- if (event->ctx->task)
- pmu = &perf_ops_task_clock;
- else
- pmu = &perf_ops_cpu_clock;
+static int task_clock_event_enable(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ u64 now;
- break;
- case PERF_COUNT_SW_PAGE_FAULTS:
- case PERF_COUNT_SW_PAGE_FAULTS_MIN:
- case PERF_COUNT_SW_PAGE_FAULTS_MAJ:
- case PERF_COUNT_SW_CONTEXT_SWITCHES:
- case PERF_COUNT_SW_CPU_MIGRATIONS:
- case PERF_COUNT_SW_ALIGNMENT_FAULTS:
- case PERF_COUNT_SW_EMULATION_FAULTS:
- if (!event->parent) {
- int err;
-
- err = swevent_hlist_get(event);
- if (err)
- return ERR_PTR(err);
+ now = event->ctx->time;
+
+ local64_set(&hwc->prev_count, now);
+
+ perf_swevent_start_hrtimer(event);
+
+ return 0;
+}
+
+static void task_clock_event_disable(struct perf_event *event)
+{
+ perf_swevent_cancel_hrtimer(event);
+ task_clock_event_update(event, event->ctx->time);
+
+}
+
+static void task_clock_event_read(struct perf_event *event)
+{
+ u64 time;
+
+ if (!in_nmi()) {
+ update_context_time(event->ctx);
+ time = event->ctx->time;
+ } else {
+ u64 now = perf_clock();
+ u64 delta = now - event->ctx->timestamp;
+ time = event->ctx->time + delta;
+ }
+
+ task_clock_event_update(event, time);
+}
+
+static int task_clock_event_init(struct perf_event *event)
+{
+ if (event->attr.type != PERF_TYPE_SOFTWARE)
+ return -ENOENT;
+
+ if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
+ return -ENOENT;
+
+ return 0;
+}
+
+static struct pmu perf_task_clock = {
+ .event_init = task_clock_event_init,
+ .enable = task_clock_event_enable,
+ .disable = task_clock_event_disable,
+ .read = task_clock_event_read,
+};
+
+static LIST_HEAD(pmus);
+static DEFINE_MUTEX(pmus_lock);
+static struct srcu_struct pmus_srcu;
+
+int perf_pmu_register(struct pmu *pmu)
+{
+ mutex_lock(&pmus_lock);
+ list_add_rcu(&pmu->entry, &pmus);
+ mutex_unlock(&pmus_lock);
+
+ return 0;
+}
+
+void perf_pmu_unregister(struct pmu *pmu)
+{
+ mutex_lock(&pmus_lock);
+ list_del_rcu(&pmu->entry);
+ mutex_unlock(&pmus_lock);
- atomic_inc(&perf_swevent_enabled[event_id]);
- event->destroy = sw_perf_event_destroy;
+ synchronize_srcu(&pmus_srcu);
+}
+
+struct pmu *perf_init_event(struct perf_event *event)
+{
+ struct pmu *pmu = NULL;
+ int idx;
+
+ idx = srcu_read_lock(&pmus_srcu);
+ list_for_each_entry_rcu(pmu, &pmus, entry) {
+ int ret = pmu->event_init(event);
+ if (!ret)
+ break;
+ if (ret != -ENOENT) {
+ pmu = ERR_PTR(ret);
+ break;
}
- pmu = &perf_ops_generic;
- break;
}
+ srcu_read_unlock(&pmus_srcu, idx);
return pmu;
}
@@ -4882,29 +4902,8 @@ perf_event_alloc(struct perf_event_attr
if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
goto done;
- switch (attr->type) {
- case PERF_TYPE_RAW:
- case PERF_TYPE_HARDWARE:
- case PERF_TYPE_HW_CACHE:
- pmu = hw_perf_event_init(event);
- break;
-
- case PERF_TYPE_SOFTWARE:
- pmu = sw_perf_event_init(event);
- break;
-
- case PERF_TYPE_TRACEPOINT:
- pmu = tp_perf_event_init(event);
- break;
+ pmu = perf_init_event(event);
- case PERF_TYPE_BREAKPOINT:
- pmu = bp_perf_event_init(event);
- break;
-
-
- default:
- break;
- }
done:
err = 0;
if (!pmu)
@@ -5762,22 +5761,15 @@ perf_cpu_notify(struct notifier_block *s
return NOTIFY_OK;
}
-/*
- * This has to have a higher priority than migration_notifier in sched.c.
- */
-static struct notifier_block __cpuinitdata perf_cpu_nb = {
- .notifier_call = perf_cpu_notify,
- .priority = 20,
-};
-
void __init perf_event_init(void)
{
perf_event_init_all_cpus();
- perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
- (void *)(long)smp_processor_id());
- perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
- (void *)(long)smp_processor_id());
- register_cpu_notifier(&perf_cpu_nb);
+ init_srcu_struct(&pmus_srcu);
+ perf_pmu_register(&perf_swevent);
+ perf_pmu_register(&perf_cpu_clock);
+ perf_pmu_register(&perf_task_clock);
+ perf_tp_register();
+ perf_cpu_notifier(perf_cpu_notify);
}
static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
Index: linux-2.6/arch/alpha/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/perf_event.c
+++ linux-2.6/arch/alpha/kernel/perf_event.c
@@ -642,34 +642,39 @@ static int __hw_perf_event_init(struct p
return 0;
}
-static struct pmu pmu = {
- .enable = alpha_pmu_enable,
- .disable = alpha_pmu_disable,
- .read = alpha_pmu_read,
- .unthrottle = alpha_pmu_unthrottle,
-};
-
-
/*
* Main entry point to initialise a HW performance event.
*/
-struct pmu *hw_perf_event_init(struct perf_event *event)
+static int alpha_pmu_event_init(struct perf_event *event)
{
int err;
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
+ break;
+
+ default:
+ return -ENOENT;
+ }
+
if (!alpha_pmu)
- return ERR_PTR(-ENODEV);
+ return -ENODEV;
/* Do the real initialisation work. */
err = __hw_perf_event_init(event);
- if (err)
- return ERR_PTR(err);
-
- return &pmu;
+ return err;
}
-
+static struct pmu pmu = {
+ .event_init = alpha_pmu_event_init,
+ .enable = alpha_pmu_enable,
+ .disable = alpha_pmu_disable,
+ .read = alpha_pmu_read,
+ .unthrottle = alpha_pmu_unthrottle,
+};
/*
* Main entry point - enable HW performance counters.
@@ -838,5 +843,7 @@ void __init init_hw_perf_events(void)
/* And set up PMU specification */
alpha_pmu = &ev67_pmu;
perf_max_events = alpha_pmu->num_pmcs;
+
+ perf_pmu_register(&pmu);
}
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 04/19] perf: Unindent labels
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (2 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 03/19] perf: register pmu implementations Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 05/19] perf: Reduce perf_disable() usage Peter Zijlstra
` (15 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-whitespace.patch --]
[-- Type: text/plain, Size: 4256 bytes --]
Fixup random annoying style bits
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 43 ++++++++++++++++++++++++-------------------
1 file changed, 24 insertions(+), 19 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -147,7 +147,7 @@ perf_lock_task_context(struct task_struc
struct perf_event_context *ctx;
rcu_read_lock();
- retry:
+retry:
ctx = rcu_dereference(task->perf_event_ctxp);
if (ctx) {
/*
@@ -601,7 +601,7 @@ void perf_event_disable(struct perf_even
return;
}
- retry:
+retry:
task_oncpu_function_call(task, __perf_event_disable, event);
raw_spin_lock_irq(&ctx->lock);
@@ -831,7 +831,7 @@ static void __perf_install_in_context(vo
if (!err && !ctx->task && cpuctx->max_pertask)
cpuctx->max_pertask--;
- unlock:
+unlock:
perf_enable();
raw_spin_unlock(&ctx->lock);
@@ -904,10 +904,12 @@ static void __perf_event_mark_enabled(st
event->state = PERF_EVENT_STATE_INACTIVE;
event->tstamp_enabled = ctx->time - event->total_time_enabled;
- list_for_each_entry(sub, &event->sibling_list, group_entry)
- if (sub->state >= PERF_EVENT_STATE_INACTIVE)
+ list_for_each_entry(sub, &event->sibling_list, group_entry) {
+ if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
sub->tstamp_enabled =
ctx->time - sub->total_time_enabled;
+ }
+ }
}
/*
@@ -973,7 +975,7 @@ static void __perf_event_enable(void *in
}
}
- unlock:
+unlock:
raw_spin_unlock(&ctx->lock);
}
@@ -1014,7 +1016,7 @@ void perf_event_enable(struct perf_event
if (event->state == PERF_EVENT_STATE_ERROR)
event->state = PERF_EVENT_STATE_OFF;
- retry:
+retry:
raw_spin_unlock_irq(&ctx->lock);
task_oncpu_function_call(task, __perf_event_enable, event);
@@ -1034,7 +1036,7 @@ void perf_event_enable(struct perf_event
if (event->state == PERF_EVENT_STATE_OFF)
__perf_event_mark_enabled(event, ctx);
- out:
+out:
raw_spin_unlock_irq(&ctx->lock);
}
@@ -1074,17 +1076,19 @@ static void ctx_sched_out(struct perf_ev
if (!ctx->nr_active)
goto out_enable;
- if (event_type & EVENT_PINNED)
+ if (event_type & EVENT_PINNED) {
list_for_each_entry(event, &ctx->pinned_groups, group_entry)
group_sched_out(event, cpuctx, ctx);
+ }
- if (event_type & EVENT_FLEXIBLE)
+ if (event_type & EVENT_FLEXIBLE) {
list_for_each_entry(event, &ctx->flexible_groups, group_entry)
group_sched_out(event, cpuctx, ctx);
+ }
out_enable:
perf_enable();
- out:
+out:
raw_spin_unlock(&ctx->lock);
}
@@ -1323,9 +1327,10 @@ ctx_flexible_sched_in(struct perf_event_
if (event->cpu != -1 && event->cpu != smp_processor_id())
continue;
- if (group_can_go_on(event, cpuctx, can_add_hw))
+ if (group_can_go_on(event, cpuctx, can_add_hw)) {
if (group_sched_in(event, cpuctx, ctx))
can_add_hw = 0;
+ }
}
}
@@ -1355,7 +1360,7 @@ ctx_sched_in(struct perf_event_context *
ctx_flexible_sched_in(ctx, cpuctx);
perf_enable();
- out:
+out:
raw_spin_unlock(&ctx->lock);
}
@@ -1696,7 +1701,7 @@ static void perf_event_enable_on_exec(st
raw_spin_unlock(&ctx->lock);
perf_event_task_sched_in(task);
- out:
+out:
local_irq_restore(flags);
}
@@ -1825,7 +1830,7 @@ static struct perf_event_context *find_g
if (!ptrace_may_access(task, PTRACE_MODE_READ))
goto errout;
- retry:
+retry:
ctx = perf_lock_task_context(task, &flags);
if (ctx) {
unclone_ctx(ctx);
@@ -1853,7 +1858,7 @@ static struct perf_event_context *find_g
put_task_struct(task);
return ctx;
- errout:
+errout:
put_task_struct(task);
return ERR_PTR(err);
}
@@ -3044,7 +3049,7 @@ again:
if (handle->wakeup != local_read(&buffer->wakeup))
perf_output_wakeup(handle);
- out:
+out:
preempt_enable();
}
@@ -4352,7 +4357,7 @@ static int swevent_hlist_get_cpu(struct
rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
}
cpuctx->hlist_refcount++;
- exit:
+exit:
mutex_unlock(&cpuctx->hlist_mutex);
return err;
@@ -4377,7 +4382,7 @@ static int swevent_hlist_get(struct perf
put_online_cpus();
return 0;
- fail:
+fail:
for_each_possible_cpu(cpu) {
if (cpu == failed_cpu)
break;
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 05/19] perf: Reduce perf_disable() usage
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (3 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 04/19] perf: Unindent labels Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 06/19] perf: Per PMU disable Peter Zijlstra
` (14 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-less-disable-1.patch --]
[-- Type: text/plain, Size: 12843 bytes --]
Since the current perf_disable() usage is only an optimization, remove
it for now. This eases the removal of the weak hw_perf_enable
interface.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/arm/kernel/perf_event.c | 3 ++
arch/powerpc/kernel/perf_event.c | 3 ++
arch/powerpc/kernel/perf_event_fsl_emb.c | 8 +++++-
arch/sh/kernel/perf_event.c | 11 ++++++---
arch/sparc/kernel/perf_event.c | 3 ++
arch/x86/kernel/cpu/perf_event.c | 22 +++++++++++-------
include/linux/perf_event.h | 20 ++++++++--------
kernel/perf_event.c | 37 -------------------------------
8 files changed, 48 insertions(+), 59 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -564,26 +564,26 @@ struct pmu {
struct list_head entry;
/*
- * Should return -ENOENT when the @event doesn't match this pmu
+ * Should return -ENOENT when the @event doesn't match this PMU.
*/
int (*event_init) (struct perf_event *event);
- int (*enable) (struct perf_event *event);
+ int (*enable) (struct perf_event *event);
void (*disable) (struct perf_event *event);
- int (*start) (struct perf_event *event);
+ int (*start) (struct perf_event *event);
void (*stop) (struct perf_event *event);
void (*read) (struct perf_event *event);
void (*unthrottle) (struct perf_event *event);
/*
- * Group events scheduling is treated as a transaction, add group
- * events as a whole and perform one schedulability test. If the test
- * fails, roll back the whole group
+ * Group events scheduling is treated as a transaction, add
+ * group events as a whole and perform one schedulability test.
+ * If the test fails, roll back the whole group
*/
/*
- * Start the transaction, after this ->enable() doesn't need
- * to do schedulability tests.
+ * Start the transaction, after this ->enable() doesn't need to
+ * do schedulability tests.
*/
void (*start_txn) (struct pmu *pmu);
/*
@@ -594,8 +594,8 @@ struct pmu {
*/
int (*commit_txn) (struct pmu *pmu);
/*
- * Will cancel the transaction, assumes ->disable() is called for
- * each successfull ->enable() during the transaction.
+ * Will cancel the transaction, assumes ->disable() is called
+ * for each successfull ->enable() during the transaction.
*/
void (*cancel_txn) (struct pmu *pmu);
};
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -460,11 +460,6 @@ static void __perf_event_remove_from_con
return;
raw_spin_lock(&ctx->lock);
- /*
- * Protect the list operation against NMI by disabling the
- * events on a global level.
- */
- perf_disable();
event_sched_out(event, cpuctx, ctx);
@@ -480,7 +475,6 @@ static void __perf_event_remove_from_con
perf_max_events - perf_reserved_percpu);
}
- perf_enable();
raw_spin_unlock(&ctx->lock);
}
@@ -785,12 +779,6 @@ static void __perf_install_in_context(vo
ctx->is_active = 1;
update_context_time(ctx);
- /*
- * Protect the list operation against NMI by disabling the
- * events on a global level. NOP for non NMI based events.
- */
- perf_disable();
-
add_event_to_ctx(event, ctx);
if (event->cpu != -1 && event->cpu != smp_processor_id())
@@ -832,8 +820,6 @@ static void __perf_install_in_context(vo
cpuctx->max_pertask--;
unlock:
- perf_enable();
-
raw_spin_unlock(&ctx->lock);
}
@@ -954,12 +940,10 @@ static void __perf_event_enable(void *in
if (!group_can_go_on(event, cpuctx, 1)) {
err = -EEXIST;
} else {
- perf_disable();
if (event == leader)
err = group_sched_in(event, cpuctx, ctx);
else
err = event_sched_in(event, cpuctx, ctx);
- perf_enable();
}
if (err) {
@@ -1072,9 +1056,8 @@ static void ctx_sched_out(struct perf_ev
goto out;
update_context_time(ctx);
- perf_disable();
if (!ctx->nr_active)
- goto out_enable;
+ goto out;
if (event_type & EVENT_PINNED) {
list_for_each_entry(event, &ctx->pinned_groups, group_entry)
@@ -1085,9 +1068,6 @@ static void ctx_sched_out(struct perf_ev
list_for_each_entry(event, &ctx->flexible_groups, group_entry)
group_sched_out(event, cpuctx, ctx);
}
-
- out_enable:
- perf_enable();
out:
raw_spin_unlock(&ctx->lock);
}
@@ -1346,8 +1326,6 @@ ctx_sched_in(struct perf_event_context *
ctx->timestamp = perf_clock();
- perf_disable();
-
/*
* First go through the list and put on any pinned groups
* in order to give them the best chance of going on.
@@ -1359,7 +1337,6 @@ ctx_sched_in(struct perf_event_context *
if (event_type & EVENT_FLEXIBLE)
ctx_flexible_sched_in(ctx, cpuctx);
- perf_enable();
out:
raw_spin_unlock(&ctx->lock);
}
@@ -1407,8 +1384,6 @@ void perf_event_task_sched_in(struct tas
if (cpuctx->task_ctx == ctx)
return;
- perf_disable();
-
/*
* We want to keep the following priority order:
* cpu pinned (that don't need to move), task pinned,
@@ -1421,8 +1396,6 @@ void perf_event_task_sched_in(struct tas
ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
cpuctx->task_ctx = ctx;
-
- perf_enable();
}
#define MAX_INTERRUPTS (~0ULL)
@@ -1537,11 +1510,9 @@ static void perf_adjust_period(struct pe
hwc->sample_period = sample_period;
if (local64_read(&hwc->period_left) > 8*sample_period) {
- perf_disable();
perf_event_stop(event);
local64_set(&hwc->period_left, 0);
perf_event_start(event);
- perf_enable();
}
}
@@ -1570,15 +1541,12 @@ static void perf_ctx_adjust_freq(struct
*/
if (interrupts == MAX_INTERRUPTS) {
perf_log_throttle(event, 1);
- perf_disable();
event->pmu->unthrottle(event);
- perf_enable();
}
if (!event->attr.freq || !event->attr.sample_freq)
continue;
- perf_disable();
event->pmu->read(event);
now = local64_read(&event->count);
delta = now - hwc->freq_count_stamp;
@@ -1586,7 +1554,6 @@ static void perf_ctx_adjust_freq(struct
if (delta > 0)
perf_adjust_period(event, TICK_NSEC, delta);
- perf_enable();
}
raw_spin_unlock(&ctx->lock);
}
@@ -1629,7 +1596,6 @@ void perf_event_task_tick(struct task_st
if (!rotate)
return;
- perf_disable();
cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
if (ctx)
task_ctx_sched_out(ctx, EVENT_FLEXIBLE);
@@ -1641,7 +1607,6 @@ void perf_event_task_tick(struct task_st
cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
if (ctx)
task_ctx_sched_in(curr, EVENT_FLEXIBLE);
- perf_enable();
}
static int event_enable_on_exec(struct perf_event *event,
Index: linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -262,7 +262,7 @@ static int collect_events(struct perf_ev
return n;
}
-/* perf must be disabled, context locked on entry */
+/* context locked on entry */
static int fsl_emb_pmu_enable(struct perf_event *event)
{
struct cpu_hw_events *cpuhw;
@@ -271,6 +271,7 @@ static int fsl_emb_pmu_enable(struct per
u64 val;
int i;
+ perf_disable();
cpuhw = &get_cpu_var(cpu_hw_events);
if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
@@ -310,15 +311,17 @@ static int fsl_emb_pmu_enable(struct per
ret = 0;
out:
put_cpu_var(cpu_hw_events);
+ perf_enable();
return ret;
}
-/* perf must be disabled, context locked on entry */
+/* context locked on entry */
static void fsl_emb_pmu_disable(struct perf_event *event)
{
struct cpu_hw_events *cpuhw;
int i = event->hw.idx;
+ perf_disable();
if (i < 0)
goto out;
@@ -346,6 +349,7 @@ static void fsl_emb_pmu_disable(struct p
cpuhw->n_events--;
out:
+ perf_enable();
put_cpu_var(cpu_hw_events);
}
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -969,10 +969,11 @@ static int x86_pmu_enable(struct perf_ev
hwc = &event->hw;
+ perf_disable();
n0 = cpuc->n_events;
- n = collect_events(cpuc, event, false);
- if (n < 0)
- return n;
+ ret = n = collect_events(cpuc, event, false);
+ if (ret < 0)
+ goto out;
/*
* If group events scheduling transaction was started,
@@ -980,23 +981,26 @@ static int x86_pmu_enable(struct perf_ev
* at commit time(->commit_txn) as a whole
*/
if (cpuc->group_flag & PERF_EVENT_TXN)
- goto out;
+ goto done_collect;
ret = x86_pmu.schedule_events(cpuc, n, assign);
if (ret)
- return ret;
+ goto out;
/*
* copy new assignment, now we know it is possible
* will be used by hw_perf_enable()
*/
memcpy(cpuc->assign, assign, n*sizeof(int));
-out:
+done_collect:
cpuc->n_events = n;
cpuc->n_added += n - n0;
cpuc->n_txn += n - n0;
- return 0;
+ ret = 0;
+out:
+ perf_enable();
+ return ret;
}
static int x86_pmu_start(struct perf_event *event)
@@ -1399,6 +1403,7 @@ static void x86_pmu_start_txn(struct pmu
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ perf_disable();
cpuc->group_flag |= PERF_EVENT_TXN;
cpuc->n_txn = 0;
}
@@ -1418,6 +1423,7 @@ static void x86_pmu_cancel_txn(struct pm
*/
cpuc->n_added -= cpuc->n_txn;
cpuc->n_events -= cpuc->n_txn;
+ perf_enable();
}
/*
@@ -1447,7 +1453,7 @@ static int x86_pmu_commit_txn(struct pmu
memcpy(cpuc->assign, assign, n*sizeof(int));
cpuc->group_flag &= ~PERF_EVENT_TXN;
-
+ perf_enable();
return 0;
}
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event.c
+++ linux-2.6/arch/powerpc/kernel/perf_event.c
@@ -861,6 +861,7 @@ void power_pmu_start_txn(struct pmu *pmu
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ perf_disable();
cpuhw->group_flag |= PERF_EVENT_TXN;
cpuhw->n_txn_start = cpuhw->n_events;
}
@@ -875,6 +876,7 @@ void power_pmu_cancel_txn(struct pmu *pm
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN;
+ perf_enable();
}
/*
@@ -901,6 +903,7 @@ int power_pmu_commit_txn(struct pmu *pmu
cpuhw->event[i]->hw.config = cpuhw->events[i];
cpuhw->group_flag &= ~PERF_EVENT_TXN;
+ perf_enable();
return 0;
}
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -1113,6 +1113,7 @@ static void sparc_pmu_start_txn(struct p
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+ perf_disable();
cpuhw->group_flag |= PERF_EVENT_TXN;
}
@@ -1126,6 +1127,7 @@ static void sparc_pmu_cancel_txn(struct
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN;
+ perf_enable();
}
/*
@@ -1149,6 +1151,7 @@ static int sparc_pmu_commit_txn(struct p
return -EAGAIN;
cpuc->group_flag &= ~PERF_EVENT_TXN;
+ perf_enable();
return 0;
}
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -277,6 +277,8 @@ armpmu_enable(struct perf_event *event)
int idx;
int err = 0;
+ perf_disable();
+
/* If we don't have a space for the counter then finish early. */
idx = armpmu->get_event_idx(cpuc, hwc);
if (idx < 0) {
@@ -303,6 +305,7 @@ armpmu_enable(struct perf_event *event)
perf_event_update_userpage(event);
out:
+ perf_enable();
return err;
}
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sh/kernel/perf_event.c
+++ linux-2.6/arch/sh/kernel/perf_event.c
@@ -230,11 +230,14 @@ static int sh_pmu_enable(struct perf_eve
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
+ int ret = -EAGAIN;
+
+ perf_disable();
if (test_and_set_bit(idx, cpuc->used_mask)) {
idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events);
if (idx == sh_pmu->num_events)
- return -EAGAIN;
+ goto out;
set_bit(idx, cpuc->used_mask);
hwc->idx = idx;
@@ -248,8 +251,10 @@ static int sh_pmu_enable(struct perf_eve
sh_pmu->enable(hwc, idx);
perf_event_update_userpage(event);
-
- return 0;
+ ret = 0;
+out:
+ perf_enable();
+ return ret;
}
static void sh_pmu_read(struct perf_event *event)
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 06/19] perf: Per PMU disable
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (4 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 05/19] perf: Reduce perf_disable() usage Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 07/19] perf: Default PMU ops Peter Zijlstra
` (13 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-disable.patch --]
[-- Type: text/plain, Size: 19123 bytes --]
Changes perf_disable() into perf_pmu_disable().
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/alpha/kernel/perf_event.c | 30 +++++++++++++-----------
arch/arm/kernel/perf_event.c | 28 +++++++++++-----------
arch/powerpc/kernel/perf_event.c | 24 ++++++++++---------
arch/powerpc/kernel/perf_event_fsl_emb.c | 18 ++++++++------
arch/sh/kernel/perf_event.c | 38 ++++++++++++++++---------------
arch/sparc/kernel/perf_event.c | 20 ++++++++--------
arch/x86/kernel/cpu/perf_event.c | 16 +++++++------
include/linux/perf_event.h | 13 +++++-----
kernel/perf_event.c | 31 +++++++++++++++----------
9 files changed, 119 insertions(+), 99 deletions(-)
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -277,7 +277,7 @@ armpmu_enable(struct perf_event *event)
int idx;
int err = 0;
- perf_disable();
+ perf_pmu_disable(event->pmu);
/* If we don't have a space for the counter then finish early. */
idx = armpmu->get_event_idx(cpuc, hwc);
@@ -305,7 +305,7 @@ armpmu_enable(struct perf_event *event)
perf_event_update_userpage(event);
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
return err;
}
@@ -534,16 +534,7 @@ static int armpmu_event_init(struct perf
return err;
}
-static struct pmu pmu = {
- .event_init = armpmu_event_init,
- .enable = armpmu_enable,
- .disable = armpmu_disable,
- .unthrottle = armpmu_unthrottle,
- .read = armpmu_read,
-};
-
-void
-hw_perf_enable(void)
+static void armpmu_pmu_enable(struct pmu *pmu)
{
/* Enable all of the perf events on hardware. */
int idx;
@@ -564,13 +555,22 @@ hw_perf_enable(void)
armpmu->start();
}
-void
-hw_perf_disable(void)
+static void armpmu_pmu_disable(struct pmu *pmu)
{
if (armpmu)
armpmu->stop();
}
+static struct pmu pmu = {
+ .pmu_enable = armpmu_pmu_enable,
+ .pmu_disable= armpmu_pmu_disable,
+ .event_init = armpmu_event_init,
+ .enable = armpmu_enable,
+ .disable = armpmu_disable,
+ .unthrottle = armpmu_unthrottle,
+ .read = armpmu_read,
+};
+
/*
* ARMv6 Performance counter handling code.
*
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event.c
+++ linux-2.6/arch/powerpc/kernel/perf_event.c
@@ -517,7 +517,7 @@ static void write_mmcr0(struct cpu_hw_ev
* Disable all events to prevent PMU interrupts and to allow
* events to be added or removed.
*/
-void hw_perf_disable(void)
+static void power_pmu_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -565,7 +565,7 @@ void hw_perf_disable(void)
* If we were previously disabled and events were added, then
* put the new config on the PMU.
*/
-void hw_perf_enable(void)
+static void power_pmu_pmu_enable(struct pmu *pmu)
{
struct perf_event *event;
struct cpu_hw_events *cpuhw;
@@ -735,7 +735,7 @@ static int power_pmu_enable(struct perf_
int ret = -EAGAIN;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
/*
* Add the event to the list (if there is room)
@@ -769,7 +769,7 @@ static int power_pmu_enable(struct perf_
ret = 0;
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
return ret;
}
@@ -784,7 +784,7 @@ static void power_pmu_disable(struct per
unsigned long flags;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
power_pmu_read(event);
@@ -821,7 +821,7 @@ static void power_pmu_disable(struct per
cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
}
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
}
@@ -837,7 +837,7 @@ static void power_pmu_unthrottle(struct
if (!event->hw.idx || !event->hw.sample_period)
return;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
power_pmu_read(event);
left = event->hw.sample_period;
event->hw.last_period = left;
@@ -848,7 +848,7 @@ static void power_pmu_unthrottle(struct
local64_set(&event->hw.prev_count, val);
local64_set(&event->hw.period_left, left);
perf_event_update_userpage(event);
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
}
@@ -861,7 +861,7 @@ void power_pmu_start_txn(struct pmu *pmu
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
- perf_disable();
+ perf_pmu_disable(pmu);
cpuhw->group_flag |= PERF_EVENT_TXN;
cpuhw->n_txn_start = cpuhw->n_events;
}
@@ -876,7 +876,7 @@ void power_pmu_cancel_txn(struct pmu *pm
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN;
- perf_enable();
+ perf_pmu_enable(pmu);
}
/*
@@ -903,7 +903,7 @@ int power_pmu_commit_txn(struct pmu *pmu
cpuhw->event[i]->hw.config = cpuhw->events[i];
cpuhw->group_flag &= ~PERF_EVENT_TXN;
- perf_enable();
+ perf_pmu_enable(pmu);
return 0;
}
@@ -1131,6 +1131,8 @@ static int power_pmu_event_init(struct p
}
struct pmu power_pmu = {
+ .pmu_enable = power_pmu_pmu_enable,
+ .pmu_disable = power_pmu_pmu_disable,
.event_init = power_pmu_event_init,
.enable = power_pmu_enable,
.disable = power_pmu_disable,
Index: linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -177,7 +177,7 @@ static void fsl_emb_pmu_read(struct perf
* Disable all events to prevent PMU interrupts and to allow
* events to be added or removed.
*/
-void hw_perf_disable(void)
+static void fsl_emb_pmu_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -216,7 +216,7 @@ void hw_perf_disable(void)
* If we were previously disabled and events were added, then
* put the new config on the PMU.
*/
-void hw_perf_enable(void)
+static void fsl_emb_pmu_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -271,7 +271,7 @@ static int fsl_emb_pmu_enable(struct per
u64 val;
int i;
- perf_disable();
+ perf_pmu_disable(event->pmu);
cpuhw = &get_cpu_var(cpu_hw_events);
if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
@@ -311,7 +311,7 @@ static int fsl_emb_pmu_enable(struct per
ret = 0;
out:
put_cpu_var(cpu_hw_events);
- perf_enable();
+ perf_pmu_enable(event->pmu);
return ret;
}
@@ -321,7 +321,7 @@ static void fsl_emb_pmu_disable(struct p
struct cpu_hw_events *cpuhw;
int i = event->hw.idx;
- perf_disable();
+ perf_pmu_disable(event->pmu);
if (i < 0)
goto out;
@@ -349,7 +349,7 @@ static void fsl_emb_pmu_disable(struct p
cpuhw->n_events--;
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
put_cpu_var(cpu_hw_events);
}
@@ -367,7 +367,7 @@ static void fsl_emb_pmu_unthrottle(struc
if (event->hw.idx < 0 || !event->hw.sample_period)
return;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
fsl_emb_pmu_read(event);
left = event->hw.sample_period;
event->hw.last_period = left;
@@ -378,7 +378,7 @@ static void fsl_emb_pmu_unthrottle(struc
local64_set(&event->hw.prev_count, val);
local64_set(&event->hw.period_left, left);
perf_event_update_userpage(event);
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
}
@@ -524,6 +524,8 @@ static int fsl_emb_pmu_event_init(struct
}
static struct pmu fsl_emb_pmu = {
+ .pmu_enable = fsl_emb_pmu_pmu_enable,
+ .pmu_disable = fsl_emb_pmu_pmu_disable,
.event_init = fsl_emb_pmu_event_init,
.enable = fsl_emb_pmu_enable,
.disable = fsl_emb_pmu_disable,
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sh/kernel/perf_event.c
+++ linux-2.6/arch/sh/kernel/perf_event.c
@@ -232,7 +232,7 @@ static int sh_pmu_enable(struct perf_eve
int idx = hwc->idx;
int ret = -EAGAIN;
- perf_disable();
+ perf_pmu_disable(event->pmu);
if (test_and_set_bit(idx, cpuc->used_mask)) {
idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events);
@@ -253,7 +253,7 @@ static int sh_pmu_enable(struct perf_eve
perf_event_update_userpage(event);
ret = 0;
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
return ret;
}
@@ -285,7 +285,25 @@ static int sh_pmu_event_init(struct perf
return err;
}
+static void sh_pmu_pmu_enable(struct pmu *pmu)
+{
+ if (!sh_pmu_initialized())
+ return;
+
+ sh_pmu->enable_all();
+}
+
+static void sh_pmu_pmu_disable(struct pmu *pmu)
+{
+ if (!sh_pmu_initialized())
+ return;
+
+ sh_pmu->disable_all();
+}
+
static struct pmu pmu = {
+ .pmu_enable = sh_pmu_pmu_enable,
+ .pmu_disable = sh_pmu_pmu_disable,
.event_init = sh_pmu_event_init,
.enable = sh_pmu_enable,
.disable = sh_pmu_disable,
@@ -316,22 +334,6 @@ sh_pmu_notifier(struct notifier_block *s
return NOTIFY_OK;
}
-void hw_perf_enable(void)
-{
- if (!sh_pmu_initialized())
- return;
-
- sh_pmu->enable_all();
-}
-
-void hw_perf_disable(void)
-{
- if (!sh_pmu_initialized())
- return;
-
- sh_pmu->disable_all();
-}
-
int __cpuinit register_sh_pmu(struct sh_pmu *pmu)
{
if (sh_pmu)
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -664,7 +664,7 @@ static u64 maybe_change_configuration(st
return pcr;
}
-void hw_perf_enable(void)
+static void sparc_pmu_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
u64 pcr;
@@ -691,7 +691,7 @@ void hw_perf_enable(void)
pcr_ops->write(cpuc->pcr);
}
-void hw_perf_disable(void)
+static void sparc_pmu_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
u64 val;
@@ -718,7 +718,7 @@ static void sparc_pmu_disable(struct per
int i;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
for (i = 0; i < cpuc->n_events; i++) {
if (event == cpuc->event[i]) {
@@ -748,7 +748,7 @@ static void sparc_pmu_disable(struct per
}
}
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
}
@@ -991,7 +991,7 @@ static int sparc_pmu_enable(struct perf_
unsigned long flags;
local_irq_save(flags);
- perf_disable();
+ perf_pmu_disable(event->pmu);
n0 = cpuc->n_events;
if (n0 >= perf_max_events)
@@ -1020,7 +1020,7 @@ static int sparc_pmu_enable(struct perf_
ret = 0;
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
local_irq_restore(flags);
return ret;
}
@@ -1113,7 +1113,7 @@ static void sparc_pmu_start_txn(struct p
{
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
- perf_disable();
+ perf_pmu_disable(pmu);
cpuhw->group_flag |= PERF_EVENT_TXN;
}
@@ -1127,7 +1127,7 @@ static void sparc_pmu_cancel_txn(struct
struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
cpuhw->group_flag &= ~PERF_EVENT_TXN;
- perf_enable();
+ perf_pmu_enable(pmu);
}
/*
@@ -1151,11 +1151,13 @@ static int sparc_pmu_commit_txn(struct p
return -EAGAIN;
cpuc->group_flag &= ~PERF_EVENT_TXN;
- perf_enable();
+ perf_pmu_enable(pmu);
return 0;
}
static struct pmu pmu = {
+ .pmu_enable = sparc_pmu_pmu_enable,
+ .pmu_disable = sparc_pmu_pmu_disable,
.event_init = sparc_pmu_event_init,
.enable = sparc_pmu_enable,
.disable = sparc_pmu_disable,
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -582,7 +582,7 @@ static void x86_pmu_disable_all(void)
}
}
-void hw_perf_disable(void)
+static void x86_pmu_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -802,7 +802,7 @@ static inline int match_prev_assignment(
static int x86_pmu_start(struct perf_event *event);
static void x86_pmu_stop(struct perf_event *event);
-void hw_perf_enable(void)
+static void x86_pmu_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct perf_event *event;
@@ -968,7 +968,7 @@ static int x86_pmu_enable(struct perf_ev
hwc = &event->hw;
- perf_disable();
+ perf_pmu_disable(event->pmu);
n0 = cpuc->n_events;
ret = n = collect_events(cpuc, event, false);
if (ret < 0)
@@ -998,7 +998,7 @@ static int x86_pmu_enable(struct perf_ev
ret = 0;
out:
- perf_enable();
+ perf_pmu_enable(event->pmu);
return ret;
}
@@ -1435,7 +1435,7 @@ static void x86_pmu_start_txn(struct pmu
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
- perf_disable();
+ perf_pmu_disable(pmu);
cpuc->group_flag |= PERF_EVENT_TXN;
cpuc->n_txn = 0;
}
@@ -1455,7 +1455,7 @@ static void x86_pmu_cancel_txn(struct pm
*/
cpuc->n_added -= cpuc->n_txn;
cpuc->n_events -= cpuc->n_txn;
- perf_enable();
+ perf_pmu_enable(pmu);
}
/*
@@ -1485,7 +1485,7 @@ static int x86_pmu_commit_txn(struct pmu
memcpy(cpuc->assign, assign, n*sizeof(int));
cpuc->group_flag &= ~PERF_EVENT_TXN;
- perf_enable();
+ perf_pmu_enable(pmu);
return 0;
}
@@ -1604,6 +1604,8 @@ int x86_pmu_event_init(struct perf_event
}
static struct pmu pmu = {
+ .pmu_enable = x86_pmu_pmu_enable,
+ .pmu_disable = x86_pmu_pmu_disable,
.event_init = x86_pmu_event_init,
.enable = x86_pmu_enable,
.disable = x86_pmu_disable,
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -563,6 +563,11 @@ struct perf_event;
struct pmu {
struct list_head entry;
+ int *pmu_disable_count;
+
+ void (*pmu_enable) (struct pmu *pmu);
+ void (*pmu_disable) (struct pmu *pmu);
+
/*
* Should return -ENOENT when the @event doesn't match this PMU.
*/
@@ -868,10 +873,8 @@ extern void perf_event_free_task(struct
extern void set_perf_event_pending(void);
extern void perf_event_do_pending(void);
extern void perf_event_print_debug(void);
-extern void __perf_disable(void);
-extern bool __perf_enable(void);
-extern void perf_disable(void);
-extern void perf_enable(void);
+extern void perf_pmu_disable(struct pmu *pmu);
+extern void perf_pmu_enable(struct pmu *pmu);
extern int perf_event_task_disable(void);
extern int perf_event_task_enable(void);
extern void perf_event_update_userpage(struct perf_event *event);
@@ -1056,8 +1059,6 @@ static inline void perf_event_exit_task(
static inline void perf_event_free_task(struct task_struct *task) { }
static inline void perf_event_do_pending(void) { }
static inline void perf_event_print_debug(void) { }
-static inline void perf_disable(void) { }
-static inline void perf_enable(void) { }
static inline int perf_event_task_disable(void) { return -EINVAL; }
static inline int perf_event_task_enable(void) { return -EINVAL; }
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -71,23 +71,20 @@ static atomic64_t perf_event_id;
*/
static DEFINE_SPINLOCK(perf_resource_lock);
-void __weak hw_perf_disable(void) { barrier(); }
-void __weak hw_perf_enable(void) { barrier(); }
-
void __weak perf_event_print_debug(void) { }
-static DEFINE_PER_CPU(int, perf_disable_count);
-
-void perf_disable(void)
+void perf_pmu_disable(struct pmu *pmu)
{
- if (!__get_cpu_var(perf_disable_count)++)
- hw_perf_disable();
+ int *count = this_cpu_ptr(pmu->pmu_disable_count);
+ if (!(*count)++)
+ pmu->pmu_disable(pmu);
}
-void perf_enable(void)
+void perf_pmu_enable(struct pmu *pmu)
{
- if (!--__get_cpu_var(perf_disable_count))
- hw_perf_enable();
+ int *count = this_cpu_ptr(pmu->pmu_disable_count);
+ if (!--(*count))
+ pmu->pmu_enable(pmu);
}
static void get_ctx(struct perf_event_context *ctx)
@@ -4970,11 +4967,19 @@ static struct srcu_struct pmus_srcu;
int perf_pmu_register(struct pmu *pmu)
{
+ int ret;
+
mutex_lock(&pmus_lock);
+ ret = -ENOMEM;
+ pmu->pmu_disable_count = alloc_percpu(int);
+ if (!pmu->pmu_disable_count)
+ goto unlock;
list_add_rcu(&pmu->entry, &pmus);
+ ret = 0;
+unlock:
mutex_unlock(&pmus_lock);
- return 0;
+ return ret;
}
void perf_pmu_unregister(struct pmu *pmu)
@@ -4984,6 +4989,8 @@ void perf_pmu_unregister(struct pmu *pmu
mutex_unlock(&pmus_lock);
synchronize_srcu(&pmus_srcu);
+
+ free_percpu(pmu->pmu_disable_count);
}
struct pmu *perf_init_event(struct perf_event *event)
Index: linux-2.6/arch/alpha/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/perf_event.c
+++ linux-2.6/arch/alpha/kernel/perf_event.c
@@ -435,7 +435,7 @@ static int alpha_pmu_enable(struct perf_
* nevertheless we disable the PMCs first to enable a potential
* final PMI to occur before we disable interrupts.
*/
- perf_disable();
+ perf_pmu_disable(event->pmu);
local_irq_save(flags);
/* Default to error to be returned */
@@ -456,7 +456,7 @@ static int alpha_pmu_enable(struct perf_
}
local_irq_restore(flags);
- perf_enable();
+ perf_pmu_enable(event->pmu);
return ret;
}
@@ -474,7 +474,7 @@ static void alpha_pmu_disable(struct per
unsigned long flags;
int j;
- perf_disable();
+ perf_pmu_disable(event->pmu);
local_irq_save(flags);
for (j = 0; j < cpuc->n_events; j++) {
@@ -502,7 +502,7 @@ static void alpha_pmu_disable(struct per
}
local_irq_restore(flags);
- perf_enable();
+ perf_pmu_enable(event->pmu);
}
@@ -668,18 +668,10 @@ static int alpha_pmu_event_init(struct p
return err;
}
-static struct pmu pmu = {
- .event_init = alpha_pmu_event_init,
- .enable = alpha_pmu_enable,
- .disable = alpha_pmu_disable,
- .read = alpha_pmu_read,
- .unthrottle = alpha_pmu_unthrottle,
-};
-
/*
* Main entry point - enable HW performance counters.
*/
-void hw_perf_enable(void)
+static void alpha_pmu_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -705,7 +697,7 @@ void hw_perf_enable(void)
* Main entry point - disable HW performance counters.
*/
-void hw_perf_disable(void)
+static void alpha_pmu_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -718,6 +710,16 @@ void hw_perf_disable(void)
wrperfmon(PERFMON_CMD_DISABLE, cpuc->idx_mask);
}
+static struct pmu pmu = {
+ .pmu_enable = alpha_pmu_pmu_enable,
+ .pmu_disable = alpha_pmu_pmu_disable,
+ .event_init = alpha_pmu_event_init,
+ .enable = alpha_pmu_enable,
+ .disable = alpha_pmu_disable,
+ .read = alpha_pmu_read,
+ .unthrottle = alpha_pmu_unthrottle,
+};
+
/*
* Main entry point - don't know when this is called but it
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 07/19] perf: Default PMU ops
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (5 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 06/19] perf: Per PMU disable Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 08/19] perf: Shrink hw_perf_event Peter Zijlstra
` (12 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-default-ops.patch --]
[-- Type: text/plain, Size: 4086 bytes --]
Provide default implementations for the pmu txn methods, this allows
us to remove some conditional code.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 10 +++----
kernel/perf_event.c | 64 ++++++++++++++++++++++++++++++++++++---------
2 files changed, 57 insertions(+), 17 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -656,21 +656,14 @@ group_sched_in(struct perf_event *group_
{
struct perf_event *event, *partial_group = NULL;
struct pmu *pmu = group_event->pmu;
- bool txn = false;
if (group_event->state == PERF_EVENT_STATE_OFF)
return 0;
- /* Check if group transaction availabe */
- if (pmu->start_txn)
- txn = true;
-
- if (txn)
- pmu->start_txn(pmu);
+ pmu->start_txn(pmu);
if (event_sched_in(group_event, cpuctx, ctx)) {
- if (txn)
- pmu->cancel_txn(pmu);
+ pmu->cancel_txn(pmu);
return -EAGAIN;
}
@@ -684,7 +677,7 @@ group_sched_in(struct perf_event *group_
}
}
- if (!txn || !pmu->commit_txn(pmu))
+ if (!pmu->commit_txn(pmu))
return 0;
group_error:
@@ -699,8 +692,7 @@ group_error:
}
event_sched_out(group_event, cpuctx, ctx);
- if (txn)
- pmu->cancel_txn(pmu);
+ pmu->cancel_txn(pmu);
return -EAGAIN;
}
@@ -4755,6 +4747,31 @@ static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
static struct srcu_struct pmus_srcu;
+static void perf_pmu_nop_void(struct pmu *pmu)
+{
+}
+
+static int perf_pmu_nop_int(struct pmu *pmu)
+{
+ return 0;
+}
+
+static void perf_pmu_start_txn(struct pmu *pmu)
+{
+ perf_pmu_disable(pmu);
+}
+
+static int perf_pmu_commit_txn(struct pmu *pmu)
+{
+ perf_pmu_enable(pmu);
+ return 0;
+}
+
+static void perf_pmu_cancel_txn(struct pmu *pmu)
+{
+ perf_pmu_enable(pmu);
+}
+
int perf_pmu_register(struct pmu *pmu)
{
int ret;
@@ -4764,6 +4781,29 @@ int perf_pmu_register(struct pmu *pmu)
pmu->pmu_disable_count = alloc_percpu(int);
if (!pmu->pmu_disable_count)
goto unlock;
+
+ if (!pmu->start_txn) {
+ if (pmu->pmu_enable) {
+ /*
+ * If we have pmu_enable/pmu_disable calls, install
+ * transaction stubs that use that to try and batch
+ * hardware accesses.
+ */
+ pmu->start_txn = perf_pmu_start_txn;
+ pmu->commit_txn = perf_pmu_commit_txn;
+ pmu->cancel_txn = perf_pmu_cancel_txn;
+ } else {
+ pmu->start_txn = perf_pmu_nop_void;
+ pmu->commit_txn = perf_pmu_nop_int;
+ pmu->cancel_txn = perf_pmu_nop_void;
+ }
+ }
+
+ if (!pmu->pmu_enable) {
+ pmu->pmu_enable = perf_pmu_nop_void;
+ pmu->pmu_disable = perf_pmu_nop_void;
+ }
+
list_add_rcu(&pmu->entry, &pmus);
ret = 0;
unlock:
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -565,8 +565,8 @@ struct pmu {
int *pmu_disable_count;
- void (*pmu_enable) (struct pmu *pmu);
- void (*pmu_disable) (struct pmu *pmu);
+ void (*pmu_enable) (struct pmu *pmu); /* optional */
+ void (*pmu_disable) (struct pmu *pmu); /* optional */
/*
* Should return -ENOENT when the @event doesn't match this PMU.
@@ -590,19 +590,19 @@ struct pmu {
* Start the transaction, after this ->enable() doesn't need to
* do schedulability tests.
*/
- void (*start_txn) (struct pmu *pmu);
+ void (*start_txn) (struct pmu *pmu); /* optional */
/*
* If ->start_txn() disabled the ->enable() schedulability test
* then ->commit_txn() is required to perform one. On success
* the transaction is closed. On error the transaction is kept
* open until ->cancel_txn() is called.
*/
- int (*commit_txn) (struct pmu *pmu);
+ int (*commit_txn) (struct pmu *pmu); /* optional */
/*
* Will cancel the transaction, assumes ->disable() is called
* for each successfull ->enable() during the transaction.
*/
- void (*cancel_txn) (struct pmu *pmu);
+ void (*cancel_txn) (struct pmu *pmu); /* optional */
};
/**
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 08/19] perf: Shrink hw_perf_event
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (6 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 07/19] perf: Default PMU ops Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 09/19] perf: Rework the PMU methods Peter Zijlstra
` (11 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-swtimer-period_left.patch --]
[-- Type: text/plain, Size: 1701 bytes --]
Use hw_perf_event::period_left instead of hw_perf_event::remaning and
win back 8 bytes.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 1 -
kernel/perf_event.c | 13 ++++++-------
2 files changed, 6 insertions(+), 8 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -529,7 +529,6 @@ struct hw_perf_event {
int last_cpu;
};
struct { /* software */
- s64 remaining;
struct hrtimer hrtimer;
};
#ifdef CONFIG_HAVE_HW_BREAKPOINT
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -4590,14 +4590,13 @@ static void perf_swevent_start_hrtimer(s
hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hwc->hrtimer.function = perf_swevent_hrtimer;
if (hwc->sample_period) {
- u64 period;
+ s64 period = local64_read(&hwc->period_left);
- if (hwc->remaining) {
- if (hwc->remaining < 0)
+ if (period) {
+ if (period < 0)
period = 10000;
- else
- period = hwc->remaining;
- hwc->remaining = 0;
+
+ local64_set(&hwc->period_left, 0);
} else {
period = max_t(u64, 10000, hwc->sample_period);
}
@@ -4613,7 +4612,7 @@ static void perf_swevent_cancel_hrtimer(
if (hwc->sample_period) {
ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
- hwc->remaining = ktime_to_ns(remaining);
+ local64_set(&hwc->period_left, ktime_to_ns(remaining));
hrtimer_cancel(&hwc->hrtimer);
}
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 09/19] perf: Rework the PMU methods
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (7 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 08/19] perf: Shrink hw_perf_event Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 10/19] perf: Remove the sysfs bits Peter Zijlstra
` (10 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-change-ops.patch --]
[-- Type: text/plain, Size: 55019 bytes --]
Replace pmu::{enable,disable,start,stop,unthrottle} with
pmu::{add,del,start,stop}, all of which take a flags argument.
The new interface extends the capability to stop a counter while
keeping it scheduled on the PMU. We replace the throttled state with
the generic stopped state.
This also allows us to efficiently stop/start counters over certain
code paths (like IRQ handlers).
It also allows scheduling a counter without it starting, allowing for
a generic frozen state (useful for rotating stopped counters).
The stopped state is implemented in two different ways, depending on
how the architecture implemented the throttled state:
1) We disable the counter:
a) the pmu has per-counter enable bits, we flip that
b) we program a NOP event, preserving the counter state
2) We store the counter state and ignore all read/overflow events
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/alpha/kernel/perf_event.c | 71 +++++++++++----
arch/arm/kernel/perf_event.c | 96 +++++++++++++-------
arch/powerpc/kernel/perf_event.c | 105 ++++++++++++++--------
arch/powerpc/kernel/perf_event_fsl_emb.c | 107 ++++++++++++++--------
arch/sh/kernel/perf_event.c | 75 ++++++++++------
arch/sparc/kernel/perf_event.c | 109 ++++++++++++++---------
arch/x86/kernel/cpu/perf_event.c | 106 +++++++++++++---------
arch/x86/kernel/cpu/perf_event_intel.c | 2
arch/x86/kernel/cpu/perf_event_intel_ds.c | 2
include/linux/ftrace_event.h | 4
include/linux/perf_event.h | 54 ++++++++---
kernel/hw_breakpoint.c | 29 +++++-
kernel/perf_event.c | 140 +++++++++++++++---------------
kernel/trace/trace_event_perf.c | 7 +
14 files changed, 576 insertions(+), 331 deletions(-)
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -582,7 +582,7 @@ static void x86_pmu_disable_all(void)
}
}
-static void x86_pmu_pmu_disable(struct pmu *pmu)
+static void x86_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -799,10 +799,10 @@ static inline int match_prev_assignment(
hwc->last_tag == cpuc->tags[i];
}
-static int x86_pmu_start(struct perf_event *event);
-static void x86_pmu_stop(struct perf_event *event);
+static void x86_pmu_start(struct perf_event *event, int flags);
+static void x86_pmu_stop(struct perf_event *event, int flags);
-static void x86_pmu_pmu_enable(struct pmu *pmu)
+static void x86_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct perf_event *event;
@@ -838,7 +838,14 @@ static void x86_pmu_pmu_enable(struct pm
match_prev_assignment(hwc, cpuc, i))
continue;
- x86_pmu_stop(event);
+ /*
+ * Ensure we don't accidentally enable a stopped
+ * counter simply because we rescheduled.
+ */
+ if (hwc->state & PERF_HES_STOPPED)
+ hwc->state |= PERF_HES_ARCH;
+
+ x86_pmu_stop(event, PERF_EF_UPDATE);
}
for (i = 0; i < cpuc->n_events; i++) {
@@ -850,7 +857,10 @@ static void x86_pmu_pmu_enable(struct pm
else if (i < n_running)
continue;
- x86_pmu_start(event);
+ if (hwc->state & PERF_HES_ARCH)
+ continue;
+
+ x86_pmu_start(event, PERF_EF_RELOAD);
}
cpuc->n_added = 0;
perf_events_lapic_init();
@@ -951,15 +961,12 @@ static void x86_pmu_enable_event(struct
}
/*
- * activate a single event
+ * Add a single event to the PMU.
*
* The event is added to the group of enabled events
* but only if it can be scehduled with existing events.
- *
- * Called with PMU disabled. If successful and return value 1,
- * then guaranteed to call perf_enable() and hw_perf_enable()
*/
-static int x86_pmu_enable(struct perf_event *event)
+static int x86_pmu_add(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc;
@@ -974,10 +981,14 @@ static int x86_pmu_enable(struct perf_ev
if (ret < 0)
goto out;
+ hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+ if (!(flags & PERF_EF_START))
+ hwc->state |= PERF_HES_ARCH;
+
/*
* If group events scheduling transaction was started,
* skip the schedulability test here, it will be peformed
- * at commit time(->commit_txn) as a whole
+ * at commit time (->commit_txn) as a whole
*/
if (cpuc->group_flag & PERF_EVENT_TXN)
goto done_collect;
@@ -1002,27 +1013,28 @@ static int x86_pmu_enable(struct perf_ev
return ret;
}
-static int x86_pmu_start(struct perf_event *event)
+static void x86_pmu_start(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int idx = event->hw.idx;
- if (idx == -1)
- return -EAGAIN;
+ if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
+ return;
+
+ if (WARN_ON_ONCE(idx == -1))
+ return;
+
+ if (flags & PERF_EF_RELOAD) {
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+ x86_perf_event_set_period(event);
+ }
+
+ event->hw.state = 0;
- x86_perf_event_set_period(event);
cpuc->events[idx] = event;
__set_bit(idx, cpuc->active_mask);
x86_pmu.enable(event);
perf_event_update_userpage(event);
-
- return 0;
-}
-
-static void x86_pmu_unthrottle(struct perf_event *event)
-{
- int ret = x86_pmu_start(event);
- WARN_ON_ONCE(ret);
}
void perf_event_print_debug(void)
@@ -1079,27 +1091,29 @@ void perf_event_print_debug(void)
local_irq_restore(flags);
}
-static void x86_pmu_stop(struct perf_event *event)
+static void x86_pmu_stop(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
- int idx = hwc->idx;
-
- if (!__test_and_clear_bit(idx, cpuc->active_mask))
- return;
-
- x86_pmu.disable(event);
- /*
- * Drain the remaining delta count out of a event
- * that we are disabling:
- */
- x86_perf_event_update(event);
+ if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
+ x86_pmu.disable(event);
+ cpuc->events[hwc->idx] = NULL;
+ WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+ hwc->state |= PERF_HES_STOPPED;
+ }
- cpuc->events[idx] = NULL;
+ if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
+ /*
+ * Drain the remaining delta count out of a event
+ * that we are disabling:
+ */
+ x86_perf_event_update(event);
+ hwc->state |= PERF_HES_UPTODATE;
+ }
}
-static void x86_pmu_disable(struct perf_event *event)
+static void x86_pmu_del(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int i;
@@ -1112,7 +1126,7 @@ static void x86_pmu_disable(struct perf_
if (cpuc->group_flag & PERF_EVENT_TXN)
return;
- x86_pmu_stop(event);
+ x86_pmu_stop(event, PERF_EF_UPDATE);
for (i = 0; i < cpuc->n_events; i++) {
if (event == cpuc->event_list[i]) {
@@ -1164,7 +1178,7 @@ static int x86_pmu_handle_irq(struct pt_
continue;
if (perf_event_overflow(event, 1, &data, regs))
- x86_pmu_stop(event);
+ x86_pmu_stop(event, 0);
}
if (handled)
@@ -1604,15 +1618,17 @@ int x86_pmu_event_init(struct perf_event
}
static struct pmu pmu = {
- .pmu_enable = x86_pmu_pmu_enable,
- .pmu_disable = x86_pmu_pmu_disable,
+ .pmu_enable = x86_pmu_enable,
+ .pmu_disable = x86_pmu_disable,
+
.event_init = x86_pmu_event_init,
- .enable = x86_pmu_enable,
- .disable = x86_pmu_disable,
+
+ .add = x86_pmu_add,
+ .del = x86_pmu_del,
.start = x86_pmu_start,
.stop = x86_pmu_stop,
.read = x86_pmu_read,
- .unthrottle = x86_pmu_unthrottle,
+
.start_txn = x86_pmu_start_txn,
.cancel_txn = x86_pmu_cancel_txn,
.commit_txn = x86_pmu_commit_txn,
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -424,7 +424,7 @@ event_sched_out(struct perf_event *event
event->state = PERF_EVENT_STATE_OFF;
}
event->tstamp_stopped = ctx->time;
- event->pmu->disable(event);
+ event->pmu->del(event, 0);
event->oncpu = -1;
if (!is_software_event(event))
@@ -649,7 +649,7 @@ event_sched_in(struct perf_event *event,
*/
smp_wmb();
- if (event->pmu->enable(event)) {
+ if (event->pmu->add(event, PERF_EF_START)) {
event->state = PERF_EVENT_STATE_INACTIVE;
event->oncpu = -1;
return -EAGAIN;
@@ -1482,22 +1482,6 @@ do { \
return div64_u64(dividend, divisor);
}
-static void perf_event_stop(struct perf_event *event)
-{
- if (!event->pmu->stop)
- return event->pmu->disable(event);
-
- return event->pmu->stop(event);
-}
-
-static int perf_event_start(struct perf_event *event)
-{
- if (!event->pmu->start)
- return event->pmu->enable(event);
-
- return event->pmu->start(event);
-}
-
static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
{
struct hw_perf_event *hwc = &event->hw;
@@ -1517,9 +1501,9 @@ static void perf_adjust_period(struct pe
hwc->sample_period = sample_period;
if (local64_read(&hwc->period_left) > 8*sample_period) {
- perf_event_stop(event);
+ event->pmu->stop(event, PERF_EF_UPDATE);
local64_set(&hwc->period_left, 0);
- perf_event_start(event);
+ event->pmu->start(event, PERF_EF_RELOAD);
}
}
@@ -1548,7 +1532,7 @@ static void perf_ctx_adjust_freq(struct
*/
if (interrupts == MAX_INTERRUPTS) {
perf_log_throttle(event, 1);
- event->pmu->unthrottle(event);
+ event->pmu->start(event, 0);
}
if (!event->attr.freq || !event->attr.sample_freq)
@@ -2506,6 +2490,9 @@ int perf_event_task_disable(void)
static int perf_event_index(struct perf_event *event)
{
+ if (event->hw.state & PERF_HES_STOPPED)
+ return 0;
+
if (event->state != PERF_EVENT_STATE_ACTIVE)
return 0;
@@ -4120,8 +4107,6 @@ static int __perf_event_overflow(struct
struct hw_perf_event *hwc = &event->hw;
int ret = 0;
- throttle = (throttle && event->pmu->unthrottle != NULL);
-
if (!throttle) {
hwc->interrupts++;
} else {
@@ -4246,7 +4231,7 @@ static void perf_swevent_overflow(struct
}
}
-static void perf_swevent_add(struct perf_event *event, u64 nr,
+static void perf_swevent_event(struct perf_event *event, u64 nr,
int nmi, struct perf_sample_data *data,
struct pt_regs *regs)
{
@@ -4272,6 +4257,9 @@ static void perf_swevent_add(struct perf
static int perf_exclude_event(struct perf_event *event,
struct pt_regs *regs)
{
+ if (event->hw.state & PERF_HES_STOPPED)
+ return 0;
+
if (regs) {
if (event->attr.exclude_user && user_mode(regs))
return 1;
@@ -4371,7 +4359,7 @@ static void do_perf_sw_event(enum perf_t
hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
if (perf_swevent_match(event, type, event_id, data, regs))
- perf_swevent_add(event, nr, nmi, data, regs);
+ perf_swevent_event(event, nr, nmi, data, regs);
}
end:
rcu_read_unlock();
@@ -4415,7 +4403,7 @@ static void perf_swevent_read(struct per
{
}
-static int perf_swevent_enable(struct perf_event *event)
+static int perf_swevent_add(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
struct perf_cpu_context *cpuctx;
@@ -4428,6 +4416,8 @@ static int perf_swevent_enable(struct pe
perf_swevent_set_period(event);
}
+ hwc->state = !(flags & PERF_EF_START);
+
head = find_swevent_head(cpuctx, event);
if (WARN_ON_ONCE(!head))
return -EINVAL;
@@ -4437,18 +4427,19 @@ static int perf_swevent_enable(struct pe
return 0;
}
-static void perf_swevent_disable(struct perf_event *event)
+static void perf_swevent_del(struct perf_event *event, int flags)
{
hlist_del_rcu(&event->hlist_entry);
}
-static void perf_swevent_void(struct perf_event *event)
+static void perf_swevent_start(struct perf_event *event, int flags)
{
+ event->hw.state = 0;
}
-static int perf_swevent_int(struct perf_event *event)
+static void perf_swevent_stop(struct perf_event *event, int flags)
{
- return 0;
+ event->hw.state = PERF_HES_STOPPED;
}
/* Deref the hlist from the update side */
@@ -4604,12 +4595,11 @@ static int perf_swevent_init(struct perf
static struct pmu perf_swevent = {
.event_init = perf_swevent_init,
- .enable = perf_swevent_enable,
- .disable = perf_swevent_disable,
- .start = perf_swevent_int,
- .stop = perf_swevent_void,
+ .add = perf_swevent_add,
+ .del = perf_swevent_del,
+ .start = perf_swevent_start,
+ .stop = perf_swevent_stop,
.read = perf_swevent_read,
- .unthrottle = perf_swevent_void, /* hwc->interrupts already reset */
};
#ifdef CONFIG_EVENT_TRACING
@@ -4657,7 +4647,7 @@ void perf_tp_event(u64 addr, u64 count,
hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
if (perf_tp_event_match(event, &data, regs))
- perf_swevent_add(event, count, 1, &data, regs);
+ perf_swevent_event(event, count, 1, &data, regs);
}
perf_swevent_put_recursion_context(rctx);
@@ -4696,12 +4686,11 @@ static int perf_tp_event_init(struct per
static struct pmu perf_tracepoint = {
.event_init = perf_tp_event_init,
- .enable = perf_trace_enable,
- .disable = perf_trace_disable,
- .start = perf_swevent_int,
- .stop = perf_swevent_void,
+ .add = perf_trace_add,
+ .del = perf_trace_del,
+ .start = perf_swevent_start,
+ .stop = perf_swevent_stop,
.read = perf_swevent_read,
- .unthrottle = perf_swevent_void,
};
static inline void perf_tp_register(void)
@@ -4757,8 +4746,8 @@ void perf_bp_event(struct perf_event *bp
perf_sample_data_init(&sample, bp->attr.bp_addr);
- if (!perf_exclude_event(bp, regs))
- perf_swevent_add(bp, 1, 1, &sample, regs);
+ if (!bp->hw.state && !perf_exclude_event(bp, regs))
+ perf_swevent_event(bp, 1, 1, &sample, regs);
}
#endif
@@ -4834,32 +4823,39 @@ static void perf_swevent_cancel_hrtimer(
static void cpu_clock_event_update(struct perf_event *event)
{
- int cpu = raw_smp_processor_id();
s64 prev;
u64 now;
- now = cpu_clock(cpu);
+ now = local_clock();
prev = local64_xchg(&event->hw.prev_count, now);
local64_add(now - prev, &event->count);
}
-static int cpu_clock_event_enable(struct perf_event *event)
+static void cpu_clock_event_start(struct perf_event *event, int flags)
{
- struct hw_perf_event *hwc = &event->hw;
- int cpu = raw_smp_processor_id();
-
- local64_set(&hwc->prev_count, cpu_clock(cpu));
+ local64_set(&event->hw.prev_count, local_clock());
perf_swevent_start_hrtimer(event);
-
- return 0;
}
-static void cpu_clock_event_disable(struct perf_event *event)
+static void cpu_clock_event_stop(struct perf_event *event, int flags)
{
perf_swevent_cancel_hrtimer(event);
cpu_clock_event_update(event);
}
+static int cpu_clock_event_add(struct perf_event *event, int flags)
+{
+ if (flags & PERF_EF_START)
+ cpu_clock_event_start(event, flags);
+
+ return 0;
+}
+
+static void cpu_clock_event_del(struct perf_event *event, int flags)
+{
+ cpu_clock_event_stop(event, flags);
+}
+
static void cpu_clock_event_read(struct perf_event *event)
{
cpu_clock_event_update(event);
@@ -4878,8 +4874,10 @@ static int cpu_clock_event_init(struct p
static struct pmu perf_cpu_clock = {
.event_init = cpu_clock_event_init,
- .enable = cpu_clock_event_enable,
- .disable = cpu_clock_event_disable,
+ .add = cpu_clock_event_add,
+ .del = cpu_clock_event_del,
+ .start = cpu_clock_event_start,
+ .stop = cpu_clock_event_stop,
.read = cpu_clock_event_read,
};
@@ -4897,25 +4895,29 @@ static void task_clock_event_update(stru
local64_add(delta, &event->count);
}
-static int task_clock_event_enable(struct perf_event *event)
+static void task_clock_event_start(struct perf_event *event, int flags)
{
- struct hw_perf_event *hwc = &event->hw;
- u64 now;
-
- now = event->ctx->time;
-
- local64_set(&hwc->prev_count, now);
-
+ local64_set(&event->hw.prev_count, event->ctx->time);
perf_swevent_start_hrtimer(event);
-
- return 0;
}
-static void task_clock_event_disable(struct perf_event *event)
+static void task_clock_event_stop(struct perf_event *event, int flags)
{
perf_swevent_cancel_hrtimer(event);
task_clock_event_update(event, event->ctx->time);
+}
+
+static int task_clock_event_add(struct perf_event *event, int flags)
+{
+ if (flags & PERF_EF_START)
+ task_clock_event_start(event, flags);
+ return 0;
+}
+
+static void task_clock_event_del(struct perf_event *event, int flags)
+{
+ task_clock_event_stop(event, PERF_EF_UPDATE);
}
static void task_clock_event_read(struct perf_event *event)
@@ -4947,8 +4949,10 @@ static int task_clock_event_init(struct
static struct pmu perf_task_clock = {
.event_init = task_clock_event_init,
- .enable = task_clock_event_enable,
- .disable = task_clock_event_disable,
+ .add = task_clock_event_add,
+ .del = task_clock_event_del,
+ .start = task_clock_event_start,
+ .stop = task_clock_event_stop,
.read = task_clock_event_read,
};
Index: linux-2.6/kernel/trace/trace_event_perf.c
===================================================================
--- linux-2.6.orig/kernel/trace/trace_event_perf.c
+++ linux-2.6/kernel/trace/trace_event_perf.c
@@ -101,7 +101,7 @@ int perf_trace_init(struct perf_event *p
return ret;
}
-int perf_trace_enable(struct perf_event *p_event)
+int perf_trace_add(struct perf_event *p_event, int flags)
{
struct ftrace_event_call *tp_event = p_event->tp_event;
struct hlist_head __percpu *pcpu_list;
@@ -111,13 +111,16 @@ int perf_trace_enable(struct perf_event
if (WARN_ON_ONCE(!pcpu_list))
return -EINVAL;
+ if (!(flags & PERF_EF_START))
+ p_event->hw.state = PERF_HES_STOPPED;
+
list = this_cpu_ptr(pcpu_list);
hlist_add_head_rcu(&p_event->hlist_entry, list);
return 0;
}
-void perf_trace_disable(struct perf_event *p_event)
+void perf_trace_del(struct perf_event *p_event, int flags)
{
hlist_del_rcu(&p_event->hlist_entry);
}
Index: linux-2.6/arch/sh/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sh/kernel/perf_event.c
+++ linux-2.6/arch/sh/kernel/perf_event.c
@@ -206,26 +206,52 @@ static void sh_perf_event_update(struct
local64_add(delta, &event->count);
}
-static void sh_pmu_disable(struct perf_event *event)
+static void sh_pmu_stop(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
- clear_bit(idx, cpuc->active_mask);
- sh_pmu->disable(hwc, idx);
+ if (!(event->hw.state & PERF_HES_STOPPED)) {
+ sh_pmu->disable(hwc, idx);
+ cpuc->events[idx] = NULL;
+ event->hw.state |= PERF_HES_STOPPED;
+ }
- barrier();
+ if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
+ sh_perf_event_update(event, &event->hw, idx);
+ event->hw.state |= PERF_HES_UPTODATE;
+ }
+}
- sh_perf_event_update(event, &event->hw, idx);
+static void sh_pmu_start(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
- cpuc->events[idx] = NULL;
- clear_bit(idx, cpuc->used_mask);
+ if (WARN_ON_ONCE(idx == -1))
+ return;
+
+ if (flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+
+ cpuc->events[idx] = event;
+ event->hw.state = 0;
+ sh_pmu->enable(hwc, idx);
+}
+
+static void sh_pmu_del(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+ sh_pmu_stop(event, PERF_EF_UPDATE);
+ __clear_bit(event->hw.idx, cpuc->used_mask);
perf_event_update_userpage(event);
}
-static int sh_pmu_enable(struct perf_event *event)
+static int sh_pmu_add(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
@@ -234,21 +260,20 @@ static int sh_pmu_enable(struct perf_eve
perf_pmu_disable(event->pmu);
- if (test_and_set_bit(idx, cpuc->used_mask)) {
+ if (__test_and_set_bit(idx, cpuc->used_mask)) {
idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events);
if (idx == sh_pmu->num_events)
goto out;
- set_bit(idx, cpuc->used_mask);
+ __set_bit(idx, cpuc->used_mask);
hwc->idx = idx;
}
sh_pmu->disable(hwc, idx);
- cpuc->events[idx] = event;
- set_bit(idx, cpuc->active_mask);
-
- sh_pmu->enable(hwc, idx);
+ event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+ if (flags & PERF_EF_START)
+ sh_pmu_start(event, PERF_EF_RELOAD);
perf_event_update_userpage(event);
ret = 0;
@@ -285,7 +310,7 @@ static int sh_pmu_event_init(struct perf
return err;
}
-static void sh_pmu_pmu_enable(struct pmu *pmu)
+static void sh_pmu_enable(struct pmu *pmu)
{
if (!sh_pmu_initialized())
return;
@@ -293,7 +318,7 @@ static void sh_pmu_pmu_enable(struct pmu
sh_pmu->enable_all();
}
-static void sh_pmu_pmu_disable(struct pmu *pmu)
+static void sh_pmu_disable(struct pmu *pmu)
{
if (!sh_pmu_initialized())
return;
@@ -302,11 +327,13 @@ static void sh_pmu_pmu_disable(struct pm
}
static struct pmu pmu = {
- .pmu_enable = sh_pmu_pmu_enable,
- .pmu_disable = sh_pmu_pmu_disable,
+ .pmu_enable = sh_pmu_enable,
+ .pmu_disable = sh_pmu_disable,
.event_init = sh_pmu_event_init,
- .enable = sh_pmu_enable,
- .disable = sh_pmu_disable,
+ .add = sh_pmu_add,
+ .del = sh_pmu_del,
+ .start = sh_pmu_start,
+ .stop = sh_pmu_stop,
.read = sh_pmu_read,
};
@@ -334,15 +361,15 @@ sh_pmu_notifier(struct notifier_block *s
return NOTIFY_OK;
}
-int __cpuinit register_sh_pmu(struct sh_pmu *pmu)
+int __cpuinit register_sh_pmu(struct sh_pmu *_pmu)
{
if (sh_pmu)
return -EBUSY;
- sh_pmu = pmu;
+ sh_pmu = _pmu;
- pr_info("Performance Events: %s support registered\n", pmu->name);
+ pr_info("Performance Events: %s support registered\n", _pmu->name);
- WARN_ON(pmu->num_events > MAX_HWEVENTS);
+ WARN_ON(_pmu->num_events > MAX_HWEVENTS);
perf_pmu_register(&pmu);
perf_cpu_notifier(sh_pmu_notifier);
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -221,46 +221,56 @@ armpmu_event_update(struct perf_event *e
}
static void
-armpmu_disable(struct perf_event *event)
+armpmu_read(struct perf_event *event)
{
- struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
- int idx = hwc->idx;
-
- WARN_ON(idx < 0);
-
- clear_bit(idx, cpuc->active_mask);
- armpmu->disable(hwc, idx);
-
- barrier();
- armpmu_event_update(event, hwc, idx);
- cpuc->events[idx] = NULL;
- clear_bit(idx, cpuc->used_mask);
+ /* Don't read disabled counters! */
+ if (hwc->idx < 0)
+ return;
- perf_event_update_userpage(event);
+ armpmu_event_update(event, hwc, hwc->idx);
}
static void
-armpmu_read(struct perf_event *event)
+armpmu_stop(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
- /* Don't read disabled counters! */
- if (hwc->idx < 0)
+ if (!armpmu)
return;
- armpmu_event_update(event, hwc, hwc->idx);
+ /*
+ * ARM pmu always has to update the counter, so ignore
+ * PERF_EF_UPDATE, see comments in armpmu_start().
+ */
+ if (!(hwc->state & PERF_HES_STOPPED)) {
+ armpmu->disable(hwc, hwc->idx);
+ barrier(); /* why? */
+ armpmu_event_update(event, hwc, hwc->idx);
+ hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ }
}
static void
-armpmu_unthrottle(struct perf_event *event)
+armpmu_start(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
+ if (!armpmu)
+ return;
+
+ /*
+ * ARM pmu always has to reprogram the period, so ignore
+ * PERF_EF_RELOAD, see the comment below.
+ */
+ if (flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+
+ hwc->state = 0;
/*
* Set the period again. Some counters can't be stopped, so when we
- * were throttled we simply disabled the IRQ source and the counter
+ * were stopped we simply disabled the IRQ source and the counter
* may have been left counting. If we don't do this step then we may
* get an interrupt too soon or *way* too late if the overflow has
* happened since disabling.
@@ -269,8 +279,25 @@ armpmu_unthrottle(struct perf_event *eve
armpmu->enable(hwc, hwc->idx);
}
+static void
+armpmu_del(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
+
+ WARN_ON(idx < 0);
+
+ clear_bit(idx, cpuc->active_mask);
+ armpmu_stop(event, PERF_EF_UPDATE);
+ cpuc->events[idx] = NULL;
+ clear_bit(idx, cpuc->used_mask);
+
+ perf_event_update_userpage(event);
+}
+
static int
-armpmu_enable(struct perf_event *event)
+armpmu_add(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
@@ -295,11 +322,9 @@ armpmu_enable(struct perf_event *event)
cpuc->events[idx] = event;
set_bit(idx, cpuc->active_mask);
- /* Set the period for the event. */
- armpmu_event_set_period(event, hwc, idx);
-
- /* Enable the event. */
- armpmu->enable(hwc, idx);
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ if (flags & PERF_EF_START)
+ armpmu_start(event, PERF_EF_RELOAD);
/* Propagate our changes to the userspace mapping. */
perf_event_update_userpage(event);
@@ -534,7 +559,7 @@ static int armpmu_event_init(struct perf
return err;
}
-static void armpmu_pmu_enable(struct pmu *pmu)
+static void armpmu_enable(struct pmu *pmu)
{
/* Enable all of the perf events on hardware. */
int idx;
@@ -555,20 +580,21 @@ static void armpmu_pmu_enable(struct pmu
armpmu->start();
}
-static void armpmu_pmu_disable(struct pmu *pmu)
+static void armpmu_disable(struct pmu *pmu)
{
if (armpmu)
armpmu->stop();
}
static struct pmu pmu = {
- .pmu_enable = armpmu_pmu_enable,
- .pmu_disable= armpmu_pmu_disable,
- .event_init = armpmu_event_init,
- .enable = armpmu_enable,
- .disable = armpmu_disable,
- .unthrottle = armpmu_unthrottle,
- .read = armpmu_read,
+ .pmu_enable = armpmu_enable,
+ .pmu_disable = armpmu_disable,
+ .event_init = armpmu_event_init,
+ .add = armpmu_add,
+ .del = armpmu_del,
+ .start = armpmu_start,
+ .stop = armpmu_stop,
+ .read = armpmu_read,
};
/*
Index: linux-2.6/arch/powerpc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event.c
+++ linux-2.6/arch/powerpc/kernel/perf_event.c
@@ -402,6 +402,9 @@ static void power_pmu_read(struct perf_e
{
s64 val, delta, prev;
+ if (event->hw.state & PERF_HES_STOPPED)
+ return;
+
if (!event->hw.idx)
return;
/*
@@ -517,7 +520,7 @@ static void write_mmcr0(struct cpu_hw_ev
* Disable all events to prevent PMU interrupts and to allow
* events to be added or removed.
*/
-static void power_pmu_pmu_disable(struct pmu *pmu)
+static void power_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -565,7 +568,7 @@ static void power_pmu_pmu_disable(struct
* If we were previously disabled and events were added, then
* put the new config on the PMU.
*/
-static void power_pmu_pmu_enable(struct pmu *pmu)
+static void power_pmu_enable(struct pmu *pmu)
{
struct perf_event *event;
struct cpu_hw_events *cpuhw;
@@ -672,6 +675,8 @@ static void power_pmu_pmu_enable(struct
}
local64_set(&event->hw.prev_count, val);
event->hw.idx = idx;
+ if (event->hw.state & PERF_HES_STOPPED)
+ val = 0;
write_pmc(idx, val);
perf_event_update_userpage(event);
}
@@ -727,7 +732,7 @@ static int collect_events(struct perf_ev
* re-enable the PMU in order to get hw_perf_enable to do the
* actual work of reconfiguring the PMU.
*/
-static int power_pmu_enable(struct perf_event *event)
+static int power_pmu_add(struct perf_event *event, int ef_flags)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -749,6 +754,9 @@ static int power_pmu_enable(struct perf_
cpuhw->events[n0] = event->hw.config;
cpuhw->flags[n0] = event->hw.event_base;
+ if (!(ef_flags & PERF_EF_START))
+ event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
/*
* If group events scheduling transaction was started,
* skip the schedulability test here, it will be peformed
@@ -777,7 +785,7 @@ static int power_pmu_enable(struct perf_
/*
* Remove a event from the PMU.
*/
-static void power_pmu_disable(struct perf_event *event)
+static void power_pmu_del(struct perf_event *event, int ef_flags)
{
struct cpu_hw_events *cpuhw;
long i;
@@ -826,27 +834,53 @@ static void power_pmu_disable(struct per
}
/*
- * Re-enable interrupts on a event after they were throttled
- * because they were coming too fast.
+ * POWER-PMU does not support disabling individual counters, hence
+ * program their cycle counter to their max value and ignore the interrupts.
*/
-static void power_pmu_unthrottle(struct perf_event *event)
+
+static void power_pmu_start(struct perf_event *event, int ef_flags)
{
- s64 val, left;
unsigned long flags;
+ s64 left;
if (!event->hw.idx || !event->hw.sample_period)
return;
+
+ if (!(event->hw.state & PERF_HES_STOPPED))
+ return;
+
+ if (ef_flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+
+ local_irq_save(flags);
+ perf_pmu_disable(event->pmu);
+
+ event->hw.state = 0;
+ left = local64_read(&event->hw.period_left);
+ write_pmc(event->hw.idx, left);
+
+ perf_event_update_userpage(event);
+ perf_pmu_enable(event->pmu);
+ local_irq_restore(flags);
+}
+
+static void power_pmu_stop(struct perf_event *event, int ef_flags)
+{
+ unsigned long flags;
+
+ if (!event->hw.idx || !event->hw.sample_period)
+ return;
+
+ if (event->hw.state & PERF_HES_STOPPED)
+ return;
+
local_irq_save(flags);
perf_pmu_disable(event->pmu);
+
power_pmu_read(event);
- left = event->hw.sample_period;
- event->hw.last_period = left;
- val = 0;
- if (left < 0x80000000L)
- val = 0x80000000L - left;
- write_pmc(event->hw.idx, val);
- local64_set(&event->hw.prev_count, val);
- local64_set(&event->hw.period_left, left);
+ event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ write_pmc(event->hw.idx, 0);
+
perf_event_update_userpage(event);
perf_pmu_enable(event->pmu);
local_irq_restore(flags);
@@ -1131,13 +1165,14 @@ static int power_pmu_event_init(struct p
}
struct pmu power_pmu = {
- .pmu_enable = power_pmu_pmu_enable,
- .pmu_disable = power_pmu_pmu_disable,
+ .pmu_enable = power_pmu_enable,
+ .pmu_disable = power_pmu_disable,
.event_init = power_pmu_event_init,
- .enable = power_pmu_enable,
- .disable = power_pmu_disable,
+ .add = power_pmu_add,
+ .del = power_pmu_del,
+ .start = power_pmu_start,
+ .stop = power_pmu_stop,
.read = power_pmu_read,
- .unthrottle = power_pmu_unthrottle,
.start_txn = power_pmu_start_txn,
.cancel_txn = power_pmu_cancel_txn,
.commit_txn = power_pmu_commit_txn,
@@ -1155,6 +1190,11 @@ static void record_and_restart(struct pe
s64 prev, delta, left;
int record = 0;
+ if (event->hw.state & PERF_HES_STOPPED) {
+ write_pmc(event->hw.idx, 0);
+ return;
+ }
+
/* we don't have to worry about interrupts here */
prev = local64_read(&event->hw.prev_count);
delta = (val - prev) & 0xfffffffful;
@@ -1177,6 +1217,11 @@ static void record_and_restart(struct pe
val = 0x80000000LL - left;
}
+ write_pmc(event->hw.idx, val);
+ local64_set(&event->hw.prev_count, val);
+ local64_set(&event->hw.period_left, left);
+ perf_event_update_userpage(event);
+
/*
* Finally record data if requested.
*/
@@ -1189,23 +1234,9 @@ static void record_and_restart(struct pe
if (event->attr.sample_type & PERF_SAMPLE_ADDR)
perf_get_data_addr(regs, &data.addr);
- if (perf_event_overflow(event, nmi, &data, regs)) {
- /*
- * Interrupts are coming too fast - throttle them
- * by setting the event to 0, so it will be
- * at least 2^30 cycles until the next interrupt
- * (assuming each event counts at most 2 counts
- * per cycle).
- */
- val = 0;
- left = ~0ULL >> 1;
- }
+ if (perf_event_overflow(event, nmi, &data, regs))
+ power_pmu_stop(event, 0);
}
-
- write_pmc(event->hw.idx, val);
- local64_set(&event->hw.prev_count, val);
- local64_set(&event->hw.period_left, left);
- perf_event_update_userpage(event);
}
/*
Index: linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
===================================================================
--- linux-2.6.orig/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ linux-2.6/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -156,6 +156,9 @@ static void fsl_emb_pmu_read(struct perf
{
s64 val, delta, prev;
+ if (event->hw.state & PERF_HES_STOPPED)
+ return;
+
/*
* Performance monitor interrupts come even when interrupts
* are soft-disabled, as long as interrupts are hard-enabled.
@@ -177,7 +180,7 @@ static void fsl_emb_pmu_read(struct perf
* Disable all events to prevent PMU interrupts and to allow
* events to be added or removed.
*/
-static void fsl_emb_pmu_pmu_disable(struct pmu *pmu)
+static void fsl_emb_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -216,7 +219,7 @@ static void fsl_emb_pmu_pmu_disable(stru
* If we were previously disabled and events were added, then
* put the new config on the PMU.
*/
-static void fsl_emb_pmu_pmu_enable(struct pmu *pmu)
+static void fsl_emb_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuhw;
unsigned long flags;
@@ -263,7 +266,7 @@ static int collect_events(struct perf_ev
}
/* context locked on entry */
-static int fsl_emb_pmu_enable(struct perf_event *event)
+static int fsl_emb_pmu_add(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuhw;
int ret = -EAGAIN;
@@ -302,6 +305,12 @@ static int fsl_emb_pmu_enable(struct per
val = 0x80000000L - left;
}
local64_set(&event->hw.prev_count, val);
+
+ if (!(flags & PERF_EF_START)) {
+ event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ val = 0;
+ }
+
write_pmc(i, val);
perf_event_update_userpage(event);
@@ -316,7 +325,7 @@ static int fsl_emb_pmu_enable(struct per
}
/* context locked on entry */
-static void fsl_emb_pmu_disable(struct perf_event *event)
+static void fsl_emb_pmu_del(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuhw;
int i = event->hw.idx;
@@ -353,30 +362,49 @@ static void fsl_emb_pmu_disable(struct p
put_cpu_var(cpu_hw_events);
}
-/*
- * Re-enable interrupts on a event after they were throttled
- * because they were coming too fast.
- *
- * Context is locked on entry, but perf is not disabled.
- */
-static void fsl_emb_pmu_unthrottle(struct perf_event *event)
+static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
+{
+ unsigned long flags;
+ s64 left;
+
+ if (event->hw.idx < 0 || !event->hw.sample_period)
+ return;
+
+ if (!(event->hw.state & PERF_HES_STOPPED))
+ return;
+
+ if (ef_flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+
+ local_irq_save(flags);
+ perf_pmu_disable(event->pmu);
+
+ event->hw.state = 0;
+ left = local64_read(&event->hw.period_left);
+ write_pmc(event->hw.idx, left);
+
+ perf_event_update_userpage(event);
+ perf_pmu_enable(event->pmu);
+ local_irq_restore(flags);
+}
+
+static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
{
- s64 val, left;
unsigned long flags;
if (event->hw.idx < 0 || !event->hw.sample_period)
return;
+
+ if (event->hw.state & PERF_HES_STOPPED)
+ return;
+
local_irq_save(flags);
perf_pmu_disable(event->pmu);
+
fsl_emb_pmu_read(event);
- left = event->hw.sample_period;
- event->hw.last_period = left;
- val = 0;
- if (left < 0x80000000L)
- val = 0x80000000L - left;
- write_pmc(event->hw.idx, val);
- local64_set(&event->hw.prev_count, val);
- local64_set(&event->hw.period_left, left);
+ event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ write_pmc(event->hw.idx, 0);
+
perf_event_update_userpage(event);
perf_pmu_enable(event->pmu);
local_irq_restore(flags);
@@ -524,13 +552,14 @@ static int fsl_emb_pmu_event_init(struct
}
static struct pmu fsl_emb_pmu = {
- .pmu_enable = fsl_emb_pmu_pmu_enable,
- .pmu_disable = fsl_emb_pmu_pmu_disable,
+ .pmu_enable = fsl_emb_pmu_enable,
+ .pmu_disable = fsl_emb_pmu_disable,
.event_init = fsl_emb_pmu_event_init,
- .enable = fsl_emb_pmu_enable,
- .disable = fsl_emb_pmu_disable,
+ .add = fsl_emb_pmu_add,
+ .del = fsl_emb_pmu_del,
+ .start = fsl_emb_pmu_start,
+ .stop = fsl_emb_pmu_stop,
.read = fsl_emb_pmu_read,
- .unthrottle = fsl_emb_pmu_unthrottle,
};
/*
@@ -545,6 +574,11 @@ static void record_and_restart(struct pe
s64 prev, delta, left;
int record = 0;
+ if (event->hw.state & PERF_HES_STOPPED) {
+ write_pmc(event->hw.idx, 0);
+ return;
+ }
+
/* we don't have to worry about interrupts here */
prev = local64_read(&event->hw.prev_count);
delta = (val - prev) & 0xfffffffful;
@@ -567,6 +601,11 @@ static void record_and_restart(struct pe
val = 0x80000000LL - left;
}
+ write_pmc(event->hw.idx, val);
+ local64_set(&event->hw.prev_count, val);
+ local64_set(&event->hw.period_left, left);
+ perf_event_update_userpage(event);
+
/*
* Finally record data if requested.
*/
@@ -576,23 +615,9 @@ static void record_and_restart(struct pe
perf_sample_data_init(&data, 0);
data.period = event->hw.last_period;
- if (perf_event_overflow(event, nmi, &data, regs)) {
- /*
- * Interrupts are coming too fast - throttle them
- * by setting the event to 0, so it will be
- * at least 2^30 cycles until the next interrupt
- * (assuming each event counts at most 2 counts
- * per cycle).
- */
- val = 0;
- left = ~0ULL >> 1;
- }
+ if (perf_event_overflow(event, nmi, &data, regs))
+ fsl_emb_pmu_stop(event, 0);
}
-
- write_pmc(event->hw.idx, val);
- local64_set(&event->hw.prev_count, val);
- local64_set(&event->hw.period_left, left);
- perf_event_update_userpage(event);
}
static void perf_event_interrupt(struct pt_regs *regs)
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -658,13 +658,16 @@ static u64 maybe_change_configuration(st
enc = perf_event_get_enc(cpuc->events[i]);
pcr &= ~mask_for_index(idx);
- pcr |= event_encoding(enc, idx);
+ if (hwc->state & PERF_HES_STOPPED)
+ pcr |= nop_for_index(idx);
+ else
+ pcr |= event_encoding(enc, idx);
}
out:
return pcr;
}
-static void sparc_pmu_pmu_enable(struct pmu *pmu)
+static void sparc_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
u64 pcr;
@@ -691,7 +694,7 @@ static void sparc_pmu_pmu_enable(struct
pcr_ops->write(cpuc->pcr);
}
-static void sparc_pmu_pmu_disable(struct pmu *pmu)
+static void sparc_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
u64 val;
@@ -710,10 +713,53 @@ static void sparc_pmu_pmu_disable(struct
pcr_ops->write(cpuc->pcr);
}
-static void sparc_pmu_disable(struct perf_event *event)
+static int active_event_index(struct cpu_hw_events *cpuc,
+ struct perf_event *event)
+{
+ int i;
+
+ for (i = 0; i < cpuc->n_events; i++) {
+ if (cpuc->event[i] == event)
+ break;
+ }
+ BUG_ON(i == cpuc->n_events);
+ return cpuc->current_idx[i];
+}
+
+static void sparc_pmu_start(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ int idx = active_event_index(cpuc, event);
+
+ if (flags & PERF_EF_RELOAD) {
+ WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+ sparc_perf_event_set_period(event, &event->hw, idx);
+ }
+
+ event->hw.state = 0;
+
+ sparc_pmu_enable_event(cpuc, &event->hw, idx);
+}
+
+static void sparc_pmu_stop(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ int idx = active_event_index(cpuc, event);
+
+ if (!(event->hw.state & PERF_HES_STOPPED)) {
+ sparc_pmu_disable_event(cpuc, &event->hw, idx);
+ event->hw.state |= PERF_HES_STOPPED;
+ }
+
+ if (!(event->hw.state & PERF_HES_UPTODATE) && (flags & PERF_EF_UPDATE)) {
+ sparc_perf_event_update(event, &event->hw, idx);
+ event->hw.state |= PERF_HES_UPTODATE;
+ }
+}
+
+static void sparc_pmu_del(struct perf_event *event, int _flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
- struct hw_perf_event *hwc = &event->hw;
unsigned long flags;
int i;
@@ -722,7 +768,10 @@ static void sparc_pmu_disable(struct per
for (i = 0; i < cpuc->n_events; i++) {
if (event == cpuc->event[i]) {
- int idx = cpuc->current_idx[i];
+ /* Absorb the final count and turn off the
+ * event.
+ */
+ sparc_pmu_stop(event, PERF_EF_UPDATE);
/* Shift remaining entries down into
* the existing slot.
@@ -734,13 +783,6 @@ static void sparc_pmu_disable(struct per
cpuc->current_idx[i];
}
- /* Absorb the final count and turn off the
- * event.
- */
- sparc_pmu_disable_event(cpuc, hwc, idx);
- barrier();
- sparc_perf_event_update(event, hwc, idx);
-
perf_event_update_userpage(event);
cpuc->n_events--;
@@ -752,19 +794,6 @@ static void sparc_pmu_disable(struct per
local_irq_restore(flags);
}
-static int active_event_index(struct cpu_hw_events *cpuc,
- struct perf_event *event)
-{
- int i;
-
- for (i = 0; i < cpuc->n_events; i++) {
- if (cpuc->event[i] == event)
- break;
- }
- BUG_ON(i == cpuc->n_events);
- return cpuc->current_idx[i];
-}
-
static void sparc_pmu_read(struct perf_event *event)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -774,15 +803,6 @@ static void sparc_pmu_read(struct perf_e
sparc_perf_event_update(event, hwc, idx);
}
-static void sparc_pmu_unthrottle(struct perf_event *event)
-{
- struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
- int idx = active_event_index(cpuc, event);
- struct hw_perf_event *hwc = &event->hw;
-
- sparc_pmu_enable_event(cpuc, hwc, idx);
-}
-
static atomic_t active_events = ATOMIC_INIT(0);
static DEFINE_MUTEX(pmc_grab_mutex);
@@ -984,7 +1004,7 @@ static int collect_events(struct perf_ev
return n;
}
-static int sparc_pmu_enable(struct perf_event *event)
+static int sparc_pmu_add(struct perf_event *event, int ef_flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int n0, ret = -EAGAIN;
@@ -1001,6 +1021,10 @@ static int sparc_pmu_enable(struct perf_
cpuc->events[n0] = event->hw.event_base;
cpuc->current_idx[n0] = PIC_NO_INDEX;
+ event->hw.state = PERF_HES_UPTODATE;
+ if (!(ef_flags & PERF_EF_START))
+ event->hw.state |= PERF_HES_STOPPED;
+
/*
* If group events scheduling transaction was started,
* skip the schedulability test here, it will be peformed
@@ -1156,13 +1180,14 @@ static int sparc_pmu_commit_txn(struct p
}
static struct pmu pmu = {
- .pmu_enable = sparc_pmu_pmu_enable,
- .pmu_disable = sparc_pmu_pmu_disable,
+ .pmu_enable = sparc_pmu_enable,
+ .pmu_disable = sparc_pmu_disable,
.event_init = sparc_pmu_event_init,
- .enable = sparc_pmu_enable,
- .disable = sparc_pmu_disable,
+ .add = sparc_pmu_add,
+ .del = sparc_pmu_del,
+ .start = sparc_pmu_start,
+ .stop = sparc_pmu_stop,
.read = sparc_pmu_read,
- .unthrottle = sparc_pmu_unthrottle,
.start_txn = sparc_pmu_start_txn,
.cancel_txn = sparc_pmu_cancel_txn,
.commit_txn = sparc_pmu_commit_txn,
@@ -1243,7 +1268,7 @@ static int __kprobes perf_event_nmi_hand
continue;
if (perf_event_overflow(event, 1, &data, regs))
- sparc_pmu_disable_event(cpuc, hwc, idx);
+ sparc_pmu_stop(event, 0);
}
return NOTIFY_STOP;
Index: linux-2.6/include/linux/ftrace_event.h
===================================================================
--- linux-2.6.orig/include/linux/ftrace_event.h
+++ linux-2.6/include/linux/ftrace_event.h
@@ -252,8 +252,8 @@ DECLARE_PER_CPU(struct pt_regs, perf_tra
extern int perf_trace_init(struct perf_event *event);
extern void perf_trace_destroy(struct perf_event *event);
-extern int perf_trace_enable(struct perf_event *event);
-extern void perf_trace_disable(struct perf_event *event);
+extern int perf_trace_add(struct perf_event *event, int flags);
+extern void perf_trace_del(struct perf_event *event, int flags);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
char *filter_str);
extern void ftrace_profile_free_filter(struct perf_event *event);
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -538,6 +538,7 @@ struct hw_perf_event {
};
#endif
};
+ int state;
local64_t prev_count;
u64 sample_period;
u64 last_period;
@@ -549,6 +550,13 @@ struct hw_perf_event {
#endif
};
+/*
+ * hw_perf_event::state flags
+ */
+#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
+#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
+#define PERF_HES_ARCH 0x04
+
struct perf_event;
/*
@@ -564,42 +572,62 @@ struct pmu {
int *pmu_disable_count;
+ /*
+ * Fully disable/enable this PMU, can be used to protect from the PMI
+ * as well as for lazy/batch writing of the MSRs.
+ */
void (*pmu_enable) (struct pmu *pmu); /* optional */
void (*pmu_disable) (struct pmu *pmu); /* optional */
/*
+ * Try and initialize the event for this PMU.
* Should return -ENOENT when the @event doesn't match this PMU.
*/
int (*event_init) (struct perf_event *event);
- int (*enable) (struct perf_event *event);
- void (*disable) (struct perf_event *event);
- int (*start) (struct perf_event *event);
- void (*stop) (struct perf_event *event);
+#define PERF_EF_START 0x01 /* start the counter when adding */
+#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
+#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
+
+ /*
+ * Adds/Removes a counter to/from the PMU, can be done inside
+ * a transaction, see the ->*_txn() methods.
+ */
+ int (*add) (struct perf_event *event, int flags);
+ void (*del) (struct perf_event *event, int flags);
+
+ /*
+ * Starts/Stops a counter present on the PMU. The PMI handler
+ * should stop the counter when perf_event_overflow() returns
+ * !0. ->start() will be used to continue.
+ */
+ void (*start) (struct perf_event *event, int flags);
+ void (*stop) (struct perf_event *event, int flags);
+
+ /*
+ * Updates the counter value of the event.
+ */
void (*read) (struct perf_event *event);
- void (*unthrottle) (struct perf_event *event);
/*
* Group events scheduling is treated as a transaction, add
* group events as a whole and perform one schedulability test.
* If the test fails, roll back the whole group
- */
-
- /*
- * Start the transaction, after this ->enable() doesn't need to
+ *
+ * Start the transaction, after this ->add() doesn't need to
* do schedulability tests.
*/
void (*start_txn) (struct pmu *pmu); /* optional */
/*
- * If ->start_txn() disabled the ->enable() schedulability test
+ * If ->start_txn() disabled the ->add() schedulability test
* then ->commit_txn() is required to perform one. On success
* the transaction is closed. On error the transaction is kept
* open until ->cancel_txn() is called.
*/
int (*commit_txn) (struct pmu *pmu); /* optional */
/*
- * Will cancel the transaction, assumes ->disable() is called
- * for each successfull ->enable() during the transaction.
+ * Will cancel the transaction, assumes ->del() is called
+ * for each successfull ->add() during the transaction.
*/
void (*cancel_txn) (struct pmu *pmu); /* optional */
};
@@ -680,7 +708,7 @@ struct perf_event {
int nr_siblings;
int group_flags;
struct perf_event *group_leader;
- struct pmu *pmu;
+ struct pmu *pmu;
enum perf_event_active_state state;
unsigned int attach_state;
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel.c
@@ -763,7 +763,7 @@ static int intel_pmu_handle_irq(struct p
data.period = event->hw.last_period;
if (perf_event_overflow(event, 1, &data, regs))
- x86_pmu_stop(event);
+ x86_pmu_stop(event, 0);
}
/*
Index: linux-2.6/arch/x86/kernel/cpu/perf_event_intel_ds.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -491,7 +491,7 @@ static void __intel_pmu_pebs_event(struc
regs.flags &= ~PERF_EFLAGS_EXACT;
if (perf_event_overflow(event, 1, &data, ®s))
- x86_pmu_stop(event);
+ x86_pmu_stop(event, 0);
}
static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
Index: linux-2.6/kernel/hw_breakpoint.c
===================================================================
--- linux-2.6.orig/kernel/hw_breakpoint.c
+++ linux-2.6/kernel/hw_breakpoint.c
@@ -586,10 +586,35 @@ static int hw_breakpoint_event_init(stru
return 0;
}
+static int hw_breakpoint_add(struct perf_event *bp, int flags)
+{
+ if (!(flags & PERF_EF_START))
+ bp->hw.state = PERF_HES_STOPPED;
+
+ return arch_install_hw_breakpoint(bp);
+}
+
+static void hw_breakpoint_del(struct perf_event *bp, int flags)
+{
+ arch_uninstall_hw_breakpoint(bp);
+}
+
+static void hw_breakpoint_start(struct perf_event *bp, int flags)
+{
+ bp->hw.state = 0;
+}
+
+static void hw_breakpoint_stop(struct perf_event *bp, int flags)
+{
+ bp->hw.state = PERF_HES_STOPPED;
+}
+
static struct pmu perf_breakpoint = {
.event_init = hw_breakpoint_event_init,
- .enable = arch_install_hw_breakpoint,
- .disable = arch_uninstall_hw_breakpoint,
+ .add = hw_breakpoint_add,
+ .del = hw_breakpoint_del,
+ .start = hw_breakpoint_start,
+ .stop = hw_breakpoint_stop,
.read = hw_breakpoint_pmu_read,
};
Index: linux-2.6/arch/alpha/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/perf_event.c
+++ linux-2.6/arch/alpha/kernel/perf_event.c
@@ -307,7 +307,7 @@ static unsigned long alpha_perf_event_up
new_raw_count) != prev_raw_count)
goto again;
- delta = (new_raw_count - (prev_raw_count & alpha_pmu->pmc_count_mask[idx])) + ovf;
+ delta = (new_raw_count - (prev_raw_count & alpha_pmu->pmc_count_mask[idx])) + ovf;
/* It is possible on very rare occasions that the PMC has overflowed
* but the interrupt is yet to come. Detect and fix this situation.
@@ -402,14 +402,13 @@ static void maybe_change_configuration(s
struct hw_perf_event *hwc = &pe->hw;
int idx = hwc->idx;
- if (cpuc->current_idx[j] != PMC_NO_INDEX) {
- cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
- continue;
+ if (cpuc->current_idx[j] == PMC_NO_INDEX) {
+ alpha_perf_event_set_period(pe, hwc, idx);
+ cpuc->current_idx[j] = idx;
}
- alpha_perf_event_set_period(pe, hwc, idx);
- cpuc->current_idx[j] = idx;
- cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
+ if (!(hwc->state & PERF_HES_STOPPED))
+ cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
}
cpuc->config = cpuc->event[0]->hw.config_base;
}
@@ -420,7 +419,7 @@ static void maybe_change_configuration(s
* - this function is called from outside this module via the pmu struct
* returned from perf event initialisation.
*/
-static int alpha_pmu_enable(struct perf_event *event)
+static int alpha_pmu_add(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
int n0;
@@ -455,6 +454,10 @@ static int alpha_pmu_enable(struct perf_
}
}
+ hwc->state = PERF_HES_UPTODATE;
+ if (!(flags & PERF_EF_START))
+ hwc->state |= PERF_HES_STOPPED;
+
local_irq_restore(flags);
perf_pmu_enable(event->pmu);
@@ -467,7 +470,7 @@ static int alpha_pmu_enable(struct perf_
* - this function is called from outside this module via the pmu struct
* returned from perf event initialisation.
*/
-static void alpha_pmu_disable(struct perf_event *event)
+static void alpha_pmu_del(struct perf_event *event, int flags)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
@@ -514,13 +517,44 @@ static void alpha_pmu_read(struct perf_e
}
-static void alpha_pmu_unthrottle(struct perf_event *event)
+static void alpha_pmu_stop(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+ if (!(hwc->state & PERF_HES_STOPPED)) {
+ cpuc->idx_mask &= !(1UL<<hwc->idx);
+ hwc->state |= PERF_HES_STOPPED;
+ }
+
+ if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
+ alpha_perf_event_update(event, hwc, hwc->idx, 0);
+ hwc->state |= PERF_HES_UPTODATE;
+ }
+
+ if (cpuc->enabled)
+ wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
+}
+
+
+static void alpha_pmu_start(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+
+ if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
+ return;
+
+ if (flags & PERF_EF_RELOAD) {
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+ alpha_perf_event_set_period(event, hwc, hwc->idx);
+ }
+
+ hwc->state = 0;
+
cpuc->idx_mask |= 1UL<<hwc->idx;
- wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
+ if (cpuc->enabled)
+ wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
}
@@ -671,7 +705,7 @@ static int alpha_pmu_event_init(struct p
/*
* Main entry point - enable HW performance counters.
*/
-static void alpha_pmu_pmu_enable(struct pmu *pmu)
+static void alpha_pmu_enable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -697,7 +731,7 @@ static void alpha_pmu_pmu_enable(struct
* Main entry point - disable HW performance counters.
*/
-static void alpha_pmu_pmu_disable(struct pmu *pmu)
+static void alpha_pmu_disable(struct pmu *pmu)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -711,13 +745,14 @@ static void alpha_pmu_pmu_disable(struct
}
static struct pmu pmu = {
- .pmu_enable = alpha_pmu_pmu_enable,
- .pmu_disable = alpha_pmu_pmu_disable,
+ .pmu_enable = alpha_pmu_enable,
+ .pmu_disable = alpha_pmu_disable,
.event_init = alpha_pmu_event_init,
- .enable = alpha_pmu_enable,
- .disable = alpha_pmu_disable,
+ .add = alpha_pmu_add,
+ .del = alpha_pmu_del,
+ .start = alpha_pmu_start,
+ .stop = alpha_pmu_stop,
.read = alpha_pmu_read,
- .unthrottle = alpha_pmu_unthrottle,
};
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 10/19] perf: Remove the sysfs bits
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (8 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 09/19] perf: Rework the PMU methods Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 11/19] perf: Separate find_get_context() from event initialization Peter Zijlstra
` (9 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-kill-sysfs.patch --]
[-- Type: text/plain, Size: 10080 bytes --]
Neither the overcommit nor the reservation sysfs parameter were actually
working, remove them as they'll only get in the way.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/alpha/kernel/perf_event.c | 3
arch/arm/kernel/perf_event.c | 9 --
arch/sparc/kernel/perf_event.c | 9 --
arch/x86/kernel/cpu/perf_event.c | 1
include/linux/perf_event.h | 6 -
kernel/perf_event.c | 124 ---------------------------------------
6 files changed, 5 insertions(+), 147 deletions(-)
Index: linux-2.6/arch/alpha/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/perf_event.c
+++ linux-2.6/arch/alpha/kernel/perf_event.c
@@ -808,7 +808,7 @@ static void alpha_perf_event_irq_handler
wrperfmon(PERFMON_CMD_DISABLE, cpuc->idx_mask);
/* la_ptr is the counter that overflowed. */
- if (unlikely(la_ptr >= perf_max_events)) {
+ if (unlikely(la_ptr >= alpha_pmu->num_pmcs)) {
/* This should never occur! */
irq_err_count++;
pr_warning("PMI: silly index %ld\n", la_ptr);
@@ -879,7 +879,6 @@ void __init init_hw_perf_events(void)
/* And set up PMU specification */
alpha_pmu = &ev67_pmu;
- perf_max_events = alpha_pmu->num_pmcs;
perf_pmu_register(&pmu);
}
Index: linux-2.6/arch/arm/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/arm/kernel/perf_event.c
+++ linux-2.6/arch/arm/kernel/perf_event.c
@@ -534,7 +534,7 @@ static int armpmu_event_init(struct perf
event->destroy = hw_perf_event_destroy;
if (!atomic_inc_not_zero(&active_events)) {
- if (atomic_read(&active_events) > perf_max_events) {
+ if (atomic_read(&active_events) > armpmu.num_events) {
atomic_dec(&active_events);
return -ENOSPC;
}
@@ -2974,14 +2974,12 @@ init_hw_perf_events(void)
armpmu = &armv6pmu;
memcpy(armpmu_perf_cache_map, armv6_perf_cache_map,
sizeof(armv6_perf_cache_map));
- perf_max_events = armv6pmu.num_events;
break;
case 0xB020: /* ARM11mpcore */
armpmu = &armv6mpcore_pmu;
memcpy(armpmu_perf_cache_map,
armv6mpcore_perf_cache_map,
sizeof(armv6mpcore_perf_cache_map));
- perf_max_events = armv6mpcore_pmu.num_events;
break;
case 0xC080: /* Cortex-A8 */
armv7pmu.id = ARM_PERF_PMU_ID_CA8;
@@ -2993,7 +2991,6 @@ init_hw_perf_events(void)
/* Reset PMNC and read the nb of CNTx counters
supported */
armv7pmu.num_events = armv7_reset_read_pmnc();
- perf_max_events = armv7pmu.num_events;
break;
case 0xC090: /* Cortex-A9 */
armv7pmu.id = ARM_PERF_PMU_ID_CA9;
@@ -3005,7 +3002,6 @@ init_hw_perf_events(void)
/* Reset PMNC and read the nb of CNTx counters
supported */
armv7pmu.num_events = armv7_reset_read_pmnc();
- perf_max_events = armv7pmu.num_events;
break;
}
/* Intel CPUs [xscale]. */
@@ -3016,13 +3012,11 @@ init_hw_perf_events(void)
armpmu = &xscale1pmu;
memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
sizeof(xscale_perf_cache_map));
- perf_max_events = xscale1pmu.num_events;
break;
case 2:
armpmu = &xscale2pmu;
memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
sizeof(xscale_perf_cache_map));
- perf_max_events = xscale2pmu.num_events;
break;
}
}
@@ -3032,7 +3026,6 @@ init_hw_perf_events(void)
arm_pmu_names[armpmu->id], armpmu->num_events);
} else {
pr_info("no hardware support available\n");
- perf_max_events = -1;
}
perf_pmu_register(&pmu);
Index: linux-2.6/arch/sparc/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/arch/sparc/kernel/perf_event.c
+++ linux-2.6/arch/sparc/kernel/perf_event.c
@@ -897,7 +897,7 @@ static int sparc_check_constraints(struc
if (!n_ev)
return 0;
- if (n_ev > perf_max_events)
+ if (n_ev > MAX_HWEVENTS)
return -1;
msk0 = perf_event_get_msk(events[0]);
@@ -1014,7 +1014,7 @@ static int sparc_pmu_add(struct perf_eve
perf_pmu_disable(event->pmu);
n0 = cpuc->n_events;
- if (n0 >= perf_max_events)
+ if (n0 >= MAX_HWEVENTS)
goto out;
cpuc->event[n0] = event;
@@ -1097,7 +1097,7 @@ static int sparc_pmu_event_init(struct p
n = 0;
if (event->group_leader != event) {
n = collect_events(event->group_leader,
- perf_max_events - 1,
+ MAX_HWEVENTS - 1,
evts, events, current_idx_dmy);
if (n < 0)
return -EINVAL;
@@ -1309,9 +1309,6 @@ void __init init_hw_perf_events(void)
pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
- /* All sparc64 PMUs currently have 2 events. */
- perf_max_events = 2;
-
perf_pmu_register(&pmu);
register_die_notifier(&perf_event_nmi_notifier);
}
Index: linux-2.6/arch/x86/kernel/cpu/perf_event.c
===================================================================
--- linux-2.6.orig/arch/x86/kernel/cpu/perf_event.c
+++ linux-2.6/arch/x86/kernel/cpu/perf_event.c
@@ -1395,7 +1395,6 @@ void __init init_hw_perf_events(void)
x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
}
x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
- perf_max_events = x86_pmu.num_counters;
if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -860,7 +860,6 @@ struct perf_cpu_context {
struct perf_event_context ctx;
struct perf_event_context *task_ctx;
int active_oncpu;
- int max_pertask;
int exclusive;
struct swevent_hlist *swevent_hlist;
struct mutex hlist_mutex;
@@ -883,11 +882,6 @@ struct perf_output_handle {
#ifdef CONFIG_PERF_EVENTS
-/*
- * Set by architecture code:
- */
-extern int perf_max_events;
-
extern int perf_pmu_register(struct pmu *pmu);
extern void perf_pmu_unregister(struct pmu *pmu);
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -39,10 +39,6 @@
*/
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
-int perf_max_events __read_mostly = 1;
-static int perf_reserved_percpu __read_mostly;
-static int perf_overcommit __read_mostly = 1;
-
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
@@ -66,11 +62,6 @@ int sysctl_perf_event_sample_rate __read
static atomic64_t perf_event_id;
-/*
- * Lock for (sysadmin-configurable) event reservations:
- */
-static DEFINE_SPINLOCK(perf_resource_lock);
-
void __weak perf_event_print_debug(void) { }
void perf_pmu_disable(struct pmu *pmu)
@@ -480,16 +471,6 @@ static void __perf_event_remove_from_con
list_del_event(event, ctx);
- if (!ctx->task) {
- /*
- * Allow more per task events with respect to the
- * reservation:
- */
- cpuctx->max_pertask =
- min(perf_max_events - ctx->nr_events,
- perf_max_events - perf_reserved_percpu);
- }
-
raw_spin_unlock(&ctx->lock);
}
@@ -823,9 +804,6 @@ static void __perf_install_in_context(vo
}
}
- if (!err && !ctx->task && cpuctx->max_pertask)
- cpuctx->max_pertask--;
-
unlock:
raw_spin_unlock(&ctx->lock);
}
@@ -5929,10 +5907,6 @@ static void __cpuinit perf_event_init_cp
cpuctx = &per_cpu(perf_cpu_context, cpu);
- spin_lock(&perf_resource_lock);
- cpuctx->max_pertask = perf_max_events - perf_reserved_percpu;
- spin_unlock(&perf_resource_lock);
-
mutex_lock(&cpuctx->hlist_mutex);
if (cpuctx->hlist_refcount > 0) {
struct swevent_hlist *hlist;
@@ -6007,101 +5981,3 @@ void __init perf_event_init(void)
perf_tp_register();
perf_cpu_notifier(perf_cpu_notify);
}
-
-static ssize_t perf_show_reserve_percpu(struct sysdev_class *class,
- struct sysdev_class_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%d\n", perf_reserved_percpu);
-}
-
-static ssize_t
-perf_set_reserve_percpu(struct sysdev_class *class,
- struct sysdev_class_attribute *attr,
- const char *buf,
- size_t count)
-{
- struct perf_cpu_context *cpuctx;
- unsigned long val;
- int err, cpu, mpt;
-
- err = strict_strtoul(buf, 10, &val);
- if (err)
- return err;
- if (val > perf_max_events)
- return -EINVAL;
-
- spin_lock(&perf_resource_lock);
- perf_reserved_percpu = val;
- for_each_online_cpu(cpu) {
- cpuctx = &per_cpu(perf_cpu_context, cpu);
- raw_spin_lock_irq(&cpuctx->ctx.lock);
- mpt = min(perf_max_events - cpuctx->ctx.nr_events,
- perf_max_events - perf_reserved_percpu);
- cpuctx->max_pertask = mpt;
- raw_spin_unlock_irq(&cpuctx->ctx.lock);
- }
- spin_unlock(&perf_resource_lock);
-
- return count;
-}
-
-static ssize_t perf_show_overcommit(struct sysdev_class *class,
- struct sysdev_class_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "%d\n", perf_overcommit);
-}
-
-static ssize_t
-perf_set_overcommit(struct sysdev_class *class,
- struct sysdev_class_attribute *attr,
- const char *buf, size_t count)
-{
- unsigned long val;
- int err;
-
- err = strict_strtoul(buf, 10, &val);
- if (err)
- return err;
- if (val > 1)
- return -EINVAL;
-
- spin_lock(&perf_resource_lock);
- perf_overcommit = val;
- spin_unlock(&perf_resource_lock);
-
- return count;
-}
-
-static SYSDEV_CLASS_ATTR(
- reserve_percpu,
- 0644,
- perf_show_reserve_percpu,
- perf_set_reserve_percpu
- );
-
-static SYSDEV_CLASS_ATTR(
- overcommit,
- 0644,
- perf_show_overcommit,
- perf_set_overcommit
- );
-
-static struct attribute *perfclass_attrs[] = {
- &attr_reserve_percpu.attr,
- &attr_overcommit.attr,
- NULL
-};
-
-static struct attribute_group perfclass_attr_group = {
- .attrs = perfclass_attrs,
- .name = "perf_events",
-};
-
-static int __init perf_event_sysfs_init(void)
-{
- return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
- &perfclass_attr_group);
-}
-device_initcall(perf_event_sysfs_init);
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 11/19] perf: Separate find_get_context() from event initialization
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (9 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 10/19] perf: Remove the sysfs bits Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 12/19] perf: Remove the swevent hash-table from the cpu context Peter Zijlstra
` (8 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context.patch --]
[-- Type: text/plain, Size: 5376 bytes --]
Separate find_get_context() from the event allocation and initialization
so that we may make find_get_context() depend on the event pmu in a later
patch.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 73 ++++++++++++++++++++++++----------------------------
1 file changed, 35 insertions(+), 38 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -827,6 +827,8 @@ perf_install_in_context(struct perf_even
{
struct task_struct *task = ctx->task;
+ event->ctx = ctx;
+
if (!task) {
/*
* Per cpu events are installed via an smp call and
@@ -5037,20 +5039,17 @@ struct pmu *perf_init_event(struct perf_
* Allocate and initialize a event structure
*/
static struct perf_event *
-perf_event_alloc(struct perf_event_attr *attr,
- int cpu,
- struct perf_event_context *ctx,
+perf_event_alloc(struct perf_event_attr *attr, int cpu,
struct perf_event *group_leader,
struct perf_event *parent_event,
- perf_overflow_handler_t overflow_handler,
- gfp_t gfpflags)
+ perf_overflow_handler_t overflow_handler)
{
struct pmu *pmu;
struct perf_event *event;
struct hw_perf_event *hwc;
long err;
- event = kzalloc(sizeof(*event), gfpflags);
+ event = kzalloc(sizeof(*event), GFP_KERNEL);
if (!event)
return ERR_PTR(-ENOMEM);
@@ -5075,7 +5074,6 @@ perf_event_alloc(struct perf_event_attr
event->attr = *attr;
event->group_leader = group_leader;
event->pmu = NULL;
- event->ctx = ctx;
event->oncpu = -1;
event->parent = parent_event;
@@ -5320,20 +5318,26 @@ SYSCALL_DEFINE5(perf_event_open,
if (event_fd < 0)
return event_fd;
+ event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
+ if (IS_ERR(event)) {
+ err = PTR_ERR(event);
+ goto err_fd;
+ }
+
/*
* Get the target context (task or percpu):
*/
ctx = find_get_context(pid, cpu);
if (IS_ERR(ctx)) {
err = PTR_ERR(ctx);
- goto err_fd;
+ goto err_alloc;
}
if (group_fd != -1) {
group_leader = perf_fget_light(group_fd, &fput_needed);
if (IS_ERR(group_leader)) {
err = PTR_ERR(group_leader);
- goto err_put_context;
+ goto err_context;
}
group_file = group_leader->filp;
if (flags & PERF_FLAG_FD_OUTPUT)
@@ -5353,37 +5357,30 @@ SYSCALL_DEFINE5(perf_event_open,
* becoming part of another group-sibling):
*/
if (group_leader->group_leader != group_leader)
- goto err_put_context;
+ goto err_context;
/*
* Do not allow to attach to a group in a different
* task or CPU context:
*/
if (group_leader->ctx != ctx)
- goto err_put_context;
+ goto err_context;
/*
* Only a group leader can be exclusive or pinned
*/
if (attr.exclusive || attr.pinned)
- goto err_put_context;
- }
-
- event = perf_event_alloc(&attr, cpu, ctx, group_leader,
- NULL, NULL, GFP_KERNEL);
- if (IS_ERR(event)) {
- err = PTR_ERR(event);
- goto err_put_context;
+ goto err_context;
}
if (output_event) {
err = perf_event_set_output(event, output_event);
if (err)
- goto err_free_put_context;
+ goto err_context;
}
event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
if (IS_ERR(event_file)) {
err = PTR_ERR(event_file);
- goto err_free_put_context;
+ goto err_context;
}
event->filp = event_file;
@@ -5409,11 +5406,11 @@ SYSCALL_DEFINE5(perf_event_open,
fd_install(event_fd, event_file);
return event_fd;
-err_free_put_context:
- free_event(event);
-err_put_context:
+err_context:
fput_light(group_file, fput_needed);
put_ctx(ctx);
+err_alloc:
+ free_event(event);
err_fd:
put_unused_fd(event_fd);
return err;
@@ -5431,25 +5428,24 @@ perf_event_create_kernel_counter(struct
pid_t pid,
perf_overflow_handler_t overflow_handler)
{
- struct perf_event *event;
struct perf_event_context *ctx;
+ struct perf_event *event;
int err;
/*
* Get the target context (task or percpu):
*/
+ event = perf_event_alloc(attr, cpu, NULL, NULL, overflow_handler);
+ if (IS_ERR(event)) {
+ err = PTR_ERR(event);
+ goto err;
+ }
+
ctx = find_get_context(pid, cpu);
if (IS_ERR(ctx)) {
err = PTR_ERR(ctx);
- goto err_exit;
- }
-
- event = perf_event_alloc(attr, cpu, ctx, NULL,
- NULL, overflow_handler, GFP_KERNEL);
- if (IS_ERR(event)) {
- err = PTR_ERR(event);
- goto err_put_context;
+ goto err_free;
}
event->filp = NULL;
@@ -5467,9 +5463,9 @@ perf_event_create_kernel_counter(struct
return event;
- err_put_context:
- put_ctx(ctx);
- err_exit:
+err_free:
+ free_event(event);
+err:
return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
@@ -5497,9 +5493,9 @@ inherit_event(struct perf_event *parent_
parent_event = parent_event->parent;
child_event = perf_event_alloc(&parent_event->attr,
- parent_event->cpu, child_ctx,
+ parent_event->cpu,
group_leader, parent_event,
- NULL, GFP_KERNEL);
+ NULL);
if (IS_ERR(child_event))
return child_event;
get_ctx(child_ctx);
@@ -5524,6 +5520,7 @@ inherit_event(struct perf_event *parent_
local64_set(&hwc->period_left, sample_period);
}
+ child_event->ctx = child_ctx;
child_event->overflow_handler = parent_event->overflow_handler;
/*
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 12/19] perf: Remove the swevent hash-table from the cpu context
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (10 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 11/19] perf: Separate find_get_context() from event initialization Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 13/19] perf: Per cpu-context rotation timer Peter Zijlstra
` (7 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-5.patch --]
[-- Type: text/plain, Size: 9328 bytes --]
Separate the swevent hash-table from the cpu_context bits in preparation
for per pmu cpu contexts.
This keeps the swevent hash a global entity.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 6 --
kernel/perf_event.c | 104 +++++++++++++++++++++++++--------------------
2 files changed, 58 insertions(+), 52 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -866,12 +866,6 @@ struct perf_cpu_context {
struct perf_event_context *task_ctx;
int active_oncpu;
int exclusive;
- struct swevent_hlist *swevent_hlist;
- struct mutex hlist_mutex;
- int hlist_refcount;
-
- /* Recursion avoidance in each contexts */
- int recursion[PERF_NR_CONTEXTS];
};
struct perf_output_handle {
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -4154,6 +4154,17 @@ int perf_event_overflow(struct perf_even
* Generic software event infrastructure
*/
+struct swevent_htable {
+ struct swevent_hlist *swevent_hlist;
+ struct mutex hlist_mutex;
+ int hlist_refcount;
+
+ /* Recursion avoidance in each contexts */
+ int recursion[PERF_NR_CONTEXTS];
+};
+
+static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
+
/*
* We directly increment event->count and keep a second value in
* event->hw.period_left to count intervals. This period event
@@ -4286,11 +4297,11 @@ __find_swevent_head(struct swevent_hlist
/* For the read side: events when they trigger */
static inline struct hlist_head *
-find_swevent_head_rcu(struct perf_cpu_context *ctx, u64 type, u32 event_id)
+find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
{
struct swevent_hlist *hlist;
- hlist = rcu_dereference(ctx->swevent_hlist);
+ hlist = rcu_dereference(swhash->swevent_hlist);
if (!hlist)
return NULL;
@@ -4299,7 +4310,7 @@ find_swevent_head_rcu(struct perf_cpu_co
/* For the event head insertion and removal in the hlist */
static inline struct hlist_head *
-find_swevent_head(struct perf_cpu_context *ctx, struct perf_event *event)
+find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
{
struct swevent_hlist *hlist;
u32 event_id = event->attr.config;
@@ -4310,7 +4321,7 @@ find_swevent_head(struct perf_cpu_contex
* and release. Which makes the protected version suitable here.
* The context lock guarantees that.
*/
- hlist = rcu_dereference_protected(ctx->swevent_hlist,
+ hlist = rcu_dereference_protected(swhash->swevent_hlist,
lockdep_is_held(&event->ctx->lock));
if (!hlist)
return NULL;
@@ -4323,17 +4334,13 @@ static void do_perf_sw_event(enum perf_t
struct perf_sample_data *data,
struct pt_regs *regs)
{
- struct perf_cpu_context *cpuctx;
+ struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
struct perf_event *event;
struct hlist_node *node;
struct hlist_head *head;
- cpuctx = &__get_cpu_var(perf_cpu_context);
-
rcu_read_lock();
-
- head = find_swevent_head_rcu(cpuctx, type, event_id);
-
+ head = find_swevent_head_rcu(swhash, type, event_id);
if (!head)
goto end;
@@ -4347,17 +4354,17 @@ static void do_perf_sw_event(enum perf_t
int perf_swevent_get_recursion_context(void)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
- return get_recursion_context(cpuctx->recursion);
+ return get_recursion_context(swhash->recursion);
}
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
void inline perf_swevent_put_recursion_context(int rctx)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
- put_recursion_context(cpuctx->recursion, rctx);
+ put_recursion_context(swhash->recursion, rctx);
}
void __perf_sw_event(u32 event_id, u64 nr, int nmi,
@@ -4385,12 +4392,10 @@ static void perf_swevent_read(struct per
static int perf_swevent_add(struct perf_event *event, int flags)
{
+ struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
struct hw_perf_event *hwc = &event->hw;
- struct perf_cpu_context *cpuctx;
struct hlist_head *head;
- cpuctx = &__get_cpu_var(perf_cpu_context);
-
if (hwc->sample_period) {
hwc->last_period = hwc->sample_period;
perf_swevent_set_period(event);
@@ -4398,7 +4403,7 @@ static int perf_swevent_add(struct perf_
hwc->state = !(flags & PERF_EF_START);
- head = find_swevent_head(cpuctx, event);
+ head = find_swevent_head(swhash, event);
if (WARN_ON_ONCE(!head))
return -EINVAL;
@@ -4424,10 +4429,10 @@ static void perf_swevent_stop(struct per
/* Deref the hlist from the update side */
static inline struct swevent_hlist *
-swevent_hlist_deref(struct perf_cpu_context *cpuctx)
+swevent_hlist_deref(struct swevent_htable *swhash)
{
- return rcu_dereference_protected(cpuctx->swevent_hlist,
- lockdep_is_held(&cpuctx->hlist_mutex));
+ return rcu_dereference_protected(swhash->swevent_hlist,
+ lockdep_is_held(&swhash->hlist_mutex));
}
static void swevent_hlist_release_rcu(struct rcu_head *rcu_head)
@@ -4438,27 +4443,27 @@ static void swevent_hlist_release_rcu(st
kfree(hlist);
}
-static void swevent_hlist_release(struct perf_cpu_context *cpuctx)
+static void swevent_hlist_release(struct swevent_htable *swhash)
{
- struct swevent_hlist *hlist = swevent_hlist_deref(cpuctx);
+ struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
if (!hlist)
return;
- rcu_assign_pointer(cpuctx->swevent_hlist, NULL);
+ rcu_assign_pointer(swhash->swevent_hlist, NULL);
call_rcu(&hlist->rcu_head, swevent_hlist_release_rcu);
}
static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
{
- struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
+ struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
- mutex_lock(&cpuctx->hlist_mutex);
+ mutex_lock(&swhash->hlist_mutex);
- if (!--cpuctx->hlist_refcount)
- swevent_hlist_release(cpuctx);
+ if (!--swhash->hlist_refcount)
+ swevent_hlist_release(swhash);
- mutex_unlock(&cpuctx->hlist_mutex);
+ mutex_unlock(&swhash->hlist_mutex);
}
static void swevent_hlist_put(struct perf_event *event)
@@ -4476,12 +4481,12 @@ static void swevent_hlist_put(struct per
static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
{
- struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
+ struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
int err = 0;
- mutex_lock(&cpuctx->hlist_mutex);
+ mutex_lock(&swhash->hlist_mutex);
- if (!swevent_hlist_deref(cpuctx) && cpu_online(cpu)) {
+ if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
struct swevent_hlist *hlist;
hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
@@ -4489,11 +4494,11 @@ static int swevent_hlist_get_cpu(struct
err = -ENOMEM;
goto exit;
}
- rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
+ rcu_assign_pointer(swhash->swevent_hlist, hlist);
}
- cpuctx->hlist_refcount++;
+ swhash->hlist_refcount++;
exit:
- mutex_unlock(&cpuctx->hlist_mutex);
+ mutex_unlock(&swhash->hlist_mutex);
return err;
}
@@ -5891,12 +5896,15 @@ int perf_event_init_task(struct task_str
static void __init perf_event_init_all_cpus(void)
{
- int cpu;
struct perf_cpu_context *cpuctx;
+ struct swevent_htable *swhash;
+ int cpu;
for_each_possible_cpu(cpu) {
+ swhash = &per_cpu(swevent_htable, cpu);
+ mutex_init(&swhash->hlist_mutex);
+
cpuctx = &per_cpu(perf_cpu_context, cpu);
- mutex_init(&cpuctx->hlist_mutex);
__perf_event_init_context(&cpuctx->ctx, NULL);
}
}
@@ -5904,18 +5912,21 @@ static void __init perf_event_init_all_c
static void __cpuinit perf_event_init_cpu(int cpu)
{
struct perf_cpu_context *cpuctx;
+ struct swevent_htable *swhash;
cpuctx = &per_cpu(perf_cpu_context, cpu);
- mutex_lock(&cpuctx->hlist_mutex);
- if (cpuctx->hlist_refcount > 0) {
+ swhash = &per_cpu(swevent_htable, cpu);
+
+ mutex_lock(&swhash->hlist_mutex);
+ if (swhash->hlist_refcount > 0) {
struct swevent_hlist *hlist;
- hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
- WARN_ON_ONCE(!hlist);
- rcu_assign_pointer(cpuctx->swevent_hlist, hlist);
+ hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
+ WARN_ON(!hlist);
+ rcu_assign_pointer(swhash->swevent_hlist, hlist);
}
- mutex_unlock(&cpuctx->hlist_mutex);
+ mutex_unlock(&swhash->hlist_mutex);
}
#ifdef CONFIG_HOTPLUG_CPU
@@ -5933,11 +5944,12 @@ static void __perf_event_exit_cpu(void *
static void perf_event_exit_cpu(int cpu)
{
struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
+ struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
struct perf_event_context *ctx = &cpuctx->ctx;
- mutex_lock(&cpuctx->hlist_mutex);
- swevent_hlist_release(cpuctx);
- mutex_unlock(&cpuctx->hlist_mutex);
+ mutex_lock(&swhash->hlist_mutex);
+ swevent_hlist_release(swhash);
+ mutex_unlock(&swhash->hlist_mutex);
mutex_lock(&ctx->mutex);
smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (11 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 12/19] perf: Remove the swevent hash-table from the cpu context Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 17:07 ` Peter Zijlstra
2010-09-07 17:08 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 14/19] perf: Per-pmu-per-cpu contexts Peter Zijlstra
` (6 subsequent siblings)
19 siblings, 2 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-6.patch --]
[-- Type: text/plain, Size: 6181 bytes --]
Give each cpu-context its own timer so that it is a self contained
entity, this eases the way for per-pmu-per-cpu contexts as well as
provides means of overloading the event rotation per pmu implementation
as requested by Corey some time ago.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 10 +++++--
kernel/perf_event.c | 61 +++++++++++++++++++++++++++++++++------------
kernel/sched.c | 2 -
3 files changed, 52 insertions(+), 21 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -580,6 +580,11 @@ struct pmu {
void (*pmu_disable) (struct pmu *pmu); /* optional */
/*
+ * Provide for means to modify the normal overcommit rotation scheme.
+ */
+ void (*pmu_rotate_start) (struct pmu *pmu); /* optional */
+
+ /*
* Try and initialize the event for this PMU.
* Should return -ENOENT when the @event doesn't match this PMU.
*/
@@ -861,6 +866,8 @@ struct perf_cpu_context {
struct perf_event_context *task_ctx;
int active_oncpu;
int exclusive;
+ u64 timer_interval;
+ struct hrtimer timer;
};
struct perf_output_handle {
@@ -881,7 +888,6 @@ extern void perf_pmu_unregister(struct p
extern void perf_event_task_sched_in(struct task_struct *task);
extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
-extern void perf_event_task_tick(struct task_struct *task);
extern int perf_event_init_task(struct task_struct *child);
extern void perf_event_exit_task(struct task_struct *child);
extern void perf_event_free_task(struct task_struct *task);
@@ -1067,8 +1073,6 @@ perf_event_task_sched_in(struct task_str
static inline void
perf_event_task_sched_out(struct task_struct *task,
struct task_struct *next) { }
-static inline void
-perf_event_task_tick(struct task_struct *task) { }
static inline int perf_event_init_task(struct task_struct *child) { return 0; }
static inline void perf_event_exit_task(struct task_struct *child) { }
static inline void perf_event_free_task(struct task_struct *task) { }
Index: linux-2.6/kernel/sched.c
===================================================================
--- linux-2.6.orig/kernel/sched.c
+++ linux-2.6/kernel/sched.c
@@ -3441,8 +3441,6 @@ void scheduler_tick(void)
curr->sched_class->task_tick(rq, curr, 0);
raw_spin_unlock(&rq->lock);
- perf_event_task_tick(curr);
-
#ifdef CONFIG_SMP
rq->idle_at_tick = idle_cpu(cpu);
trigger_load_balance(rq, cpu);
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -281,6 +281,8 @@ list_add_event(struct perf_event *event,
}
list_add_rcu(&event->event_entry, &ctx->event_list);
+ if (!ctx->nr_events)
+ event->pmu->pmu_rotate_start(event->pmu);
ctx->nr_events++;
if (event->attr.inherit_stat)
ctx->nr_stat++;
@@ -1487,7 +1489,7 @@ static void perf_adjust_period(struct pe
}
}
-static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
+static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
{
struct perf_event *event;
struct hw_perf_event *hwc;
@@ -1524,7 +1526,7 @@ static void perf_ctx_adjust_freq(struct
hwc->freq_count_stamp = now;
if (delta > 0)
- perf_adjust_period(event, TICK_NSEC, delta);
+ perf_adjust_period(event, period, delta);
}
raw_spin_unlock(&ctx->lock);
}
@@ -1542,30 +1544,34 @@ static void rotate_ctx(struct perf_event
raw_spin_unlock(&ctx->lock);
}
-void perf_event_task_tick(struct task_struct *curr)
+static enum hrtimer_restart perf_event_context_tick(struct hrtimer *timer)
{
+ enum hrtimer_restart restart = HRTIMER_NORESTART;
struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx;
int rotate = 0;
- if (!atomic_read(&nr_events))
- return;
+ cpuctx = container_of(timer, struct perf_cpu_context, timer);
- cpuctx = &__get_cpu_var(perf_cpu_context);
- if (cpuctx->ctx.nr_events &&
- cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
- rotate = 1;
+ if (cpuctx->ctx.nr_events) {
+ restart = HRTIMER_RESTART;
+ if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
+ rotate = 1;
+ }
- ctx = curr->perf_event_ctxp;
- if (ctx && ctx->nr_events && ctx->nr_events != ctx->nr_active)
- rotate = 1;
+ ctx = current->perf_event_ctxp;
+ if (ctx && ctx->nr_events) {
+ restart = HRTIMER_RESTART;
+ if (ctx->nr_events != ctx->nr_active)
+ rotate = 1;
+ }
- perf_ctx_adjust_freq(&cpuctx->ctx);
+ perf_ctx_adjust_freq(&cpuctx->ctx, cpuctx->timer_interval);
if (ctx)
- perf_ctx_adjust_freq(ctx);
+ perf_ctx_adjust_freq(ctx, cpuctx->timer_interval);
if (!rotate)
- return;
+ goto done;
cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
if (ctx)
@@ -1577,7 +1583,24 @@ void perf_event_task_tick(struct task_st
cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
if (ctx)
- task_ctx_sched_in(curr, EVENT_FLEXIBLE);
+ task_ctx_sched_in(current, EVENT_FLEXIBLE);
+
+done:
+ hrtimer_forward_now(timer, ns_to_ktime(cpuctx->timer_interval));
+
+ return restart;
+}
+
+static void perf_pmu_rotate_start(struct pmu *pmu)
+{
+ struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+
+ if (hrtimer_active(&cpuctx->timer))
+ return;
+
+ __hrtimer_start_range_ns(&cpuctx->timer,
+ ns_to_ktime(cpuctx->timer_interval), 0,
+ HRTIMER_MODE_REL, 0);
}
static int event_enable_on_exec(struct perf_event *event,
@@ -5002,6 +5025,9 @@ int perf_pmu_register(struct pmu *pmu)
pmu->pmu_disable = perf_pmu_nop_void;
}
+ if (!pmu->pmu_rotate_start)
+ pmu->pmu_rotate_start = perf_pmu_rotate_start;
+
list_add_rcu(&pmu->entry, &pmus);
ret = 0;
unlock:
@@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c
cpuctx = &per_cpu(perf_cpu_context, cpu);
__perf_event_init_context(&cpuctx->ctx, NULL);
+ cpuctx->timer_interval = TICK_NSEC;
+ hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ cpuctx->timer.function = perf_event_context_tick;
}
}
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 16:46 ` [RFC][PATCH 13/19] perf: Per cpu-context rotation timer Peter Zijlstra
@ 2010-09-07 17:07 ` Peter Zijlstra
2010-09-07 17:33 ` Thomas Gleixner
2010-09-07 17:08 ` Peter Zijlstra
1 sibling, 1 reply; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 17:07 UTC (permalink / raw)
To: paulus
Cc: stephane eranian, Robert Richter, Will Deacon, Paul Mundt,
Frederic Weisbecker, Cyrill Gorcunov, Lin Ming, Yanmin,
Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel, tglx
On Tue, 2010-09-07 at 18:46 +0200, Peter Zijlstra wrote:
> @@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c
>
> cpuctx = &per_cpu(perf_cpu_context, cpu);
> __perf_event_init_context(&cpuctx->ctx, NULL);
> + cpuctx->timer_interval = TICK_NSEC;
> + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> + cpuctx->timer.function = perf_event_context_tick;
> }
> +static void perf_pmu_rotate_start(struct pmu *pmu)
> +{
> + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
> +
> + if (hrtimer_active(&cpuctx->timer))
> + return;
> +
> + __hrtimer_start_range_ns(&cpuctx->timer,
> + ns_to_ktime(cpuctx->timer_interval), 0,
> + HRTIMER_MODE_REL, 0);
> }
This probably wants a fuzz factor that lets it fold into the tick we
already have. Thomas what's the easiest way to do that, give it a soft
limit of 1ns and hardlimit of TICK_NSEC?
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 17:07 ` Peter Zijlstra
@ 2010-09-07 17:33 ` Thomas Gleixner
2010-09-07 17:36 ` Peter Zijlstra
0 siblings, 1 reply; 29+ messages in thread
From: Thomas Gleixner @ 2010-09-07 17:33 UTC (permalink / raw)
To: Peter Zijlstra
Cc: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel
On Tue, 7 Sep 2010, Peter Zijlstra wrote:
> On Tue, 2010-09-07 at 18:46 +0200, Peter Zijlstra wrote:
>
> > @@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c
> >
> > cpuctx = &per_cpu(perf_cpu_context, cpu);
> > __perf_event_init_context(&cpuctx->ctx, NULL);
> > + cpuctx->timer_interval = TICK_NSEC;
> > + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> > + cpuctx->timer.function = perf_event_context_tick;
> > }
>
> > +static void perf_pmu_rotate_start(struct pmu *pmu)
> > +{
> > + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
> > +
> > + if (hrtimer_active(&cpuctx->timer))
> > + return;
> > +
> > + __hrtimer_start_range_ns(&cpuctx->timer,
> > + ns_to_ktime(cpuctx->timer_interval), 0,
> > + HRTIMER_MODE_REL, 0);
> > }
>
> This probably wants a fuzz factor that lets it fold into the tick we
> already have. Thomas what's the easiest way to do that, give it a soft
> limit of 1ns and hardlimit of TICK_NSEC?
Hmm, why don't you hang it off the tick right away ?
Thanks,
tglx
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 17:33 ` Thomas Gleixner
@ 2010-09-07 17:36 ` Peter Zijlstra
2010-09-08 13:50 ` Thomas Gleixner
0 siblings, 1 reply; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 17:36 UTC (permalink / raw)
To: Thomas Gleixner
Cc: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel,
Corey Ashford
On Tue, 2010-09-07 at 19:33 +0200, Thomas Gleixner wrote:
> On Tue, 7 Sep 2010, Peter Zijlstra wrote:
>
> > On Tue, 2010-09-07 at 18:46 +0200, Peter Zijlstra wrote:
> >
> > > @@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c
> > >
> > > cpuctx = &per_cpu(perf_cpu_context, cpu);
> > > __perf_event_init_context(&cpuctx->ctx, NULL);
> > > + cpuctx->timer_interval = TICK_NSEC;
> > > + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> > > + cpuctx->timer.function = perf_event_context_tick;
> > > }
> >
> > > +static void perf_pmu_rotate_start(struct pmu *pmu)
> > > +{
> > > + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
> > > +
> > > + if (hrtimer_active(&cpuctx->timer))
> > > + return;
> > > +
> > > + __hrtimer_start_range_ns(&cpuctx->timer,
> > > + ns_to_ktime(cpuctx->timer_interval), 0,
> > > + HRTIMER_MODE_REL, 0);
> > > }
> >
> > This probably wants a fuzz factor that lets it fold into the tick we
> > already have. Thomas what's the easiest way to do that, give it a soft
> > limit of 1ns and hardlimit of TICK_NSEC?
>
> Hmm, why don't you hang it off the tick right away ?
Because some people (Corey) want to be able to have different rotation
periods per pmu driver, so a timer per context was the easy way out.
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 17:36 ` Peter Zijlstra
@ 2010-09-08 13:50 ` Thomas Gleixner
2010-09-08 13:56 ` Peter Zijlstra
0 siblings, 1 reply; 29+ messages in thread
From: Thomas Gleixner @ 2010-09-08 13:50 UTC (permalink / raw)
To: Peter Zijlstra
Cc: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel,
Corey Ashford
On Tue, 7 Sep 2010, Peter Zijlstra wrote:
> On Tue, 2010-09-07 at 19:33 +0200, Thomas Gleixner wrote:
> > On Tue, 7 Sep 2010, Peter Zijlstra wrote:
> >
> > > On Tue, 2010-09-07 at 18:46 +0200, Peter Zijlstra wrote:
> > >
> > > > @@ -5904,6 +5930,9 @@ static void __init perf_event_init_all_c
> > > >
> > > > cpuctx = &per_cpu(perf_cpu_context, cpu);
> > > > __perf_event_init_context(&cpuctx->ctx, NULL);
> > > > + cpuctx->timer_interval = TICK_NSEC;
> > > > + hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> > > > + cpuctx->timer.function = perf_event_context_tick;
> > > > }
> > >
> > > > +static void perf_pmu_rotate_start(struct pmu *pmu)
> > > > +{
> > > > + struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
> > > > +
> > > > + if (hrtimer_active(&cpuctx->timer))
> > > > + return;
> > > > +
> > > > + __hrtimer_start_range_ns(&cpuctx->timer,
> > > > + ns_to_ktime(cpuctx->timer_interval), 0,
> > > > + HRTIMER_MODE_REL, 0);
> > > > }
> > >
> > > This probably wants a fuzz factor that lets it fold into the tick we
> > > already have. Thomas what's the easiest way to do that, give it a soft
> > > limit of 1ns and hardlimit of TICK_NSEC?
> >
> > Hmm, why don't you hang it off the tick right away ?
>
> Because some people (Corey) want to be able to have different rotation
> periods per pmu driver, so a timer per context was the easy way out.
Makes sense. If you want it fully aligned with the tick, then you just
might get the real tick time from the tick code and make it proper
aligned.
Thanks,
tglx
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-08 13:50 ` Thomas Gleixner
@ 2010-09-08 13:56 ` Peter Zijlstra
0 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-08 13:56 UTC (permalink / raw)
To: Thomas Gleixner
Cc: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel,
Corey Ashford
On Wed, 2010-09-08 at 15:50 +0200, Thomas Gleixner wrote:
> > Because some people (Corey) want to be able to have different rotation
> > periods per pmu driver, so a timer per context was the easy way out.
>
> Makes sense. If you want it fully aligned with the tick, then you just
> might get the real tick time from the tick code and make it proper
> aligned.
Right, I'll go see if I can make that work nicely.
^ permalink raw reply [flat|nested] 29+ messages in thread
* Re: [RFC][PATCH 13/19] perf: Per cpu-context rotation timer
2010-09-07 16:46 ` [RFC][PATCH 13/19] perf: Per cpu-context rotation timer Peter Zijlstra
2010-09-07 17:07 ` Peter Zijlstra
@ 2010-09-07 17:08 ` Peter Zijlstra
1 sibling, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 17:08 UTC (permalink / raw)
To: paulus
Cc: stephane eranian, Robert Richter, Will Deacon, Paul Mundt,
Frederic Weisbecker, Cyrill Gorcunov, Lin Ming, Yanmin,
Deng-Cheng Zhu, David Miller, Ingo Molnar, linux-kernel
On Tue, 2010-09-07 at 18:46 +0200, Peter Zijlstra wrote:
> +static enum hrtimer_restart perf_event_context_tick(struct hrtimer
> *timer)
> {
> + enum hrtimer_restart restart = HRTIMER_NORESTART;
> struct perf_cpu_context *cpuctx;
> struct perf_event_context *ctx;
> int rotate = 0;
>
> + cpuctx = container_of(timer, struct perf_cpu_context, timer);
>
> + if (cpuctx->ctx.nr_events) {
> + restart = HRTIMER_RESTART;
> + if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
> + rotate = 1;
> + }
>
> + ctx = current->perf_event_ctxp;
> + if (ctx && ctx->nr_events) {
> + restart = HRTIMER_RESTART;
> + if (ctx->nr_events != ctx->nr_active)
> + rotate = 1;
> + }
Note to self, this is probably racy against pmu_rotate_start().
^ permalink raw reply [flat|nested] 29+ messages in thread
* [RFC][PATCH 14/19] perf: Per-pmu-per-cpu contexts
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (12 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 13/19] perf: Per cpu-context rotation timer Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 15/19] perf: Move some code around Peter Zijlstra
` (5 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-4.patch --]
[-- Type: text/plain, Size: 14268 bytes --]
Allocate per-cpu contexts per pmu.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 4 -
kernel/perf_event.c | 159 +++++++++++++++++++++++++++------------------
2 files changed, 101 insertions(+), 62 deletions(-)
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -570,7 +570,8 @@ struct perf_event;
struct pmu {
struct list_head entry;
- int *pmu_disable_count;
+ int * __percpu pmu_disable_count;
+ struct perf_cpu_context * __percpu pmu_cpu_context;
/*
* Fully disable/enable this PMU, can be used to protect from the PMI
@@ -813,6 +814,7 @@ struct perf_event {
* Used as a container for task events and CPU events as well:
*/
struct perf_event_context {
+ struct pmu *pmu;
/*
* Protect the states of the events in the list,
* nr_active, and the list:
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -34,16 +34,15 @@
#include <asm/irq_regs.h>
-/*
- * Each CPU has a list of per CPU events:
- */
-static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
-
static atomic_t nr_events __read_mostly;
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
+static LIST_HEAD(pmus);
+static DEFINE_MUTEX(pmus_lock);
+static struct srcu_struct pmus_srcu;
+
/*
* perf event paranoia level:
* -1 - not paranoid at all
@@ -447,6 +446,12 @@ group_sched_out(struct perf_event *group
cpuctx->exclusive = 0;
}
+static inline struct perf_cpu_context *
+__get_cpu_context(struct perf_event_context *ctx)
+{
+ return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
+}
+
/*
* Cross CPU call to remove a performance event
*
@@ -455,9 +460,9 @@ group_sched_out(struct perf_event *group
*/
static void __perf_event_remove_from_context(void *info)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event *event = info;
struct perf_event_context *ctx = event->ctx;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
/*
* If this is a task context, we need to check whether it is
@@ -537,8 +542,8 @@ static void perf_event_remove_from_conte
static void __perf_event_disable(void *info)
{
struct perf_event *event = info;
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event_context *ctx = event->ctx;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
/*
* If this is a per-task event, need to check whether this
@@ -746,10 +751,10 @@ static void add_event_to_ctx(struct perf
*/
static void __perf_install_in_context(void *info)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event *event = info;
struct perf_event_context *ctx = event->ctx;
struct perf_event *leader = event->group_leader;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
int err;
/*
@@ -893,9 +898,9 @@ static void __perf_event_mark_enabled(st
static void __perf_event_enable(void *info)
{
struct perf_event *event = info;
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event_context *ctx = event->ctx;
struct perf_event *leader = event->group_leader;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
int err;
/*
@@ -1169,8 +1174,8 @@ static void perf_event_sync_stat(struct
void perf_event_task_sched_out(struct task_struct *task,
struct task_struct *next)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event_context *ctx = task->perf_event_ctxp;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
struct perf_event_context *next_ctx;
struct perf_event_context *parent;
int do_switch = 1;
@@ -1223,7 +1228,7 @@ void perf_event_task_sched_out(struct ta
static void task_ctx_sched_out(struct perf_event_context *ctx,
enum event_type_t event_type)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
if (!cpuctx->task_ctx)
return;
@@ -1341,8 +1346,8 @@ static void cpu_ctx_sched_in(struct perf
static void task_ctx_sched_in(struct task_struct *task,
enum event_type_t event_type)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event_context *ctx = task->perf_event_ctxp;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
if (likely(!ctx))
return;
@@ -1364,8 +1369,8 @@ static void task_ctx_sched_in(struct tas
*/
void perf_event_task_sched_in(struct task_struct *task)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event_context *ctx = task->perf_event_ctxp;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
if (likely(!ctx))
return;
@@ -1593,7 +1598,7 @@ static enum hrtimer_restart perf_event_c
static void perf_pmu_rotate_start(struct pmu *pmu)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
+ struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
if (hrtimer_active(&cpuctx->timer))
return;
@@ -1669,9 +1674,9 @@ static void perf_event_enable_on_exec(st
*/
static void __perf_event_read(void *info)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
struct perf_event *event = info;
struct perf_event_context *ctx = event->ctx;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
/*
* If this is a task context, we need to check whether it is
@@ -1944,7 +1949,8 @@ __perf_event_init_context(struct perf_ev
ctx->task = task;
}
-static struct perf_event_context *find_get_context(pid_t pid, int cpu)
+static struct perf_event_context *
+find_get_context(struct pmu *pmu, pid_t pid, int cpu)
{
struct perf_event_context *ctx;
struct perf_cpu_context *cpuctx;
@@ -1968,7 +1974,7 @@ static struct perf_event_context *find_g
if (!cpu_online(cpu))
return ERR_PTR(-ENODEV);
- cpuctx = &per_cpu(perf_cpu_context, cpu);
+ cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
ctx = &cpuctx->ctx;
get_ctx(ctx);
@@ -2012,6 +2018,7 @@ static struct perf_event_context *find_g
if (!ctx)
goto errout;
__perf_event_init_context(ctx, task);
+ ctx->pmu = pmu;
get_ctx(ctx);
if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
/*
@@ -3727,18 +3734,20 @@ static void perf_event_task_ctx(struct p
static void perf_event_task_event(struct perf_task_event *task_event)
{
- struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx = task_event->task_ctx;
+ struct perf_cpu_context *cpuctx;
+ struct pmu *pmu;
- rcu_read_lock();
- cpuctx = &get_cpu_var(perf_cpu_context);
- perf_event_task_ctx(&cpuctx->ctx, task_event);
+ rcu_read_lock_sched();
+ list_for_each_entry_rcu(pmu, &pmus, entry) {
+ cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
+ perf_event_task_ctx(&cpuctx->ctx, task_event);
+ }
if (!ctx)
ctx = rcu_dereference(current->perf_event_ctxp);
if (ctx)
perf_event_task_ctx(ctx, task_event);
- put_cpu_var(perf_cpu_context);
- rcu_read_unlock();
+ rcu_read_unlock_sched();
}
static void perf_event_task(struct task_struct *task,
@@ -3843,6 +3852,7 @@ static void perf_event_comm_event(struct
struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx;
unsigned int size;
+ struct pmu *pmu;
char comm[TASK_COMM_LEN];
memset(comm, 0, sizeof(comm));
@@ -3854,14 +3864,15 @@ static void perf_event_comm_event(struct
comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
- rcu_read_lock();
- cpuctx = &get_cpu_var(perf_cpu_context);
- perf_event_comm_ctx(&cpuctx->ctx, comm_event);
+ rcu_read_lock_sched();
+ list_for_each_entry_rcu(pmu, &pmus, entry) {
+ cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
+ perf_event_comm_ctx(&cpuctx->ctx, comm_event);
+ }
ctx = rcu_dereference(current->perf_event_ctxp);
if (ctx)
perf_event_comm_ctx(ctx, comm_event);
- put_cpu_var(perf_cpu_context);
- rcu_read_unlock();
+ rcu_read_unlock_sched();
}
void perf_event_comm(struct task_struct *task)
@@ -3971,6 +3982,7 @@ static void perf_event_mmap_event(struct
char tmp[16];
char *buf = NULL;
const char *name;
+ struct pmu *pmu;
memset(tmp, 0, sizeof(tmp));
@@ -4022,14 +4034,16 @@ static void perf_event_mmap_event(struct
mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
- rcu_read_lock();
- cpuctx = &get_cpu_var(perf_cpu_context);
- perf_event_mmap_ctx(&cpuctx->ctx, mmap_event, vma->vm_flags & VM_EXEC);
+ rcu_read_lock_sched();
+ list_for_each_entry_rcu(pmu, &pmus, entry) {
+ cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
+ perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
+ vma->vm_flags & VM_EXEC);
+ }
ctx = rcu_dereference(current->perf_event_ctxp);
if (ctx)
perf_event_mmap_ctx(ctx, mmap_event, vma->vm_flags & VM_EXEC);
- put_cpu_var(perf_cpu_context);
- rcu_read_unlock();
+ rcu_read_unlock_sched();
kfree(buf);
}
@@ -4964,10 +4978,6 @@ static struct pmu perf_task_clock = {
.read = task_clock_event_read,
};
-static LIST_HEAD(pmus);
-static DEFINE_MUTEX(pmus_lock);
-static struct srcu_struct pmus_srcu;
-
static void perf_pmu_nop_void(struct pmu *pmu)
{
}
@@ -4995,7 +5005,7 @@ static void perf_pmu_cancel_txn(struct p
int perf_pmu_register(struct pmu *pmu)
{
- int ret;
+ int cpu, ret;
mutex_lock(&pmus_lock);
ret = -ENOMEM;
@@ -5003,6 +5013,21 @@ int perf_pmu_register(struct pmu *pmu)
if (!pmu->pmu_disable_count)
goto unlock;
+ pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
+ if (!pmu->pmu_cpu_context)
+ goto free_pdc;
+
+ for_each_possible_cpu(cpu) {
+ struct perf_cpu_context *cpuctx;
+
+ cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
+ __perf_event_init_context(&cpuctx->ctx, NULL);
+ cpuctx->ctx.pmu = pmu;
+ cpuctx->timer_interval = TICK_NSEC;
+ hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ cpuctx->timer.function = perf_event_context_tick;
+ }
+
if (!pmu->start_txn) {
if (pmu->pmu_enable) {
/*
@@ -5034,6 +5059,10 @@ int perf_pmu_register(struct pmu *pmu)
mutex_unlock(&pmus_lock);
return ret;
+
+free_pdc:
+ free_percpu(pmu->pmu_disable_count);
+ goto unlock;
}
void perf_pmu_unregister(struct pmu *pmu)
@@ -5042,9 +5071,14 @@ void perf_pmu_unregister(struct pmu *pmu
list_del_rcu(&pmu->entry);
mutex_unlock(&pmus_lock);
+ /*
+ * We use the pmu list either under SRCU or preempt_disable,
+ * synchronize_srcu() implies synchronize_sched() so we're good.
+ */
synchronize_srcu(&pmus_srcu);
free_percpu(pmu->pmu_disable_count);
+ free_percpu(pmu->pmu_cpu_context);
}
struct pmu *perf_init_event(struct perf_event *event)
@@ -5359,7 +5393,7 @@ SYSCALL_DEFINE5(perf_event_open,
/*
* Get the target context (task or percpu):
*/
- ctx = find_get_context(pid, cpu);
+ ctx = find_get_context(event->pmu, pid, cpu);
if (IS_ERR(ctx)) {
err = PTR_ERR(ctx);
goto err_alloc;
@@ -5474,7 +5508,7 @@ perf_event_create_kernel_counter(struct
goto err;
}
- ctx = find_get_context(pid, cpu);
+ ctx = find_get_context(event->pmu, pid, cpu);
if (IS_ERR(ctx)) {
err = PTR_ERR(ctx);
goto err_free;
@@ -5818,6 +5852,7 @@ inherit_task_group(struct perf_event *ev
return -ENOMEM;
__perf_event_init_context(child_ctx, child);
+ child_ctx->pmu = event->pmu;
child->perf_event_ctxp = child_ctx;
get_task_struct(child);
}
@@ -5920,30 +5955,18 @@ int perf_event_init_task(struct task_str
static void __init perf_event_init_all_cpus(void)
{
- struct perf_cpu_context *cpuctx;
struct swevent_htable *swhash;
int cpu;
for_each_possible_cpu(cpu) {
swhash = &per_cpu(swevent_htable, cpu);
mutex_init(&swhash->hlist_mutex);
-
- cpuctx = &per_cpu(perf_cpu_context, cpu);
- __perf_event_init_context(&cpuctx->ctx, NULL);
- cpuctx->timer_interval = TICK_NSEC;
- hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- cpuctx->timer.function = perf_event_context_tick;
}
}
static void __cpuinit perf_event_init_cpu(int cpu)
{
- struct perf_cpu_context *cpuctx;
- struct swevent_htable *swhash;
-
- cpuctx = &per_cpu(perf_cpu_context, cpu);
-
- swhash = &per_cpu(swevent_htable, cpu);
+ struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
mutex_lock(&swhash->hlist_mutex);
if (swhash->hlist_refcount > 0) {
@@ -5957,10 +5980,9 @@ static void __cpuinit perf_event_init_cp
}
#ifdef CONFIG_HOTPLUG_CPU
-static void __perf_event_exit_cpu(void *info)
+static void __perf_event_exit_context(void *__info)
{
- struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
- struct perf_event_context *ctx = &cpuctx->ctx;
+ struct perf_event_context *ctx = __info;
struct perf_event *event, *tmp;
list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
@@ -5968,19 +5990,34 @@ static void __perf_event_exit_cpu(void *
list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
__perf_event_remove_from_context(event);
}
+
+static void perf_event_exit_cpu_context(int cpu)
+{
+ struct perf_event_context *ctx;
+ struct pmu *pmu;
+ int idx;
+
+ idx = srcu_read_lock(&pmus_srcu);
+ list_for_each_entry_rcu(pmu, &pmus, entry) {
+ ctx = &this_cpu_ptr(pmu->pmu_cpu_context)->ctx;
+
+ mutex_lock(&ctx->mutex);
+ smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
+ mutex_unlock(&ctx->mutex);
+ }
+ srcu_read_unlock(&pmus_srcu, idx);
+
+}
+
static void perf_event_exit_cpu(int cpu)
{
- struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
- struct perf_event_context *ctx = &cpuctx->ctx;
mutex_lock(&swhash->hlist_mutex);
swevent_hlist_release(swhash);
mutex_unlock(&swhash->hlist_mutex);
- mutex_lock(&ctx->mutex);
- smp_call_function_single(cpu, __perf_event_exit_cpu, NULL, 1);
- mutex_unlock(&ctx->mutex);
+ perf_event_exit_cpu_context(cpu);
}
#else
static inline void perf_event_exit_cpu(int cpu) { }
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 15/19] perf: Move some code around
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (13 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 14/19] perf: Per-pmu-per-cpu contexts Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 16/19] perf: Clean up perf_event_context allocation Peter Zijlstra
` (4 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-7.patch --]
[-- Type: text/plain, Size: 6766 bytes --]
Move all inherit code near each other.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 200 ++++++++++++++++++++++++++--------------------------
1 file changed, 100 insertions(+), 100 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -5537,106 +5537,6 @@ perf_event_create_kernel_counter(struct
}
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
-/*
- * inherit a event from parent task to child task:
- */
-static struct perf_event *
-inherit_event(struct perf_event *parent_event,
- struct task_struct *parent,
- struct perf_event_context *parent_ctx,
- struct task_struct *child,
- struct perf_event *group_leader,
- struct perf_event_context *child_ctx)
-{
- struct perf_event *child_event;
-
- /*
- * Instead of creating recursive hierarchies of events,
- * we link inherited events back to the original parent,
- * which has a filp for sure, which we use as the reference
- * count:
- */
- if (parent_event->parent)
- parent_event = parent_event->parent;
-
- child_event = perf_event_alloc(&parent_event->attr,
- parent_event->cpu,
- group_leader, parent_event,
- NULL);
- if (IS_ERR(child_event))
- return child_event;
- get_ctx(child_ctx);
-
- /*
- * Make the child state follow the state of the parent event,
- * not its attr.disabled bit. We hold the parent's mutex,
- * so we won't race with perf_event_{en, dis}able_family.
- */
- if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
- child_event->state = PERF_EVENT_STATE_INACTIVE;
- else
- child_event->state = PERF_EVENT_STATE_OFF;
-
- if (parent_event->attr.freq) {
- u64 sample_period = parent_event->hw.sample_period;
- struct hw_perf_event *hwc = &child_event->hw;
-
- hwc->sample_period = sample_period;
- hwc->last_period = sample_period;
-
- local64_set(&hwc->period_left, sample_period);
- }
-
- child_event->ctx = child_ctx;
- child_event->overflow_handler = parent_event->overflow_handler;
-
- /*
- * Link it up in the child's context:
- */
- add_event_to_ctx(child_event, child_ctx);
-
- /*
- * Get a reference to the parent filp - we will fput it
- * when the child event exits. This is safe to do because
- * we are in the parent and we know that the filp still
- * exists and has a nonzero count:
- */
- atomic_long_inc(&parent_event->filp->f_count);
-
- /*
- * Link this into the parent event's child list
- */
- WARN_ON_ONCE(parent_event->ctx->parent_ctx);
- mutex_lock(&parent_event->child_mutex);
- list_add_tail(&child_event->child_list, &parent_event->child_list);
- mutex_unlock(&parent_event->child_mutex);
-
- return child_event;
-}
-
-static int inherit_group(struct perf_event *parent_event,
- struct task_struct *parent,
- struct perf_event_context *parent_ctx,
- struct task_struct *child,
- struct perf_event_context *child_ctx)
-{
- struct perf_event *leader;
- struct perf_event *sub;
- struct perf_event *child_ctr;
-
- leader = inherit_event(parent_event, parent, parent_ctx,
- child, NULL, child_ctx);
- if (IS_ERR(leader))
- return PTR_ERR(leader);
- list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
- child_ctr = inherit_event(sub, parent, parent_ctx,
- child, leader, child_ctx);
- if (IS_ERR(child_ctr))
- return PTR_ERR(child_ctr);
- }
- return 0;
-}
-
static void sync_child_event(struct perf_event *child_event,
struct task_struct *child)
{
@@ -5825,6 +5725,106 @@ void perf_event_free_task(struct task_st
put_ctx(ctx);
}
+/*
+ * inherit a event from parent task to child task:
+ */
+static struct perf_event *
+inherit_event(struct perf_event *parent_event,
+ struct task_struct *parent,
+ struct perf_event_context *parent_ctx,
+ struct task_struct *child,
+ struct perf_event *group_leader,
+ struct perf_event_context *child_ctx)
+{
+ struct perf_event *child_event;
+
+ /*
+ * Instead of creating recursive hierarchies of events,
+ * we link inherited events back to the original parent,
+ * which has a filp for sure, which we use as the reference
+ * count:
+ */
+ if (parent_event->parent)
+ parent_event = parent_event->parent;
+
+ child_event = perf_event_alloc(&parent_event->attr,
+ parent_event->cpu,
+ group_leader, parent_event,
+ NULL);
+ if (IS_ERR(child_event))
+ return child_event;
+ get_ctx(child_ctx);
+
+ /*
+ * Make the child state follow the state of the parent event,
+ * not its attr.disabled bit. We hold the parent's mutex,
+ * so we won't race with perf_event_{en, dis}able_family.
+ */
+ if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
+ child_event->state = PERF_EVENT_STATE_INACTIVE;
+ else
+ child_event->state = PERF_EVENT_STATE_OFF;
+
+ if (parent_event->attr.freq) {
+ u64 sample_period = parent_event->hw.sample_period;
+ struct hw_perf_event *hwc = &child_event->hw;
+
+ hwc->sample_period = sample_period;
+ hwc->last_period = sample_period;
+
+ local64_set(&hwc->period_left, sample_period);
+ }
+
+ child_event->ctx = child_ctx;
+ child_event->overflow_handler = parent_event->overflow_handler;
+
+ /*
+ * Link it up in the child's context:
+ */
+ add_event_to_ctx(child_event, child_ctx);
+
+ /*
+ * Get a reference to the parent filp - we will fput it
+ * when the child event exits. This is safe to do because
+ * we are in the parent and we know that the filp still
+ * exists and has a nonzero count:
+ */
+ atomic_long_inc(&parent_event->filp->f_count);
+
+ /*
+ * Link this into the parent event's child list
+ */
+ WARN_ON_ONCE(parent_event->ctx->parent_ctx);
+ mutex_lock(&parent_event->child_mutex);
+ list_add_tail(&child_event->child_list, &parent_event->child_list);
+ mutex_unlock(&parent_event->child_mutex);
+
+ return child_event;
+}
+
+static int inherit_group(struct perf_event *parent_event,
+ struct task_struct *parent,
+ struct perf_event_context *parent_ctx,
+ struct task_struct *child,
+ struct perf_event_context *child_ctx)
+{
+ struct perf_event *leader;
+ struct perf_event *sub;
+ struct perf_event *child_ctr;
+
+ leader = inherit_event(parent_event, parent, parent_ctx,
+ child, NULL, child_ctx);
+ if (IS_ERR(leader))
+ return PTR_ERR(leader);
+ list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
+ child_ctr = inherit_event(sub, parent, parent_ctx,
+ child, leader, child_ctx);
+ if (IS_ERR(child_ctr))
+ return PTR_ERR(child_ctr);
+ }
+ return 0;
+}
+
static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
struct perf_event_context *parent_ctx,
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 16/19] perf: Clean up perf_event_context allocation
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (14 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 15/19] perf: Move some code around Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 17/19] perf: Multiple task contexts Peter Zijlstra
` (3 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-8.patch --]
[-- Type: text/plain, Size: 2953 bytes --]
Unify the two perf_event_context allocation sites.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 41 ++++++++++++++++++++++++++---------------
1 file changed, 26 insertions(+), 15 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -1936,9 +1936,7 @@ static struct perf_callchain_entry *perf
/*
* Initialize the perf_event context in a task_struct:
*/
-static void
-__perf_event_init_context(struct perf_event_context *ctx,
- struct task_struct *task)
+static void __perf_event_init_context(struct perf_event_context *ctx)
{
raw_spin_lock_init(&ctx->lock);
mutex_init(&ctx->mutex);
@@ -1946,7 +1944,25 @@ __perf_event_init_context(struct perf_ev
INIT_LIST_HEAD(&ctx->flexible_groups);
INIT_LIST_HEAD(&ctx->event_list);
atomic_set(&ctx->refcount, 1);
- ctx->task = task;
+}
+
+static struct perf_event_context *
+alloc_perf_context(struct pmu *pmu, struct task_struct *task)
+{
+ struct perf_event_context *ctx;
+
+ ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
+ if (!ctx)
+ return NULL;
+
+ __perf_event_init_context(ctx);
+ if (task) {
+ ctx->task = task;
+ get_task_struct(task);
+ }
+ ctx->pmu = pmu;
+
+ return ctx;
}
static struct perf_event_context *
@@ -2013,22 +2029,22 @@ find_get_default_context(struct pmu *pmu
}
if (!ctx) {
- ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
+ ctx = alloc_perf_context(pmu, task);
err = -ENOMEM;
if (!ctx)
goto errout;
- __perf_event_init_context(ctx, task);
- ctx->pmu = pmu;
+
get_ctx(ctx);
+
if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
/*
* We raced with some other task; use
* the context they set.
*/
+ put_task_struct(task);
kfree(ctx);
goto retry;
}
- get_task_struct(task);
}
put_task_struct(task);
@@ -5021,7 +5037,7 @@ int perf_pmu_register(struct pmu *pmu)
struct perf_cpu_context *cpuctx;
cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
- __perf_event_init_context(&cpuctx->ctx, NULL);
+ __perf_event_init_context(&cpuctx->ctx);
cpuctx->ctx.pmu = pmu;
cpuctx->timer_interval = TICK_NSEC;
hrtimer_init(&cpuctx->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -5848,15 +5864,11 @@ inherit_task_group(struct perf_event *ev
* child.
*/
- child_ctx = kzalloc(sizeof(struct perf_event_context),
- GFP_KERNEL);
+ child_ctx = alloc_perf_context(event->pmu, child);
if (!child_ctx)
return -ENOMEM;
- __perf_event_init_context(child_ctx, child);
- child_ctx->pmu = event->pmu;
child->perf_event_ctxp = child_ctx;
- get_task_struct(child);
}
ret = inherit_group(event, parent, parent_ctx,
@@ -5868,7 +5880,6 @@ inherit_task_group(struct perf_event *ev
return ret;
}
-
/*
* Initialize the perf_event context in task_struct
*/
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 17/19] perf: Multiple task contexts
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (15 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 16/19] perf: Clean up perf_event_context allocation Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 18/19] perf: Provide a separate task context for swevents Peter Zijlstra
` (2 subsequent siblings)
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-pmu-context-3.patch --]
[-- Type: text/plain, Size: 21705 bytes --]
Provide the infrastructure for multiple task contexts.
A more flexible approach would have resulted in more pointer chases
in the scheduling hot-paths. This approach has the limitation of a
static number of task contexts.
Since I expect most external PMUs to be system wide, or at least node
wide (as per the intel uncore unit) they won't actually need a task
context.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/perf_event.h | 1
include/linux/sched.h | 8 -
kernel/perf_event.c | 344 ++++++++++++++++++++++++++++++---------------
3 files changed, 243 insertions(+), 110 deletions(-)
Index: linux-2.6/include/linux/sched.h
===================================================================
--- linux-2.6.orig/include/linux/sched.h
+++ linux-2.6/include/linux/sched.h
@@ -1161,6 +1161,12 @@ struct sched_rt_entity {
struct rcu_node;
+enum perf_event_task_context {
+ perf_invalid_context = -1,
+ perf_hw_context = 0,
+ perf_nr_task_contexts,
+};
+
struct task_struct {
volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
void *stack;
@@ -1434,7 +1440,7 @@ struct task_struct {
struct futex_pi_state *pi_state_cache;
#endif
#ifdef CONFIG_PERF_EVENTS
- struct perf_event_context *perf_event_ctxp;
+ struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
struct mutex perf_event_mutex;
struct list_head perf_event_list;
#endif
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -129,13 +129,13 @@ static u64 primary_event_id(struct perf_
* the context could get moved to another task.
*/
static struct perf_event_context *
-perf_lock_task_context(struct task_struct *task, unsigned long *flags)
+perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
{
struct perf_event_context *ctx;
rcu_read_lock();
retry:
- ctx = rcu_dereference(task->perf_event_ctxp);
+ ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
if (ctx) {
/*
* If this context is a clone of another, it might
@@ -148,7 +148,7 @@ perf_lock_task_context(struct task_struc
* can't get swapped on us any more.
*/
raw_spin_lock_irqsave(&ctx->lock, *flags);
- if (ctx != rcu_dereference(task->perf_event_ctxp)) {
+ if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
raw_spin_unlock_irqrestore(&ctx->lock, *flags);
goto retry;
}
@@ -167,12 +167,13 @@ perf_lock_task_context(struct task_struc
* can't get swapped to another task. This also increments its
* reference count so that the context can't get freed.
*/
-static struct perf_event_context *perf_pin_task_context(struct task_struct *task)
+static struct perf_event_context *
+perf_pin_task_context(struct task_struct *task, int ctxn)
{
struct perf_event_context *ctx;
unsigned long flags;
- ctx = perf_lock_task_context(task, &flags);
+ ctx = perf_lock_task_context(task, ctxn, &flags);
if (ctx) {
++ctx->pin_count;
raw_spin_unlock_irqrestore(&ctx->lock, flags);
@@ -1160,34 +1161,25 @@ static void perf_event_sync_stat(struct
}
}
-/*
- * Called from scheduler to remove the events of the current task,
- * with interrupts disabled.
- *
- * We stop each event and update the event value in event->count.
- *
- * This does not protect us against NMI, but disable()
- * sets the disabled bit in the control field of event _before_
- * accessing the event control register. If a NMI hits, then it will
- * not restart the event.
- */
-void perf_event_task_sched_out(struct task_struct *task,
- struct task_struct *next)
+void perf_event_context_sched_out(struct task_struct *task, int ctxn,
+ struct task_struct *next)
{
- struct perf_event_context *ctx = task->perf_event_ctxp;
- struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
+ struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
+ struct perf_cpu_context *cpuctx;
struct perf_event_context *next_ctx;
struct perf_event_context *parent;
int do_switch = 1;
- perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
+ if (likely(!ctx))
+ return;
- if (likely(!ctx || !cpuctx->task_ctx))
+ cpuctx = __get_cpu_context(ctx);
+ if (!cpuctx->task_ctx)
return;
rcu_read_lock();
parent = rcu_dereference(ctx->parent_ctx);
- next_ctx = next->perf_event_ctxp;
+ next_ctx = next->perf_event_ctxp[ctxn];
if (parent && next_ctx &&
rcu_dereference(next_ctx->parent_ctx) == parent) {
/*
@@ -1206,8 +1198,8 @@ void perf_event_task_sched_out(struct ta
* XXX do we need a memory barrier of sorts
* wrt to rcu_dereference() of perf_event_ctxp
*/
- task->perf_event_ctxp = next_ctx;
- next->perf_event_ctxp = ctx;
+ task->perf_event_ctxp[ctxn] = next_ctx;
+ next->perf_event_ctxp[ctxn] = ctx;
ctx->task = next;
next_ctx->task = task;
do_switch = 0;
@@ -1225,6 +1217,36 @@ void perf_event_task_sched_out(struct ta
}
}
+#define for_each_task_context(ctx, task) \
+ for ((ctx) = (task)->perf_event_ctxp[0]; \
+ (ctx) - (task)->perf_event_ctxp[0] < perf_nr_task_contexts;\
+ (ctx)++)
+
+#define for_each_task_context_nr(ctxn) \
+ for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
+
+/*
+ * Called from scheduler to remove the events of the current task,
+ * with interrupts disabled.
+ *
+ * We stop each event and update the event value in event->count.
+ *
+ * This does not protect us against NMI, but disable()
+ * sets the disabled bit in the control field of event _before_
+ * accessing the event control register. If a NMI hits, then it will
+ * not restart the event.
+ */
+void perf_event_task_sched_out(struct task_struct *task,
+ struct task_struct *next)
+{
+ int ctxn;
+
+ perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
+
+ for_each_task_context_nr(ctxn)
+ perf_event_context_sched_out(task, ctxn, next);
+}
+
static void task_ctx_sched_out(struct perf_event_context *ctx,
enum event_type_t event_type)
{
@@ -1343,38 +1365,24 @@ static void cpu_ctx_sched_in(struct perf
ctx_sched_in(ctx, cpuctx, event_type);
}
-static void task_ctx_sched_in(struct task_struct *task,
+static void task_ctx_sched_in(struct perf_event_context *ctx,
enum event_type_t event_type)
{
- struct perf_event_context *ctx = task->perf_event_ctxp;
- struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
+ struct perf_cpu_context *cpuctx;
- if (likely(!ctx))
- return;
+ cpuctx = __get_cpu_context(ctx);
if (cpuctx->task_ctx == ctx)
return;
+
ctx_sched_in(ctx, cpuctx, event_type);
cpuctx->task_ctx = ctx;
}
-/*
- * Called from scheduler to add the events of the current task
- * with interrupts disabled.
- *
- * We restore the event value and then enable it.
- *
- * This does not protect us against NMI, but enable()
- * sets the enabled bit in the control field of event _before_
- * accessing the event control register. If a NMI hits, then it will
- * keep the event running.
- */
-void perf_event_task_sched_in(struct task_struct *task)
-{
- struct perf_event_context *ctx = task->perf_event_ctxp;
- struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
- if (likely(!ctx))
- return;
+void perf_event_context_sched_in(struct perf_event_context *ctx)
+{
+ struct perf_cpu_context *cpuctx;
+ cpuctx = __get_cpu_context(ctx);
if (cpuctx->task_ctx == ctx)
return;
@@ -1392,6 +1400,29 @@ void perf_event_task_sched_in(struct tas
cpuctx->task_ctx = ctx;
}
+/*
+ * Called from scheduler to add the events of the current task
+ * with interrupts disabled.
+ *
+ * We restore the event value and then enable it.
+ *
+ * This does not protect us against NMI, but enable()
+ * sets the enabled bit in the control field of event _before_
+ * accessing the event control register. If a NMI hits, then it will
+ * keep the event running.
+ */
+void perf_event_task_sched_in(struct task_struct *task)
+{
+ struct perf_event_context *ctx;
+
+ for_each_task_context(ctx, task) {
+ if (likely(!ctx))
+ continue;
+
+ perf_event_context_sched_in(ctx);
+ }
+}
+
#define MAX_INTERRUPTS (~0ULL)
static void perf_log_throttle(struct perf_event *event, int enable);
@@ -1553,7 +1584,7 @@ static enum hrtimer_restart perf_event_c
{
enum hrtimer_restart restart = HRTIMER_NORESTART;
struct perf_cpu_context *cpuctx;
- struct perf_event_context *ctx;
+ struct perf_event_context *ctx = NULL;
int rotate = 0;
cpuctx = container_of(timer, struct perf_cpu_context, timer);
@@ -1564,7 +1595,7 @@ static enum hrtimer_restart perf_event_c
rotate = 1;
}
- ctx = current->perf_event_ctxp;
+ ctx = cpuctx->task_ctx;
if (ctx && ctx->nr_events) {
restart = HRTIMER_RESTART;
if (ctx->nr_events != ctx->nr_active)
@@ -1588,7 +1619,7 @@ static enum hrtimer_restart perf_event_c
cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE);
if (ctx)
- task_ctx_sched_in(current, EVENT_FLEXIBLE);
+ task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
done:
hrtimer_forward_now(timer, ns_to_ktime(cpuctx->timer_interval));
@@ -1627,20 +1658,18 @@ static int event_enable_on_exec(struct p
* Enable all of a task's events that have been marked enable-on-exec.
* This expects task == current.
*/
-static void perf_event_enable_on_exec(struct task_struct *task)
+static void perf_event_enable_on_exec(struct perf_event_context *ctx)
{
- struct perf_event_context *ctx;
struct perf_event *event;
unsigned long flags;
int enabled = 0;
int ret;
local_irq_save(flags);
- ctx = task->perf_event_ctxp;
if (!ctx || !ctx->nr_events)
goto out;
- __perf_event_task_sched_out(ctx);
+ task_ctx_sched_out(ctx, EVENT_ALL);
raw_spin_lock(&ctx->lock);
@@ -1664,7 +1693,7 @@ static void perf_event_enable_on_exec(st
raw_spin_unlock(&ctx->lock);
- perf_event_task_sched_in(task);
+ perf_event_context_sched_in(ctx);
out:
local_irq_restore(flags);
}
@@ -1972,7 +2001,7 @@ find_get_context(struct pmu *pmu, pid_t
struct perf_cpu_context *cpuctx;
struct task_struct *task;
unsigned long flags;
- int err;
+ int ctxn, err;
if (pid == -1 && cpu != -1) {
/* Must be root to operate on a CPU event: */
@@ -2021,8 +2050,13 @@ find_get_context(struct pmu *pmu, pid_t
if (!ptrace_may_access(task, PTRACE_MODE_READ))
goto errout;
+ err = -EINVAL;
+ ctxn = pmu->task_ctx_nr;
+ if (ctxn < 0)
+ goto errout;
+
retry:
- ctx = perf_lock_task_context(task, &flags);
+ ctx = perf_lock_task_context(task, ctxn, &flags);
if (ctx) {
unclone_ctx(ctx);
raw_spin_unlock_irqrestore(&ctx->lock, flags);
@@ -2036,7 +2070,7 @@ find_get_context(struct pmu *pmu, pid_t
get_ctx(ctx);
- if (cmpxchg(&task->perf_event_ctxp, NULL, ctx)) {
+ if (cmpxchg(&task->perf_event_ctxp[ctxn], NULL, ctx)) {
/*
* We raced with some other task; use
* the context they set.
@@ -3750,19 +3784,26 @@ static void perf_event_task_ctx(struct p
static void perf_event_task_event(struct perf_task_event *task_event)
{
- struct perf_event_context *ctx = task_event->task_ctx;
struct perf_cpu_context *cpuctx;
+ struct perf_event_context *ctx;
struct pmu *pmu;
+ int ctxn;
rcu_read_lock_sched();
list_for_each_entry_rcu(pmu, &pmus, entry) {
cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
perf_event_task_ctx(&cpuctx->ctx, task_event);
+
+ ctx = task_event->task_ctx;
+ if (!ctx) {
+ ctxn = pmu->task_ctx_nr;
+ if (ctxn < 0)
+ continue;
+ ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
+ }
+ if (ctx)
+ perf_event_task_ctx(ctx, task_event);
}
- if (!ctx)
- ctx = rcu_dereference(current->perf_event_ctxp);
- if (ctx)
- perf_event_task_ctx(ctx, task_event);
rcu_read_unlock_sched();
}
@@ -3867,9 +3908,10 @@ static void perf_event_comm_event(struct
{
struct perf_cpu_context *cpuctx;
struct perf_event_context *ctx;
+ char comm[TASK_COMM_LEN];
unsigned int size;
struct pmu *pmu;
- char comm[TASK_COMM_LEN];
+ int ctxn;
memset(comm, 0, sizeof(comm));
strlcpy(comm, comm_event->task->comm, sizeof(comm));
@@ -3884,19 +3926,28 @@ static void perf_event_comm_event(struct
list_for_each_entry_rcu(pmu, &pmus, entry) {
cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
perf_event_comm_ctx(&cpuctx->ctx, comm_event);
+
+ ctxn = pmu->task_ctx_nr;
+ if (ctxn < 0)
+ continue;
+
+ ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
+ if (ctx)
+ perf_event_comm_ctx(ctx, comm_event);
}
- ctx = rcu_dereference(current->perf_event_ctxp);
- if (ctx)
- perf_event_comm_ctx(ctx, comm_event);
rcu_read_unlock_sched();
}
void perf_event_comm(struct task_struct *task)
{
struct perf_comm_event comm_event;
+ struct perf_event_context *ctx;
- if (task->perf_event_ctxp)
- perf_event_enable_on_exec(task);
+ for_each_task_context(ctx, task) {
+ if (!ctx)
+ continue;
+ perf_event_enable_on_exec(ctx);
+ }
if (!atomic_read(&nr_comm_events))
return;
@@ -3999,6 +4050,7 @@ static void perf_event_mmap_event(struct
char *buf = NULL;
const char *name;
struct pmu *pmu;
+ int ctxn;
memset(tmp, 0, sizeof(tmp));
@@ -4055,10 +4107,17 @@ static void perf_event_mmap_event(struct
cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
vma->vm_flags & VM_EXEC);
+
+ ctxn = pmu->task_ctx_nr;
+ if (ctxn < 0)
+ continue;
+
+ ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
+ if (ctx) {
+ perf_event_mmap_ctx(ctx, mmap_event,
+ vma->vm_flags & VM_EXEC);
+ }
}
- ctx = rcu_dereference(current->perf_event_ctxp);
- if (ctx)
- perf_event_mmap_ctx(ctx, mmap_event, vma->vm_flags & VM_EXEC);
rcu_read_unlock_sched();
kfree(buf);
@@ -5019,6 +5078,40 @@ static void perf_pmu_cancel_txn(struct p
perf_pmu_enable(pmu);
}
+/*
+ * Ensures all context's with the same task_ctx_nr have the same
+ * pmu_cpu_context too.
+ */
+static void *find_pmu_context(int ctxn)
+{
+ struct pmu *pmu;
+
+ if (ctxn < 0)
+ return NULL;
+
+ list_for_each_entry(pmu, &pmus, entry) {
+ if (pmu->task_ctx_nr == ctxn)
+ return pmu->pmu_cpu_context;
+ }
+
+ return NULL;
+}
+
+static void free_pmu_context(void * __percpu cpu_context)
+{
+ struct pmu *pmu;
+
+ /*
+ * Like a real lame refcount.
+ */
+ list_for_each_entry(pmu, &pmus, entry) {
+ if (pmu->pmu_cpu_context == cpu_context)
+ return;
+ }
+
+ free_percpu(cpu_context);
+}
+
int perf_pmu_register(struct pmu *pmu)
{
int cpu, ret;
@@ -5029,6 +5122,10 @@ int perf_pmu_register(struct pmu *pmu)
if (!pmu->pmu_disable_count)
goto unlock;
+ pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
+ if (pmu->pmu_cpu_context)
+ goto got_cpu_context;
+
pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
if (!pmu->pmu_cpu_context)
goto free_pdc;
@@ -5044,6 +5141,7 @@ int perf_pmu_register(struct pmu *pmu)
cpuctx->timer.function = perf_event_context_tick;
}
+got_cpu_context:
if (!pmu->start_txn) {
if (pmu->pmu_enable) {
/*
@@ -5094,7 +5192,7 @@ void perf_pmu_unregister(struct pmu *pmu
synchronize_srcu(&pmus_srcu);
free_percpu(pmu->pmu_disable_count);
- free_percpu(pmu->pmu_cpu_context);
+ free_pmu_context(pmu->pmu_cpu_context);
}
struct pmu *perf_init_event(struct perf_event *event)
@@ -5608,16 +5706,13 @@ __perf_event_exit_task(struct perf_event
}
}
-/*
- * When a child task exits, feed back event values to parent events.
- */
-void perf_event_exit_task(struct task_struct *child)
+static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
{
struct perf_event *child_event, *tmp;
struct perf_event_context *child_ctx;
unsigned long flags;
- if (likely(!child->perf_event_ctxp)) {
+ if (likely(!child->perf_event_ctxp[ctxn])) {
perf_event_task(child, NULL, 0);
return;
}
@@ -5629,7 +5724,7 @@ void perf_event_exit_task(struct task_st
* scheduled, so we are now safe from rescheduling changing
* our context.
*/
- child_ctx = child->perf_event_ctxp;
+ child_ctx = child->perf_event_ctxp[ctxn];
__perf_event_task_sched_out(child_ctx);
/*
@@ -5638,7 +5733,7 @@ void perf_event_exit_task(struct task_st
* incremented the context's refcount before we do put_ctx below.
*/
raw_spin_lock(&child_ctx->lock);
- child->perf_event_ctxp = NULL;
+ child->perf_event_ctxp[ctxn] = NULL;
/*
* If this context is a clone; unclone it so it can't get
* swapped to another process while we're removing all
@@ -5691,6 +5786,17 @@ void perf_event_exit_task(struct task_st
put_ctx(child_ctx);
}
+/*
+ * When a child task exits, feed back event values to parent events.
+ */
+void perf_event_exit_task(struct task_struct *child)
+{
+ int ctxn;
+
+ for_each_task_context_nr(ctxn)
+ perf_event_exit_task_context(child, ctxn);
+}
+
static void perf_free_event(struct perf_event *event,
struct perf_event_context *ctx)
{
@@ -5712,32 +5818,35 @@ static void perf_free_event(struct perf_
/*
* free an unexposed, unused context as created by inheritance by
- * init_task below, used by fork() in case of fail.
+ * perf_event_init_task below, used by fork() in case of fail.
*/
void perf_event_free_task(struct task_struct *task)
{
- struct perf_event_context *ctx = task->perf_event_ctxp;
+ struct perf_event_context *ctx;
struct perf_event *event, *tmp;
- if (!ctx)
- return;
+ for_each_task_context(ctx, task) {
+ if (!ctx)
+ continue;
- mutex_lock(&ctx->mutex);
+ mutex_lock(&ctx->mutex);
again:
- list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
- perf_free_event(event, ctx);
+ list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
+ group_entry)
+ perf_free_event(event, ctx);
+
+ list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
+ group_entry)
+ perf_free_event(event, ctx);
+
+ if (!list_empty(&ctx->pinned_groups) ||
+ !list_empty(&ctx->flexible_groups))
+ goto again;
- list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
- group_entry)
- perf_free_event(event, ctx);
-
- if (!list_empty(&ctx->pinned_groups) ||
- !list_empty(&ctx->flexible_groups))
- goto again;
-
- mutex_unlock(&ctx->mutex);
+ mutex_unlock(&ctx->mutex);
- put_ctx(ctx);
+ put_ctx(ctx);
+ }
}
/*
@@ -5843,17 +5952,18 @@ static int inherit_group(struct perf_eve
static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
struct perf_event_context *parent_ctx,
- struct task_struct *child,
+ struct task_struct *child, int ctxn,
int *inherited_all)
{
int ret;
- struct perf_event_context *child_ctx = child->perf_event_ctxp;
+ struct perf_event_context *child_ctx;
if (!event->attr.inherit) {
*inherited_all = 0;
return 0;
}
+ child_ctx = child->perf_event_ctxp[ctxn];
if (!child_ctx) {
/*
* This is executed from the parent task context, so
@@ -5866,7 +5976,7 @@ inherit_task_group(struct perf_event *ev
if (!child_ctx)
return -ENOMEM;
- child->perf_event_ctxp = child_ctx;
+ child->perf_event_ctxp[ctxn] = child_ctx;
}
ret = inherit_group(event, parent, parent_ctx,
@@ -5881,7 +5991,7 @@ inherit_task_group(struct perf_event *ev
/*
* Initialize the perf_event context in task_struct
*/
-int perf_event_init_task(struct task_struct *child)
+int perf_event_init_context(struct task_struct *child, int ctxn)
{
struct perf_event_context *child_ctx, *parent_ctx;
struct perf_event_context *cloned_ctx;
@@ -5890,19 +6000,19 @@ int perf_event_init_task(struct task_str
int inherited_all = 1;
int ret = 0;
- child->perf_event_ctxp = NULL;
+ child->perf_event_ctxp[ctxn] = NULL;
mutex_init(&child->perf_event_mutex);
INIT_LIST_HEAD(&child->perf_event_list);
- if (likely(!parent->perf_event_ctxp))
+ if (likely(!parent->perf_event_ctxp[ctxn]))
return 0;
/*
* If the parent's context is a clone, pin it so it won't get
* swapped under us.
*/
- parent_ctx = perf_pin_task_context(parent);
+ parent_ctx = perf_pin_task_context(parent, ctxn);
/*
* No need to check if parent_ctx != NULL here; since we saw
@@ -5922,20 +6032,20 @@ int perf_event_init_task(struct task_str
* the list, not manipulating it:
*/
list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
- ret = inherit_task_group(event, parent, parent_ctx, child,
- &inherited_all);
+ ret = inherit_task_group(event, parent, parent_ctx,
+ child, ctxn, &inherited_all);
if (ret)
break;
}
list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
- ret = inherit_task_group(event, parent, parent_ctx, child,
- &inherited_all);
+ ret = inherit_task_group(event, parent, parent_ctx,
+ child, ctxn, &inherited_all);
if (ret)
break;
}
- child_ctx = child->perf_event_ctxp;
+ child_ctx = child->perf_event_ctxp[ctxn];
if (child_ctx && inherited_all) {
/*
@@ -5964,6 +6074,22 @@ int perf_event_init_task(struct task_str
return ret;
}
+/*
+ * Initialize the perf_event context in task_struct
+ */
+int perf_event_init_task(struct task_struct *child)
+{
+ int ctxn, ret;
+
+ for_each_task_context_nr(ctxn) {
+ ret = perf_event_init_context(child, ctxn);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
static void __init perf_event_init_all_cpus(void)
{
struct swevent_htable *swhash;
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -572,6 +572,7 @@ struct pmu {
int * __percpu pmu_disable_count;
struct perf_cpu_context * __percpu pmu_cpu_context;
+ int task_ctx_nr;
/*
* Fully disable/enable this PMU, can be used to protect from the PMI
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 18/19] perf: Provide a separate task context for swevents
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (16 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 17/19] perf: Multiple task contexts Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-07 16:46 ` [RFC][PATCH 19/19] perf: Optimize context ops Peter Zijlstra
2010-09-10 4:39 ` [RFC][PATCH 00/19] perf pmu interface changes -v4 Paul Mackerras
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-swevent-context.patch --]
[-- Type: text/plain, Size: 2332 bytes --]
Since software events are always schedulable, mixing them up with
hardware events (who are not) can lead to funny scheduling oddities.
Giving them their own context solves this.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/sched.h | 1 +
kernel/hw_breakpoint.c | 2 ++
kernel/perf_event.c | 8 ++++++++
3 files changed, 11 insertions(+)
Index: linux-2.6/include/linux/sched.h
===================================================================
--- linux-2.6.orig/include/linux/sched.h
+++ linux-2.6/include/linux/sched.h
@@ -1164,6 +1164,7 @@ struct rcu_node;
enum perf_event_task_context {
perf_invalid_context = -1,
perf_hw_context = 0,
+ perf_sw_context,
perf_nr_task_contexts,
};
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -4691,6 +4691,8 @@ static int perf_swevent_init(struct perf
}
static struct pmu perf_swevent = {
+ .task_ctx_nr = perf_sw_context,
+
.event_init = perf_swevent_init,
.add = perf_swevent_add,
.del = perf_swevent_del,
@@ -4782,6 +4784,8 @@ static int perf_tp_event_init(struct per
}
static struct pmu perf_tracepoint = {
+ .task_ctx_nr = perf_sw_context,
+
.event_init = perf_tp_event_init,
.add = perf_trace_add,
.del = perf_trace_del,
@@ -4970,6 +4974,8 @@ static int cpu_clock_event_init(struct p
}
static struct pmu perf_cpu_clock = {
+ .task_ctx_nr = perf_sw_context,
+
.event_init = cpu_clock_event_init,
.add = cpu_clock_event_add,
.del = cpu_clock_event_del,
@@ -5045,6 +5051,8 @@ static int task_clock_event_init(struct
}
static struct pmu perf_task_clock = {
+ .task_ctx_nr = perf_sw_context,
+
.event_init = task_clock_event_init,
.add = task_clock_event_add,
.del = task_clock_event_del,
Index: linux-2.6/kernel/hw_breakpoint.c
===================================================================
--- linux-2.6.orig/kernel/hw_breakpoint.c
+++ linux-2.6/kernel/hw_breakpoint.c
@@ -610,6 +610,8 @@ static void hw_breakpoint_stop(struct pe
}
static struct pmu perf_breakpoint = {
+ .task_ctx_nr = perf_sw_context, /* could eventually get its own */
+
.event_init = hw_breakpoint_event_init,
.add = hw_breakpoint_add,
.del = hw_breakpoint_del,
^ permalink raw reply [flat|nested] 29+ messages in thread* [RFC][PATCH 19/19] perf: Optimize context ops
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (17 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 18/19] perf: Provide a separate task context for swevents Peter Zijlstra
@ 2010-09-07 16:46 ` Peter Zijlstra
2010-09-10 4:39 ` [RFC][PATCH 00/19] perf pmu interface changes -v4 Paul Mackerras
19 siblings, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-07 16:46 UTC (permalink / raw)
To: paulus, stephane eranian, Robert Richter, Will Deacon,
Paul Mundt, Frederic Weisbecker, Cyrill Gorcunov, Lin Ming,
Yanmin, Deng-Cheng Zhu, David Miller, Ingo Molnar
Cc: linux-kernel, Peter Zijlstra
[-- Attachment #1: perf-more-disable.patch --]
[-- Type: text/plain, Size: 2021 bytes --]
Assuming we don't mix events of different pmus onto a single context
(with the exeption of software events inside a hardware group) we can
now assume that all events on a particular context belong to the same
pmu, hence we can disable the pmu for the entire context operations.
This reduces the amount of hardware writes.
The exception for swevents comes from the fact that the sw pmu disable
is a nop.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
kernel/perf_event.c | 6 ++++++
1 file changed, 6 insertions(+)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -1046,6 +1046,7 @@ static void ctx_sched_out(struct perf_ev
struct perf_event *event;
raw_spin_lock(&ctx->lock);
+ perf_pmu_disable(ctx->pmu);
ctx->is_active = 0;
if (likely(!ctx->nr_events))
goto out;
@@ -1064,6 +1065,7 @@ static void ctx_sched_out(struct perf_ev
group_sched_out(event, cpuctx, ctx);
}
out:
+ perf_pmu_enable(ctx->pmu);
raw_spin_unlock(&ctx->lock);
}
@@ -1386,6 +1388,7 @@ void perf_event_context_sched_in(struct
if (cpuctx->task_ctx == ctx)
return;
+ perf_pmu_disable(ctx->pmu);
/*
* We want to keep the following priority order:
* cpu pinned (that don't need to move), task pinned,
@@ -1398,6 +1401,7 @@ void perf_event_context_sched_in(struct
ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE);
cpuctx->task_ctx = ctx;
+ perf_pmu_enable(ctx->pmu);
}
/*
@@ -1602,6 +1606,7 @@ static enum hrtimer_restart perf_event_c
rotate = 1;
}
+ perf_pmu_disable(cpuctx->ctx.pmu);
perf_ctx_adjust_freq(&cpuctx->ctx, cpuctx->timer_interval);
if (ctx)
perf_ctx_adjust_freq(ctx, cpuctx->timer_interval);
@@ -1622,6 +1627,7 @@ static enum hrtimer_restart perf_event_c
task_ctx_sched_in(ctx, EVENT_FLEXIBLE);
done:
+ perf_pmu_enable(cpuctx->ctx.pmu);
hrtimer_forward_now(timer, ns_to_ktime(cpuctx->timer_interval));
return restart;
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 00/19] perf pmu interface changes -v4
2010-09-07 16:46 [RFC][PATCH 00/19] perf pmu interface changes -v4 Peter Zijlstra
` (18 preceding siblings ...)
2010-09-07 16:46 ` [RFC][PATCH 19/19] perf: Optimize context ops Peter Zijlstra
@ 2010-09-10 4:39 ` Paul Mackerras
2010-09-10 7:43 ` Peter Zijlstra
2010-09-14 16:56 ` Peter Zijlstra
19 siblings, 2 replies; 29+ messages in thread
From: Paul Mackerras @ 2010-09-10 4:39 UTC (permalink / raw)
To: Peter Zijlstra
Cc: stephane eranian, Robert Richter, Will Deacon, Paul Mundt,
Frederic Weisbecker, Cyrill Gorcunov, Lin Ming, Yanmin,
Deng-Cheng Zhu, David Miller, Ingo Molnar, Michael Cree,
linux-kernel
On Tue, Sep 07, 2010 at 06:46:33PM +0200, Peter Zijlstra wrote:
> These patches prepare the perf code for multiple PMUs.
>
> About the first half of these patches deals with removing all the weak
> functions, the second half provides per pmu contexts.
>
> New to the first half is Alpha, which got merged into mainline recently.
>
> The second half is completely new and lightly tested, one known breakage is
> that it currently doesn't allow to mix software and hardware events into a
> single group (should be fixed soon).
I see Ingo committed this series into the tip perf/core branch. Did
this problem get fixed before that, or does it still need to be fixed?
Paul.
^ permalink raw reply [flat|nested] 29+ messages in thread* Re: [RFC][PATCH 00/19] perf pmu interface changes -v4
2010-09-10 4:39 ` [RFC][PATCH 00/19] perf pmu interface changes -v4 Paul Mackerras
@ 2010-09-10 7:43 ` Peter Zijlstra
2010-09-14 16:56 ` Peter Zijlstra
1 sibling, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-10 7:43 UTC (permalink / raw)
To: Paul Mackerras
Cc: stephane eranian, Robert Richter, Will Deacon, Paul Mundt,
Frederic Weisbecker, Cyrill Gorcunov, Lin Ming, Yanmin,
Deng-Cheng Zhu, David Miller, Ingo Molnar, Michael Cree,
linux-kernel
On Fri, 2010-09-10 at 14:39 +1000, Paul Mackerras wrote:
> > that it currently doesn't allow to mix software and hardware events into a
> > single group (should be fixed soon).
>
> I see Ingo committed this series into the tip perf/core branch. Did
> this problem get fixed before that, or does it still need to be fixed?
I thought I fixed it, but I realized I only fixed half of it.
The current state is that it allows to add software events to hardware
groups, but does not allow adding a hardware event to a pure software
group.
I'll try and get a patch done that attempts fixing that.
^ permalink raw reply [flat|nested] 29+ messages in thread
* Re: [RFC][PATCH 00/19] perf pmu interface changes -v4
2010-09-10 4:39 ` [RFC][PATCH 00/19] perf pmu interface changes -v4 Paul Mackerras
2010-09-10 7:43 ` Peter Zijlstra
@ 2010-09-14 16:56 ` Peter Zijlstra
1 sibling, 0 replies; 29+ messages in thread
From: Peter Zijlstra @ 2010-09-14 16:56 UTC (permalink / raw)
To: Paul Mackerras
Cc: stephane eranian, Robert Richter, Will Deacon, Paul Mundt,
Frederic Weisbecker, Cyrill Gorcunov, Lin Ming, Yanmin,
Deng-Cheng Zhu, David Miller, Ingo Molnar, Michael Cree,
linux-kernel
On Fri, 2010-09-10 at 14:39 +1000, Paul Mackerras wrote:
> On Tue, Sep 07, 2010 at 06:46:33PM +0200, Peter Zijlstra wrote:
>
> > These patches prepare the perf code for multiple PMUs.
> >
> > About the first half of these patches deals with removing all the weak
> > functions, the second half provides per pmu contexts.
> >
> > New to the first half is Alpha, which got merged into mainline recently.
> >
> > The second half is completely new and lightly tested, one known breakage is
> > that it currently doesn't allow to mix software and hardware events into a
> > single group (should be fixed soon).
>
> I see Ingo committed this series into the tip perf/core branch. Did
> this problem get fixed before that, or does it still need to be fixed?
I was thinking something like the below, except I obviously need to sort
that XXX thing.
---
kernel/perf_event.c | 48 ++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 40 insertions(+), 8 deletions(-)
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -494,7 +494,7 @@ static void __perf_event_remove_from_con
raw_spin_lock(&ctx->lock);
- event_sched_out(event, cpuctx, ctx);
+ group_sched_out(event, cpuctx, ctx);
list_del_event(event, ctx);
@@ -5522,9 +5522,10 @@ SYSCALL_DEFINE5(perf_event_open,
struct perf_event_context *ctx;
struct file *event_file = NULL;
struct file *group_file = NULL;
- struct task_struct *task;
+ struct task_struct *task = NULL;
struct pmu *pmu;
int event_fd;
+ int move_group_leader = 0;
int fput_needed = 0;
int err;
@@ -5574,15 +5575,34 @@ SYSCALL_DEFINE5(perf_event_open,
* any hardware group.
*/
pmu = event->pmu;
- if ((pmu->task_ctx_nr == perf_sw_context) && group_leader)
- pmu = group_leader->pmu;
+
+ if (group_leader &&
+ (is_software_event(event) != is_software_event(group_leader))) {
+ if (is_software_event(event)) {
+ /*
+ * If event and group_leader are not both a software
+ * event, and event it, then group leader is not.
+ *
+ * Allow the addition of software events to !software
+ * groups, this is safe because software events never
+ * fail to schedule.
+ */
+ pmu = group_leader->pmu;
+ } else if (is_software_event(group_leader) &&
+ (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
+ /*
+ * In case the group is a pure software group, and we
+ * try to add a hardware event, move the whole group to
+ * the hardware context.
+ */
+ move_group_leader = 1;
+ }
+ }
/*
* Get the target context (task or percpu):
*/
- if (pid == -1 && cpu != -1)
- task = NULL;
- else
+ if (!(pid == -1 && cpu != -1))
task = find_lively_task_by_vpid(pid);
ctx = find_get_context(pmu, task, cpu);
@@ -5606,8 +5626,10 @@ SYSCALL_DEFINE5(perf_event_open,
/*
* Do not allow to attach to a group in a different
* task or CPU context:
+ *
+ * XXX fix the task vs cpu context thing for move_group_leader
*/
- if (group_leader->ctx != ctx)
+ if (!move_group_leader && group_leader->ctx != ctx)
goto err_context;
/*
* Only a group leader can be exclusive or pinned
@@ -5628,9 +5650,19 @@ SYSCALL_DEFINE5(perf_event_open,
goto err_context;
}
+ if (move_group_leader) {
+ struct perf_event_context *gctx = group_leader->ctx;
+
+ mutex_lock(&gctx->mutex);
+ perf_event_remove_from_context(group_leader);
+ mutex_unlock(&gctx->mutex);
+ }
+
event->filp = event_file;
WARN_ON_ONCE(ctx->parent_ctx);
mutex_lock(&ctx->mutex);
+ if (move_group_leader)
+ perf_install_in_context(ctx, group_leader, cpu);
perf_install_in_context(ctx, event, cpu);
++ctx->generation;
mutex_unlock(&ctx->mutex);
^ permalink raw reply [flat|nested] 29+ messages in thread