[RFC][PATCH 0/7] perf and jump_label bits

[RFC][PATCH 0/7] perf and jump_label bits

[Peter Zijlstra]

Only compile tested, but should fix the pending hw_breakpoint issue and
use jump_labels to optimize some of perf

[RFC][PATCH 1/7] perf: Fix task refcount issues

[Peter Zijlstra]

[-- Attachment #1: perf-fix-hw_bp.patch --]
[-- Type: text/plain, Size: 1193 bytes --]

Currently it looks like find_lively_task_by_vpid() takes a task ref
and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be
dropping task refs.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Matt Helsley <matthltc@us.ibm.com>
---
 kernel/perf_event.c |    7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);

[RFC][PATCH 2/7] perf: Find task before event alloc

[Peter Zijlstra]

[-- Attachment #1: perf-fix-hw_bp-1.patch --]
[-- Type: text/plain, Size: 1561 bytes --]

So that we can pass the task pointer to the event allocation, so that
we can use task associated data during event initialization.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 kernel/perf_event.c |   23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -5485,10 +5485,18 @@ SYSCALL_DEFINE5(perf_event_open,
 			group_leader = NULL;
 	}
 
+	if (pid != -1) {
+		task = find_lively_task_by_vpid(pid);
+		if (IS_ERR(task)) {
+			err = PTR_ERR(task);
+			goto err_group_fd;
+		}
+	}
+
 	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
-		goto err_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5520,21 +5528,13 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	if (pid != -1) {
-		task = find_lively_task_by_vpid(pid);
-		if (IS_ERR(task)) {
-			err = PTR_ERR(task);
-			goto err_group_fd;
-		}
-	}
-
 	/*
 	 * Get the target context (task or percpu):
 	 */
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_task;
+		goto err_alloc;
 	}
 
 	/*
@@ -5630,12 +5630,13 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_alloc:
+	free_event(event);
 err_task:
 	if (task)
 		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
-	free_event(event);
 err_fd:
 	put_unused_fd(event_fd);
 	return err;

[RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

[Peter Zijlstra]

[-- Attachment #1: perf-fix-hw_bp-2.patch --]
[-- Type: text/plain, Size: 4708 bytes --]

hw_breakpoint creation needs to account stuff per-task to ensure there
is always sufficient hardware resources to back these things due to
ptrace.

With the perf per pmu context changes the event initialization no
longer has access to the event context, for the simple reason that we
need to first find the pmu (result of initialization) before we can
find the context.

This makes hw_breakpoints unhappy, because it can no longer do per
task accounting, cure this by frobbing a task pointer in the event::hw
bits for now...

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
---
 include/linux/perf_event.h |    7 +++++++
 kernel/hw_breakpoint.c     |    8 ++++----
 kernel/perf_event.c        |   21 ++++++++++++++++-----
 3 files changed, 27 insertions(+), 9 deletions(-)

Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -536,6 +536,12 @@ struct hw_perf_event {
 		struct { /* breakpoint */
 			struct arch_hw_breakpoint	info;
 			struct list_head		bp_list;
+			/*
+			 * Crufty hack to avoid the chicken and egg
+			 * problem hw_breakpoint has with context
+			 * creation and event initalization.
+			 */
+			struct task_struct		*bp_target;
 		};
 #endif
 	};
@@ -693,6 +699,7 @@ struct swevent_hlist {
 
 #define PERF_ATTACH_CONTEXT	0x01
 #define PERF_ATTACH_GROUP	0x02
+#define PERF_ATTACH_TASK	0x04
 
 /**
  * struct perf_event - performance event kernel representation:
Index: linux-2.6/kernel/hw_breakpoint.c
===================================================================
--- linux-2.6.orig/kernel/hw_breakpoint.c
+++ linux-2.6/kernel/hw_breakpoint.c
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(i
  */
 static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
 {
-	struct perf_event_context *ctx = bp->ctx;
+	struct task_struct *tsk = bp->hw.bp_target;
 	struct perf_event *iter;
 	int count = 0;
 
 	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-		if (iter->ctx == ctx && find_slot_idx(iter) == type)
+		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
 			count += hw_breakpoint_weight(iter);
 	}
 
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots
 		    enum bp_type_idx type)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	if (cpu >= 0) {
 		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bo
 	       int weight)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	/* Pinned counter cpu profiling */
 	if (!tsk) {
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -5189,9 +5189,10 @@ struct pmu *perf_init_event(struct perf_
  */
 static struct perf_event *
 perf_event_alloc(struct perf_event_attr *attr, int cpu,
-		   struct perf_event *group_leader,
-		   struct perf_event *parent_event,
-		   perf_overflow_handler_t overflow_handler)
+		 struct task_struct *task,
+		 struct perf_event *group_leader,
+		 struct perf_event *parent_event,
+		 perf_overflow_handler_t overflow_handler)
 {
 	struct pmu *pmu;
 	struct perf_event *event;
@@ -5233,6 +5234,15 @@ perf_event_alloc(struct perf_event_attr 
 
 	event->state		= PERF_EVENT_STATE_INACTIVE;
 
+	if (task) {
+		event->attach_state = PERF_ATTACH_TASK;
+		/*
+		 * hw_breakpoint is a bit difficult here..
+		 */
+		if (attr->type == PERF_TYPE_BREAKPOINT)
+			event->hw.bp_target = task;
+	}
+
 	if (!overflow_handler && parent_event)
 		overflow_handler = parent_event->overflow_handler;
 	
@@ -5493,7 +5503,7 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
+	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err_task;
@@ -5662,7 +5672,7 @@ perf_event_create_kernel_counter(struct 
 	 * Get the target context (task or percpu):
 	 */
 
-	event = perf_event_alloc(attr, cpu, NULL, NULL, overflow_handler);
+	event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err;
@@ -5930,6 +5940,7 @@ inherit_event(struct perf_event *parent_
 
 	child_event = perf_event_alloc(&parent_event->attr,
 					   parent_event->cpu,
+					   child,
 					   group_leader, parent_event,
 					   NULL);
 	if (IS_ERR(child_event))

[RFC][PATCH 4/7] jump_label: More consitent naming

[Peter Zijlstra]

[-- Attachment #1: jump_label_rename.patch --]
[-- Type: text/plain, Size: 2619 bytes --]

Now that there's only a few users around, rename things.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 include/linux/jump_label.h |    8 ++++----
 kernel/tracepoint.c        |    6 +++---
 lib/dynamic_debug.c        |    4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

Index: linux-2.6/include/linux/jump_label.h
===================================================================
--- linux-2.6.orig/include/linux/jump_label.h
+++ linux-2.6/include/linux/jump_label.h
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned l
 extern void jump_label_apply_nops(struct module *mod);
 extern int jump_label_text_reserved(void *start, void *end);
 
-#define enable_jump_label(key) \
+#define jump_label_enable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
 
-#define disable_jump_label(key) \
+#define jump_label_disable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
 
 #else
@@ -39,12 +39,12 @@ do {						\
 		goto label;			\
 } while (0)
 
-#define enable_jump_label(cond_var)	\
+#define jump_label_enable(cond_var)	\
 do {					\
        *(cond_var) = 1;			\
 } while (0)
 
-#define disable_jump_label(cond_var)	\
+#define jump_label_disable(cond_var)	\
 do {					\
        *(cond_var) = 0;			\
 } while (0)
Index: linux-2.6/kernel/tracepoint.c
===================================================================
--- linux-2.6.orig/kernel/tracepoint.c
+++ linux-2.6/kernel/tracepoint.c
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracep
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
 	if (!elem->state && active) {
-		enable_jump_label(&elem->state);
+		jump_label_enable(&elem->state);
 		elem->state = active;
 	} else if (elem->state && !active) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = active;
 	}
 }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tr
 		elem->unregfunc();
 
 	if (elem->state) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = 0;
 	}
 	rcu_assign_pointer(elem->funcs, NULL);
Index: linux-2.6/lib/dynamic_debug.c
===================================================================
--- linux-2.6.orig/lib/dynamic_debug.c
+++ linux-2.6/lib/dynamic_debug.c
@@ -142,9 +142,9 @@ static void ddebug_change(const struct d
 				dt->num_enabled++;
 			dp->flags = newflags;
 			if (newflags) {
-				enable_jump_label(&dp->enabled);
+				jump_label_enable(&dp->enabled);
 			} else {
-				disable_jump_label(&dp->enabled);
+				jump_label_disable(&dp->enabled);
 			}
 			if (verbose)
 				printk(KERN_INFO

[RFC][PATCH 5/7] jump_label: atomic_t interface

[Peter Zijlstra]

[-- Attachment #1: jump_label_ref.patch --]
[-- Type: text/plain, Size: 1360 bytes --]

Add an interface to allow usage of jump_labels with atomic counters

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 include/linux/jump_label_ref.h |   44 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

Index: linux-2.6/include/linux/jump_label_ref.h
===================================================================
--- /dev/null
+++ linux-2.6/include/linux/jump_label_ref.h
@@ -0,0 +1,44 @@
+#ifndef _LINUX_JUMP_LABEL_REF_H
+#define _LINUX_JUMP_LABEL_REF_H
+
+#include <linux/jump_label.h>
+#include <asm/atomic.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	if (atomic_add_return(1, key) == 1)
+		jump_label_enable(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	if (atomic_dec_and_test(key))
+		jump_label_disable(key);
+}
+
+#else /* !HAVE_JUMP_LABEL */
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	atomic_inc(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	atomic_dec(key);
+}
+
+#undef JUMP_LABEL
+#define JUMP_LABEL(key, label)						\
+do {									\
+	if (unlikely(__builtin_choose_expr(				\
+	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
+	      atomic_read((atomic_t *)(key)), *(key))))			\
+		goto label;						\
+} while (0)
+
+#endif /* HAVE_JUMP_LABEL */
+
+#endif /* _LINUX_JUMP_LABEL_REF_H */

[RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

[Peter Zijlstra]

[-- Attachment #1: perf-jump-label.patch --]
[-- Type: text/plain, Size: 4239 bytes --]


Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 include/linux/perf_event.h |   27 +++++++++++++++++++++++++--
 kernel/perf_event.c        |   24 +++++++++---------------
 2 files changed, 34 insertions(+), 17 deletions(-)

Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/irq_work.h>
+#include <linux/jump_label_ref.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
 
@@ -893,8 +894,30 @@ struct perf_output_handle {
 extern int perf_pmu_register(struct pmu *pmu);
 extern void perf_pmu_unregister(struct pmu *pmu);
 
-extern void perf_event_task_sched_in(struct task_struct *task);
-extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+extern void __perf_event_task_sched_in(struct task_struct *task);
+extern void __perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+
+extern atomic_t perf_task_events;
+
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_in(task);
+}
+
+static inline
+void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_out(task, next);
+}
+
 extern int perf_event_init_task(struct task_struct *child);
 extern void perf_event_exit_task(struct task_struct *child);
 extern void perf_event_free_task(struct task_struct *task);
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -34,7 +34,7 @@
 
 #include <asm/irq_regs.h>
 
-static atomic_t nr_events __read_mostly;
+atomic_t perf_task_events __read_mostly;
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
@@ -1251,8 +1251,8 @@ void perf_event_context_sched_out(struct
  * accessing the event control register. If a NMI hits, then it will
  * not restart the event.
  */
-void perf_event_task_sched_out(struct task_struct *task,
-			       struct task_struct *next)
+void __perf_event_task_sched_out(struct task_struct *task,
+				 struct task_struct *next)
 {
 	int ctxn;
 
@@ -1280,14 +1280,6 @@ static void task_ctx_sched_out(struct pe
 /*
  * Called with IRQs disabled
  */
-static void __perf_event_task_sched_out(struct perf_event_context *ctx)
-{
-	task_ctx_sched_out(ctx, EVENT_ALL);
-}
-
-/*
- * Called with IRQs disabled
- */
 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 			      enum event_type_t event_type)
 {
@@ -1434,7 +1426,7 @@ void perf_event_context_sched_in(struct 
  * accessing the event control register. If a NMI hits, then it will
  * keep the event running.
  */
-void perf_event_task_sched_in(struct task_struct *task)
+void __perf_event_task_sched_in(struct task_struct *task)
 {
 	struct perf_event_context *ctx;
 	int ctxn;
@@ -2150,7 +2142,8 @@ static void free_event(struct perf_event
 	irq_work_sync(&event->pending);
 
 	if (!event->parent) {
-		atomic_dec(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_dec(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_dec(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5286,7 +5279,8 @@ perf_event_alloc(struct perf_event_attr 
 	event->pmu = pmu;
 
 	if (!event->parent) {
-		atomic_inc(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_inc(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_inc(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5781,7 +5775,7 @@ static void perf_event_exit_task_context
 	 * our context.
 	 */
 	child_ctx = child->perf_event_ctxp[ctxn];
-	__perf_event_task_sched_out(child_ctx);
+	task_ctx_sched_out(child_ctx, EVENT_ALL);
 
 	/*
 	 * Take the context lock here so that if find_get_context is

[RFC][PATCH 7/7] perf: Optimize sw events

[Peter Zijlstra]

[-- Attachment #1: perf-jump-label-2.patch --]
[-- Type: text/plain, Size: 1734 bytes --]


Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 include/linux/perf_event.h |   16 +++++++++-------
 kernel/perf_event.c        |    4 ++--
 2 files changed, 11 insertions(+), 9 deletions(-)

Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
 static inline void
 perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
-	if (atomic_read(&perf_swevent_enabled[event_id])) {
-		struct pt_regs hot_regs;
+	struct pt_regs hot_regs;
 
-		if (!regs) {
-			perf_fetch_caller_regs(&hot_regs);
-			regs = &hot_regs;
-		}
-		__perf_sw_event(event_id, nr, nmi, regs, addr);
+	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
+	return;
+
+have_event:
+	if (!regs) {
+		perf_fetch_caller_regs(&hot_regs);
+		regs = &hot_regs;
 	}
+	__perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 
 extern void perf_event_mmap(struct vm_area_struct *vma);
Index: linux-2.6/kernel/perf_event.c
===================================================================
--- linux-2.6.orig/kernel/perf_event.c
+++ linux-2.6/kernel/perf_event.c
@@ -4603,7 +4603,7 @@ static void sw_perf_event_destroy(struct
 
 	WARN_ON(event->parent);
 
-	atomic_dec(&perf_swevent_enabled[event_id]);
+	jump_label_dec(&perf_swevent_enabled[event_id]);
 	swevent_hlist_put(event);
 }
 
@@ -4633,7 +4633,7 @@ static int perf_swevent_init(struct perf
 		if (err)
 			return err;
 
-		atomic_inc(&perf_swevent_enabled[event_id]);
+		jump_label_inc(&perf_swevent_enabled[event_id]);
 		event->destroy = sw_perf_event_destroy;
 	}
 

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Peter Zijlstra]

On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:

> Index: linux-2.6/include/linux/perf_event.h
> ===================================================================
> --- linux-2.6.orig/include/linux/perf_event.h
> +++ linux-2.6/include/linux/perf_event.h
> @@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
>  static inline void
>  perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
>  {
> -	if (atomic_read(&perf_swevent_enabled[event_id])) {
> -		struct pt_regs hot_regs;
> +	struct pt_regs hot_regs;
>  
> -		if (!regs) {
> -			perf_fetch_caller_regs(&hot_regs);
> -			regs = &hot_regs;
> -		}
> -		__perf_sw_event(event_id, nr, nmi, regs, addr);
> +	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
> +	return;
> +
> +have_event:
> +	if (!regs) {
> +		perf_fetch_caller_regs(&hot_regs);
> +		regs = &hot_regs;
>  	}
> +	__perf_sw_event(event_id, nr, nmi, regs, addr);
>  }
>  

OK, so it appears I only compile tested this bit without jump_label
support, with that bit added back this horribly fails to compile like:

In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
/usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
/usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints
/usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’


The relevant snippet from the .i file reads:

static inline void
perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
{
 struct pt_regs hot_regs;

 do { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table,  \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : : have_event); } while (0);
 return;

have_event:
 if (!regs) {
  perf_fetch_caller_regs(&hot_regs);
  regs = &hot_regs;
 }
 __perf_sw_event(event_id, nr, nmi, regs, addr);
}


Anybody got any clue as to why this goes splat?

I tried both:

gcc (GCC) 4.4.4 20100630 (Red Hat 4.4.4-10)

and

x86_64-linux-gcc (GCC) 4.5.1

Re: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

[Frederic Weisbecker]

On Thu, Oct 14, 2010 at 10:34:07PM +0200, Peter Zijlstra wrote:
> hw_breakpoint creation needs to account stuff per-task to ensure there
> is always sufficient hardware resources to back these things due to
> ptrace.
> 
> With the perf per pmu context changes the event initialization no
> longer has access to the event context, for the simple reason that we
> need to first find the pmu (result of initialization) before we can
> find the context.
> 
> This makes hw_breakpoints unhappy, because it can no longer do per
> task accounting, cure this by frobbing a task pointer in the event::hw
> bits for now...
> 
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> ---
>  include/linux/perf_event.h |    7 +++++++
>  kernel/hw_breakpoint.c     |    8 ++++----
>  kernel/perf_event.c        |   21 ++++++++++++++++-----
>  3 files changed, 27 insertions(+), 9 deletions(-)
> 
> Index: linux-2.6/include/linux/perf_event.h
> ===================================================================
> --- linux-2.6.orig/include/linux/perf_event.h
> +++ linux-2.6/include/linux/perf_event.h
> @@ -536,6 +536,12 @@ struct hw_perf_event {
>  		struct { /* breakpoint */
>  			struct arch_hw_breakpoint	info;
>  			struct list_head		bp_list;
> +			/*
> +			 * Crufty hack to avoid the chicken and egg
> +			 * problem hw_breakpoint has with context
> +			 * creation and event initalization.
> +			 */
> +			struct task_struct		*bp_target;
>  		};
>  #endif
>  	};
> @@ -693,6 +699,7 @@ struct swevent_hlist {
>  
>  #define PERF_ATTACH_CONTEXT	0x01
>  #define PERF_ATTACH_GROUP	0x02
> +#define PERF_ATTACH_TASK	0x04



Thanks, the patch looks good.

I'm just not sure about the point of this flag...

Re: [RFC][PATCH 3/7] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

[Peter Zijlstra]

On Fri, 2010-10-15 at 15:47 +0200, Frederic Weisbecker wrote:

> > +#define PERF_ATTACH_TASK	0x04
> 
> 
> 
> Thanks, the patch looks good.
> 
> I'm just not sure about the point of this flag...

Ah, see 6/7..

Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

[Frederic Weisbecker]

On Thu, Oct 14, 2010 at 10:34:10PM +0200, Peter Zijlstra wrote:
> 
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>



Cool!

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>

Re: [RFC][PATCH 5/7] jump_label: atomic_t interface

[Frederic Weisbecker]

On Thu, Oct 14, 2010 at 10:34:09PM +0200, Peter Zijlstra wrote:
> Add an interface to allow usage of jump_labels with atomic counters
> 
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> ---
>  include/linux/jump_label_ref.h |   44 +++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 44 insertions(+)
> 
> Index: linux-2.6/include/linux/jump_label_ref.h
> ===================================================================
> --- /dev/null
> +++ linux-2.6/include/linux/jump_label_ref.h
> @@ -0,0 +1,44 @@
> +#ifndef _LINUX_JUMP_LABEL_REF_H
> +#define _LINUX_JUMP_LABEL_REF_H
> +
> +#include <linux/jump_label.h>
> +#include <asm/atomic.h>
> +
> +#ifdef HAVE_JUMP_LABEL
> +
> +static inline void jump_label_inc(atomic_t *key)
> +{
> +	if (atomic_add_return(1, key) == 1)
> +		jump_label_enable(key);
> +}
> +
> +static inline void jump_label_dec(atomic_t *key)
> +{
> +	if (atomic_dec_and_test(key))
> +		jump_label_disable(key);
> +}



Nice. I was trying to find a more self-explanatory name,
like jump_label_get/put or jump_label_inc_enable/jump_label_dec_disable.

But in fact the current name looks good eventually.

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>




> +#else /* !HAVE_JUMP_LABEL */
> +
> +static inline void jump_label_inc(atomic_t *key)
> +{
> +	atomic_inc(key);
> +}
> +
> +static inline void jump_label_dec(atomic_t *key)
> +{
> +	atomic_dec(key);
> +}
> +
> +#undef JUMP_LABEL
> +#define JUMP_LABEL(key, label)						\
> +do {									\
> +	if (unlikely(__builtin_choose_expr(				\
> +	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
> +	      atomic_read((atomic_t *)(key)), *(key))))			\
> +		goto label;						\
> +} while (0)
> +
> +#endif /* HAVE_JUMP_LABEL */
> +
> +#endif /* _LINUX_JUMP_LABEL_REF_H */
> 
> 

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Frederic Weisbecker]

On Thu, Oct 14, 2010 at 10:34:11PM +0200, Peter Zijlstra wrote:
> 
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>



Acked-by: Frederic Weisbecker <fweisbec@gmail.com>

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Peter Zijlstra]

On Fri, 2010-10-15 at 16:04 +0200, Frederic Weisbecker wrote:
> On Thu, Oct 14, 2010 at 10:34:11PM +0200, Peter Zijlstra wrote:
> > 
> > Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> 
> 
> 
> Acked-by: Frederic Weisbecker <fweisbec@gmail.com>


Shall we get it compiling first? ;-)

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Frederic Weisbecker]

On Fri, Oct 15, 2010 at 04:08:27PM +0200, Peter Zijlstra wrote:
> On Fri, 2010-10-15 at 16:04 +0200, Frederic Weisbecker wrote:
> > On Thu, Oct 14, 2010 at 10:34:11PM +0200, Peter Zijlstra wrote:
> > > 
> > > Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> > 
> > 
> > 
> > Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
> 
> 
> Shall we get it compiling first? ;-)


Why bother ;-)

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Jason Baron]

On Fri, Oct 15, 2010 at 11:14:03AM +0200, Peter Zijlstra wrote:
> > Index: linux-2.6/include/linux/perf_event.h
> > ===================================================================
> > --- linux-2.6.orig/include/linux/perf_event.h
> > +++ linux-2.6/include/linux/perf_event.h
> > @@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
> >  static inline void
> >  perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
> >  {
> > -	if (atomic_read(&perf_swevent_enabled[event_id])) {
> > -		struct pt_regs hot_regs;
> > +	struct pt_regs hot_regs;
> >  
> > -		if (!regs) {
> > -			perf_fetch_caller_regs(&hot_regs);
> > -			regs = &hot_regs;
> > -		}
> > -		__perf_sw_event(event_id, nr, nmi, regs, addr);
> > +	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
> > +	return;
> > +
> > +have_event:
> > +	if (!regs) {
> > +		perf_fetch_caller_regs(&hot_regs);
> > +		regs = &hot_regs;
> >  	}
> > +	__perf_sw_event(event_id, nr, nmi, regs, addr);
> >  }
> >  
> 
> OK, so it appears I only compile tested this bit without jump_label
> support, with that bit added back this horribly fails to compile like:
> 
> In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
> /usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
> /usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints
> /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
> 
> 
> The relevant snippet from the .i file reads:
> 
> static inline void
> perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
> {
>  struct pt_regs hot_regs;
> 
>  do { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table,  \"a\" \n\t" " " ".quad" " " "1b, %l[" "have_event" "], %c0 \n\t" ".popsection \n\t" : : "i" (&perf_swevent_enabled[event_id]) : : have_event); } while (0);
>  return;
> 
> have_event:
>  if (!regs) {
>   perf_fetch_caller_regs(&hot_regs);
>   regs = &hot_regs;
>  }
>  __perf_sw_event(event_id, nr, nmi, regs, addr);
> }
> 
> 
> Anybody got any clue as to why this goes splat?
> 

Couple issues here. The key value: "&perf_swevent_enabled[event_id]"
is determined at run-time, so we can't associated the jump label with
keys that are run-time dependent. The second thing is that there is
array indexing, so essentially the jump label code would need to be
smart enough to associated the correct index with each instance of
'perf_sw_event' - unfortunately its not that smart :(

So, two possible suggestions:

1)

I see that PERF_COUNT_SW_MAX is 9. So, we could explicitly split out the
9 cases into 9 separate inline functions where the keys would be:

&perf_swevent_enabled[PERF_COUNT_SW_CPU_CLOCK]
&perf_swevent_enabled[PERF_COUNT_SW_TASK_CLOCK]

this probably wouldn't look to ugly if there was a macro that took each
software id, and spit out the appropriate inline. I see there aren't
that many 'perf_sw_event()' calls either to update.

2)

If you could define a single global variable that was set if any of the
indexes of the perf_swevent_enabled[] array were set. This potentially
woulnd't be as efficient as #1 as it would create false positives.

thanks,

-Jason

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Peter Zijlstra]

On Fri, 2010-10-15 at 11:16 +0200, Peter Zijlstra wrote:
> On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:
> 
> > Index: linux-2.6/include/linux/perf_event.h
> > ===================================================================
> > --- linux-2.6.orig/include/linux/perf_event.h
> > +++ linux-2.6/include/linux/perf_event.h
> > @@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
> >  static inline void
> >  perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
> >  {
> > -	if (atomic_read(&perf_swevent_enabled[event_id])) {
> > -		struct pt_regs hot_regs;
> > +	struct pt_regs hot_regs;
> >  
> > -		if (!regs) {
> > -			perf_fetch_caller_regs(&hot_regs);
> > -			regs = &hot_regs;
> > -		}
> > -		__perf_sw_event(event_id, nr, nmi, regs, addr);
> > +	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
> > +	return;
> > +
> > +have_event:
> > +	if (!regs) {
> > +		perf_fetch_caller_regs(&hot_regs);
> > +		regs = &hot_regs;
> >  	}
> > +	__perf_sw_event(event_id, nr, nmi, regs, addr);
> >  }
> >  
> 
> OK, so it appears I only compile tested this bit without jump_label
> support, with that bit added back this horribly fails to compile like:
> 
> In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
> /usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
> /usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints
> /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
> 

Ah, figured it out, it needs __attribute__((always_inline)), otherwise
we end up with multiple keys for one label, which will clealy not work.

I thought we had a __force_inline macro but I can't seem to find that.

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Jason Baron]

On Fri, Oct 15, 2010 at 04:57:07PM +0200, Peter Zijlstra wrote:
> On Fri, 2010-10-15 at 11:16 +0200, Peter Zijlstra wrote:
> > On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:
> > 
> > > Index: linux-2.6/include/linux/perf_event.h
> > > ===================================================================
> > > --- linux-2.6.orig/include/linux/perf_event.h
> > > +++ linux-2.6/include/linux/perf_event.h
> > > @@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
> > >  static inline void
> > >  perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
> > >  {
> > > -	if (atomic_read(&perf_swevent_enabled[event_id])) {
> > > -		struct pt_regs hot_regs;
> > > +	struct pt_regs hot_regs;
> > >  
> > > -		if (!regs) {
> > > -			perf_fetch_caller_regs(&hot_regs);
> > > -			regs = &hot_regs;
> > > -		}
> > > -		__perf_sw_event(event_id, nr, nmi, regs, addr);
> > > +	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
> > > +	return;
> > > +
> > > +have_event:
> > > +	if (!regs) {
> > > +		perf_fetch_caller_regs(&hot_regs);
> > > +		regs = &hot_regs;
> > >  	}
> > > +	__perf_sw_event(event_id, nr, nmi, regs, addr);
> > >  }
> > >  
> > 
> > OK, so it appears I only compile tested this bit without jump_label
> > support, with that bit added back this horribly fails to compile like:
> > 
> > In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
> > /usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
> > /usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints
> > /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
> > 
> 
> Ah, figured it out, it needs __attribute__((always_inline)), otherwise
> we end up with multiple keys for one label, which will clealy not work.
> 

ah...cool! that should work. ignore my other suggestions...

thanks,

-Jason

Re: [RFC][PATCH 1/7] perf: Fix task refcount issues

[Frederic Weisbecker]

On Thu, Oct 14, 2010 at 10:34:05PM +0200, Peter Zijlstra wrote:
> Currently it looks like find_lively_task_by_vpid() takes a task ref
> and relies on find_get_context() to drop it.
> 
> The problem is that perf_event_create_kernel_counter() shouldn't be
> dropping task refs.
> 
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: Matt Helsley <matthltc@us.ibm.com>


Looks good.

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>



> ---
>  kernel/perf_event.c |    7 ++++---
>  1 file changed, 4 insertions(+), 3 deletions(-)
> 
> Index: linux-2.6/kernel/perf_event.c
> ===================================================================
> --- linux-2.6.orig/kernel/perf_event.c
> +++ linux-2.6/kernel/perf_event.c
> @@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
>  		}
>  	}
>  
> -	put_task_struct(task);
>  	return ctx;
>  
>  errout:
> -	put_task_struct(task);
>  	return ERR_PTR(err);
>  }
>  
> @@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
>  	ctx = find_get_context(pmu, task, cpu);
>  	if (IS_ERR(ctx)) {
>  		err = PTR_ERR(ctx);
> -		goto err_group_fd;
> +		goto err_task;
>  	}
>  
>  	/*
> @@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
>  
>  err_context:
>  	put_ctx(ctx);
> +err_task:
> +	if (task)
> +		put_task_struct(task);
>  err_group_fd:
>  	fput_light(group_file, fput_needed);
>  	free_event(event);
> 
> 

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Steven Rostedt]

On Fri, 2010-10-15 at 16:57 +0200, Peter Zijlstra wrote:

> Ah, figured it out, it needs __attribute__((always_inline)), otherwise
> we end up with multiple keys for one label, which will clealy not work.
> 
> I thought we had a __force_inline macro but I can't seem to find that.

Do you mean __always_inline?

	See include/linux/compiler-gcc4.h

-- Steve

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Peter Zijlstra]

On Fri, 2010-10-15 at 15:32 -0400, Steven Rostedt wrote:
> On Fri, 2010-10-15 at 16:57 +0200, Peter Zijlstra wrote:
> 
> > Ah, figured it out, it needs __attribute__((always_inline)), otherwise
> > we end up with multiple keys for one label, which will clealy not work.
> > 
> > I thought we had a __force_inline macro but I can't seem to find that.
> 
> Do you mean __always_inline?
> 
> 	See include/linux/compiler-gcc4.h

Yeah, cyrillos already pointed that out..

Re: [RFC][PATCH 1/7] perf: Fix task refcount issues

[Matt Helsley]

On Thu, Oct 14, 2010 at 10:34:05PM +0200, Peter Zijlstra wrote:
> Currently it looks like find_lively_task_by_vpid() takes a task ref
> and relies on find_get_context() to drop it.
> 
> The problem is that perf_event_create_kernel_counter() shouldn't be
> dropping task refs.

Doh.

Looks good.

Acked-by: Matt Helsley <matthltc@us.ibm.com>

> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: Matt Helsley <matthltc@us.ibm.com>
> ---
>  kernel/perf_event.c |    7 ++++---
>  1 file changed, 4 insertions(+), 3 deletions(-)
> 
> Index: linux-2.6/kernel/perf_event.c
> ===================================================================
> --- linux-2.6.orig/kernel/perf_event.c
> +++ linux-2.6/kernel/perf_event.c
> @@ -2124,11 +2124,9 @@ find_get_context(struct pmu *pmu, struct
>  		}
>  	}
> 
> -	put_task_struct(task);
>  	return ctx;
> 
>  errout:
> -	put_task_struct(task);
>  	return ERR_PTR(err);
>  }
> 
> @@ -5536,7 +5534,7 @@ SYSCALL_DEFINE5(perf_event_open,
>  	ctx = find_get_context(pmu, task, cpu);
>  	if (IS_ERR(ctx)) {
>  		err = PTR_ERR(ctx);
> -		goto err_group_fd;
> +		goto err_task;
>  	}
> 
>  	/*
> @@ -5632,6 +5630,9 @@ SYSCALL_DEFINE5(perf_event_open,
> 
>  err_context:
>  	put_ctx(ctx);
> +err_task:
> +	if (task)
> +		put_task_struct(task);
>  err_group_fd:
>  	fput_light(group_file, fput_needed);
>  	free_event(event);
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [RFC][PATCH 7/7] perf: Optimize sw events

[Ingo Molnar]

* Peter Zijlstra <peterz@infradead.org> wrote:

> On Fri, 2010-10-15 at 11:16 +0200, Peter Zijlstra wrote:
> > On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:
> > 
> > > Index: linux-2.6/include/linux/perf_event.h
> > > ===================================================================
> > > --- linux-2.6.orig/include/linux/perf_event.h
> > > +++ linux-2.6/include/linux/perf_event.h
> > > @@ -1013,15 +1013,17 @@ static inline void perf_fetch_caller_reg
> > >  static inline void
> > >  perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
> > >  {
> > > -	if (atomic_read(&perf_swevent_enabled[event_id])) {
> > > -		struct pt_regs hot_regs;
> > > +	struct pt_regs hot_regs;
> > >  
> > > -		if (!regs) {
> > > -			perf_fetch_caller_regs(&hot_regs);
> > > -			regs = &hot_regs;
> > > -		}
> > > -		__perf_sw_event(event_id, nr, nmi, regs, addr);
> > > +	JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
> > > +	return;
> > > +
> > > +have_event:
> > > +	if (!regs) {
> > > +		perf_fetch_caller_regs(&hot_regs);
> > > +		regs = &hot_regs;
> > >  	}
> > > +	__perf_sw_event(event_id, nr, nmi, regs, addr);
> > >  }
> > >  
> > 
> > OK, so it appears I only compile tested this bit without jump_label
> > support, with that bit added back this horribly fails to compile like:
> > 
> > In file included from /usr/src/linux-2.6/arch/x86/mm/fault.c:13:
> > /usr/src/linux-2.6/include/linux/perf_event.h: In function ‘perf_sw_event.clone.0’:
> > /usr/src/linux-2.6/include/linux/perf_event.h:1018: warning: asm operand 0 probably doesn’t match constraints
> > /usr/src/linux-2.6/include/linux/perf_event.h:1018: error: impossible constraint in ‘asm’
> > 
> 
> Ah, figured it out, it needs __attribute__((always_inline)), otherwise
> we end up with multiple keys for one label, which will clealy not work.
> 
> I thought we had a __force_inline macro but I can't seem to find that.

It's called __always_inline.

Thanks,

	Ingo

Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

[Peter Zijlstra]

On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:
> +static inline void perf_event_task_sched_in(struct task_struct *task)
> +{
> +       JUMP_LABEL(&perf_task_events, have_events);
> +       return;
> +
> +have_events:
> +       __perf_event_task_sched_in(task);
> +} 

OK, so I hate the JUMP_LABEL() interface for it means we have to keep
writing these silly stubs.

How about something like:

#define COND_STMT(key, stmt)					\
do {								\
	__label__ jl_enabled;					\
	JUMP_LABEL(key, jl_enabled);				\
	if (0) {						\
jl_enabled:							\
		stmt;						\
	}							\
} while (0)

Re: [RFC][PATCH 6/7] perf: use jump_label to optimize the scheduler hooks

[Peter Zijlstra]

On Sun, 2010-10-17 at 11:52 +0200, Peter Zijlstra wrote:
> On Thu, 2010-10-14 at 22:34 +0200, Peter Zijlstra wrote:
> > +static inline void perf_event_task_sched_in(struct task_struct *task)
> > +{
> > +       JUMP_LABEL(&perf_task_events, have_events);
> > +       return;
> > +
> > +have_events:
> > +       __perf_event_task_sched_in(task);
> > +} 
> 
> OK, so I hate the JUMP_LABEL() interface for it means we have to keep
> writing these silly stubs.

The below seems to actually compile too, awesome! ;-)

---
Index: linux-2.6/include/linux/jump_label.h
===================================================================
--- linux-2.6.orig/include/linux/jump_label.h
+++ linux-2.6/include/linux/jump_label.h
@@ -61,4 +61,14 @@ static inline int jump_label_text_reserv
 
 #endif
 
+#define COND_STMT(key, stmt)					\
+do {								\
+	__label__ jl_enabled;					\
+	JUMP_LABEL(key, jl_enabled);				\
+	if (0) {						\
+jl_enabled:							\
+		stmt;						\
+	}							\
+} while (0)
+
 #endif
Index: linux-2.6/include/linux/perf_event.h
===================================================================
--- linux-2.6.orig/include/linux/perf_event.h
+++ linux-2.6/include/linux/perf_event.h
@@ -901,21 +901,13 @@ extern atomic_t perf_task_events;
 
 static inline void perf_event_task_sched_in(struct task_struct *task)
 {
-	JUMP_LABEL(&perf_task_events, have_events);
-	return;
-
-have_events:
-	__perf_event_task_sched_in(task);
+	COND_STMT(&perf_task_events, __perf_event_task_sched_in(task));
 }
 
 static inline
 void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
 {
-	JUMP_LABEL(&perf_task_events, have_events);
-	return;
-
-have_events:
-	__perf_event_task_sched_out(task, next);
+	COND_STMT(&perf_task_events, __perf_event_task_sched_out(task, next));
 }
 
 extern int perf_event_init_task(struct task_struct *child);

[tip:perf/core] perf: Fix task refcount bugs

[tip-bot for Peter Zijlstra]

Commit-ID:  e7d0bc047548d76feee6b23f7d3d9da927189a50
Gitweb:     http://git.kernel.org/tip/e7d0bc047548d76feee6b23f7d3d9da927189a50
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 16:54:51 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:52 +0200

perf: Fix task refcount bugs

Currently it looks like find_lively_task_by_vpid() takes a task ref
and relies on find_get_context() to drop it.

The problem is that perf_event_create_kernel_counter() shouldn't be
dropping task refs.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Acked-by: Matt Helsley <matthltc@us.ibm.com>
LKML-Reference: <20101014203625.278436085@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 kernel/perf_event.c |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 346dc0e..f928878 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -2190,11 +2190,9 @@ retry:
 		}
 	}
 
-	put_task_struct(task);
 	return ctx;
 
 errout:
-	put_task_struct(task);
 	return ERR_PTR(err);
 }
 
@@ -5602,7 +5600,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_group_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5698,6 +5696,9 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_task:
+	if (task)
+		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
 	free_event(event);

[tip:perf/core] perf: Find task before event alloc

[tip-bot for Peter Zijlstra]

Commit-ID:  c6be5a5cb62592d9d661899a2aa78236eb00ffa5
Gitweb:     http://git.kernel.org/tip/c6be5a5cb62592d9d661899a2aa78236eb00ffa5
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 16:59:46 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:54 +0200

perf: Find task before event alloc

So that we can pass the task pointer to the event allocation, so that
we can use task associated data during event initialization.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20101014203625.340789919@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 kernel/perf_event.c |   23 ++++++++++++-----------
 1 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index f928878..b21d06a 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -5551,10 +5551,18 @@ SYSCALL_DEFINE5(perf_event_open,
 			group_leader = NULL;
 	}
 
+	if (pid != -1) {
+		task = find_lively_task_by_vpid(pid);
+		if (IS_ERR(task)) {
+			err = PTR_ERR(task);
+			goto err_group_fd;
+		}
+	}
+
 	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
-		goto err_fd;
+		goto err_task;
 	}
 
 	/*
@@ -5586,21 +5594,13 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	if (pid != -1) {
-		task = find_lively_task_by_vpid(pid);
-		if (IS_ERR(task)) {
-			err = PTR_ERR(task);
-			goto err_group_fd;
-		}
-	}
-
 	/*
 	 * Get the target context (task or percpu):
 	 */
 	ctx = find_get_context(pmu, task, cpu);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
-		goto err_task;
+		goto err_alloc;
 	}
 
 	/*
@@ -5696,12 +5696,13 @@ SYSCALL_DEFINE5(perf_event_open,
 
 err_context:
 	put_ctx(ctx);
+err_alloc:
+	free_event(event);
 err_task:
 	if (task)
 		put_task_struct(task);
 err_group_fd:
 	fput_light(group_file, fput_needed);
-	free_event(event);
 err_fd:
 	put_unused_fd(event_fd);
 	return err;

[tip:perf/core] perf, hw_breakpoint: Fix crash in hw_breakpoint creation

[tip-bot for Peter Zijlstra]

Commit-ID:  d580ff8699e8811a9af37e9de4dea375401bdeec
Gitweb:     http://git.kernel.org/tip/d580ff8699e8811a9af37e9de4dea375401bdeec
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 17:43:23 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:55 +0200

perf, hw_breakpoint: Fix crash in hw_breakpoint creation

hw_breakpoint creation needs to account stuff per-task to ensure there
is always sufficient hardware resources to back these things due to
ptrace.

With the perf per pmu context changes the event initialization no
longer has access to the event context, for the simple reason that we
need to first find the pmu (result of initialization) before we can
find the context.

This makes hw_breakpoints unhappy, because it can no longer do per
task accounting, cure this by frobbing a task pointer in the event::hw
bits for now...

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
LKML-Reference: <20101014203625.391543667@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/perf_event.h |    7 +++++++
 kernel/hw_breakpoint.c     |    8 ++++----
 kernel/perf_event.c        |   23 ++++++++++++++++++-----
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 2ebfc9a..97965fa 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -536,6 +536,12 @@ struct hw_perf_event {
 		struct { /* breakpoint */
 			struct arch_hw_breakpoint	info;
 			struct list_head		bp_list;
+			/*
+			 * Crufty hack to avoid the chicken and egg
+			 * problem hw_breakpoint has with context
+			 * creation and event initalization.
+			 */
+			struct task_struct		*bp_target;
 		};
 #endif
 	};
@@ -693,6 +699,7 @@ struct swevent_hlist {
 
 #define PERF_ATTACH_CONTEXT	0x01
 #define PERF_ATTACH_GROUP	0x02
+#define PERF_ATTACH_TASK	0x04
 
 /**
  * struct perf_event - performance event kernel representation:
diff --git a/kernel/hw_breakpoint.c b/kernel/hw_breakpoint.c
index 3b714e8..2c9120f 100644
--- a/kernel/hw_breakpoint.c
+++ b/kernel/hw_breakpoint.c
@@ -113,12 +113,12 @@ static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
  */
 static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
 {
-	struct perf_event_context *ctx = bp->ctx;
+	struct task_struct *tsk = bp->hw.bp_target;
 	struct perf_event *iter;
 	int count = 0;
 
 	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-		if (iter->ctx == ctx && find_slot_idx(iter) == type)
+		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
 			count += hw_breakpoint_weight(iter);
 	}
 
@@ -134,7 +134,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
 		    enum bp_type_idx type)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	if (cpu >= 0) {
 		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
@@ -213,7 +213,7 @@ toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
 	       int weight)
 {
 	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->ctx->task;
+	struct task_struct *tsk = bp->hw.bp_target;
 
 	/* Pinned counter cpu profiling */
 	if (!tsk) {
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index b21d06a..856e20b 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -5255,9 +5255,10 @@ unlock:
  */
 static struct perf_event *
 perf_event_alloc(struct perf_event_attr *attr, int cpu,
-		   struct perf_event *group_leader,
-		   struct perf_event *parent_event,
-		   perf_overflow_handler_t overflow_handler)
+		 struct task_struct *task,
+		 struct perf_event *group_leader,
+		 struct perf_event *parent_event,
+		 perf_overflow_handler_t overflow_handler)
 {
 	struct pmu *pmu;
 	struct perf_event *event;
@@ -5299,6 +5300,17 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
 
 	event->state		= PERF_EVENT_STATE_INACTIVE;
 
+	if (task) {
+		event->attach_state = PERF_ATTACH_TASK;
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+		/*
+		 * hw_breakpoint is a bit difficult here..
+		 */
+		if (attr->type == PERF_TYPE_BREAKPOINT)
+			event->hw.bp_target = task;
+#endif
+	}
+
 	if (!overflow_handler && parent_event)
 		overflow_handler = parent_event->overflow_handler;
 	
@@ -5559,7 +5571,7 @@ SYSCALL_DEFINE5(perf_event_open,
 		}
 	}
 
-	event = perf_event_alloc(&attr, cpu, group_leader, NULL, NULL);
+	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err_task;
@@ -5728,7 +5740,7 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
 	 * Get the target context (task or percpu):
 	 */
 
-	event = perf_event_alloc(attr, cpu, NULL, NULL, overflow_handler);
+	event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler);
 	if (IS_ERR(event)) {
 		err = PTR_ERR(event);
 		goto err;
@@ -5996,6 +6008,7 @@ inherit_event(struct perf_event *parent_event,
 
 	child_event = perf_event_alloc(&parent_event->attr,
 					   parent_event->cpu,
+					   child,
 					   group_leader, parent_event,
 					   NULL);
 	if (IS_ERR(child_event))

[tip:perf/core] jump_label: Use more consistent naming

[tip-bot for Peter Zijlstra]

Commit-ID:  3b6e901f839f42afb40f614418df82c08b01320a
Gitweb:     http://git.kernel.org/tip/3b6e901f839f42afb40f614418df82c08b01320a
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 21:10:38 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:56 +0200

jump_label: Use more consistent naming

Now that there's still only a few users around, rename things to make
them more consistent.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20101014203625.448565169@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/jump_label.h |    8 ++++----
 kernel/tracepoint.c        |    6 +++---
 lib/dynamic_debug.c        |    4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index b72cd9f..81be496 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -25,10 +25,10 @@ extern void jump_label_update(unsigned long key, enum jump_label_type type);
 extern void jump_label_apply_nops(struct module *mod);
 extern int jump_label_text_reserved(void *start, void *end);
 
-#define enable_jump_label(key) \
+#define jump_label_enable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
 
-#define disable_jump_label(key) \
+#define jump_label_disable(key) \
 	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
 
 #else
@@ -39,12 +39,12 @@ do {						\
 		goto label;			\
 } while (0)
 
-#define enable_jump_label(cond_var)	\
+#define jump_label_enable(cond_var)	\
 do {					\
        *(cond_var) = 1;			\
 } while (0)
 
-#define disable_jump_label(cond_var)	\
+#define jump_label_disable(cond_var)	\
 do {					\
        *(cond_var) = 0;			\
 } while (0)
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index d6073a5..e95ee7f 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -265,10 +265,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
 	if (!elem->state && active) {
-		enable_jump_label(&elem->state);
+		jump_label_enable(&elem->state);
 		elem->state = active;
 	} else if (elem->state && !active) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = active;
 	}
 }
@@ -285,7 +285,7 @@ static void disable_tracepoint(struct tracepoint *elem)
 		elem->unregfunc();
 
 	if (elem->state) {
-		disable_jump_label(&elem->state);
+		jump_label_disable(&elem->state);
 		elem->state = 0;
 	}
 	rcu_assign_pointer(elem->funcs, NULL);
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index e925c7b..7bd6df7 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -142,9 +142,9 @@ static void ddebug_change(const struct ddebug_query *query,
 				dt->num_enabled++;
 			dp->flags = newflags;
 			if (newflags) {
-				enable_jump_label(&dp->enabled);
+				jump_label_enable(&dp->enabled);
 			} else {
-				disable_jump_label(&dp->enabled);
+				jump_label_disable(&dp->enabled);
 			}
 			if (verbose)
 				printk(KERN_INFO

[tip:perf/core] jump_label: Add atomic_t interface

[tip-bot for Peter Zijlstra]

Commit-ID:  8b92538d84e50062560ba33adbaed7887b6e4a42
Gitweb:     http://git.kernel.org/tip/8b92538d84e50062560ba33adbaed7887b6e4a42
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 21:39:02 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:57 +0200

jump_label: Add atomic_t interface

Add an interface to allow usage of jump_labels with atomic counters.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
LKML-Reference: <20101014203625.501657727@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/jump_label_ref.h |   44 ++++++++++++++++++++++++++++++++++++++++
 1 files changed, 44 insertions(+), 0 deletions(-)

diff --git a/include/linux/jump_label_ref.h b/include/linux/jump_label_ref.h
new file mode 100644
index 0000000..e5d012a
--- /dev/null
+++ b/include/linux/jump_label_ref.h
@@ -0,0 +1,44 @@
+#ifndef _LINUX_JUMP_LABEL_REF_H
+#define _LINUX_JUMP_LABEL_REF_H
+
+#include <linux/jump_label.h>
+#include <asm/atomic.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	if (atomic_add_return(1, key) == 1)
+		jump_label_enable(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	if (atomic_dec_and_test(key))
+		jump_label_disable(key);
+}
+
+#else /* !HAVE_JUMP_LABEL */
+
+static inline void jump_label_inc(atomic_t *key)
+{
+	atomic_inc(key);
+}
+
+static inline void jump_label_dec(atomic_t *key)
+{
+	atomic_dec(key);
+}
+
+#undef JUMP_LABEL
+#define JUMP_LABEL(key, label)						\
+do {									\
+	if (unlikely(__builtin_choose_expr(				\
+	      __builtin_types_compatible_p(typeof(key), atomic_t *),	\
+	      atomic_read((atomic_t *)(key)), *(key))))			\
+		goto label;						\
+} while (0)
+
+#endif /* HAVE_JUMP_LABEL */
+
+#endif /* _LINUX_JUMP_LABEL_REF_H */

[tip:perf/core] perf: Use jump_labels to optimize the scheduler hooks

[tip-bot for Peter Zijlstra]

Commit-ID:  82cd6def9806dcb6a325fb6abbc1d61388a15f6a
Gitweb:     http://git.kernel.org/tip/82cd6def9806dcb6a325fb6abbc1d61388a15f6a
Author:     Peter Zijlstra <a.p.zijlstra@chello.nl>
AuthorDate: Thu, 14 Oct 2010 17:57:23 +0200
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Mon, 18 Oct 2010 19:58:58 +0200

perf: Use jump_labels to optimize the scheduler hooks

Trades a call + conditional + ret for an unconditional jmp.

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20101014203625.501657727@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/perf_event.h |   27 +++++++++++++++++++++++++--
 kernel/perf_event.c        |   24 +++++++++---------------
 2 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 97965fa..7f0e7f5 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -487,6 +487,7 @@ struct perf_guest_info_callbacks {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/irq_work.h>
+#include <linux/jump_label_ref.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
 
@@ -895,8 +896,30 @@ extern void perf_pmu_unregister(struct pmu *pmu);
 
 extern int perf_num_counters(void);
 extern const char *perf_pmu_name(void);
-extern void perf_event_task_sched_in(struct task_struct *task);
-extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+extern void __perf_event_task_sched_in(struct task_struct *task);
+extern void __perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
+
+extern atomic_t perf_task_events;
+
+static inline void perf_event_task_sched_in(struct task_struct *task)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_in(task);
+}
+
+static inline
+void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
+{
+	JUMP_LABEL(&perf_task_events, have_events);
+	return;
+
+have_events:
+	__perf_event_task_sched_out(task, next);
+}
+
 extern int perf_event_init_task(struct task_struct *child);
 extern void perf_event_exit_task(struct task_struct *child);
 extern void perf_event_free_task(struct task_struct *task);
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 856e20b..f7febb0 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -34,7 +34,7 @@
 
 #include <asm/irq_regs.h>
 
-static atomic_t nr_events __read_mostly;
+atomic_t perf_task_events __read_mostly;
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
@@ -1311,8 +1311,8 @@ void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  * accessing the event control register. If a NMI hits, then it will
  * not restart the event.
  */
-void perf_event_task_sched_out(struct task_struct *task,
-			       struct task_struct *next)
+void __perf_event_task_sched_out(struct task_struct *task,
+				 struct task_struct *next)
 {
 	int ctxn;
 
@@ -1340,14 +1340,6 @@ static void task_ctx_sched_out(struct perf_event_context *ctx,
 /*
  * Called with IRQs disabled
  */
-static void __perf_event_task_sched_out(struct perf_event_context *ctx)
-{
-	task_ctx_sched_out(ctx, EVENT_ALL);
-}
-
-/*
- * Called with IRQs disabled
- */
 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 			      enum event_type_t event_type)
 {
@@ -1494,7 +1486,7 @@ void perf_event_context_sched_in(struct perf_event_context *ctx)
  * accessing the event control register. If a NMI hits, then it will
  * keep the event running.
  */
-void perf_event_task_sched_in(struct task_struct *task)
+void __perf_event_task_sched_in(struct task_struct *task)
 {
 	struct perf_event_context *ctx;
 	int ctxn;
@@ -2216,7 +2208,8 @@ static void free_event(struct perf_event *event)
 	irq_work_sync(&event->pending);
 
 	if (!event->parent) {
-		atomic_dec(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_dec(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_dec(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5354,7 +5347,8 @@ done:
 	event->pmu = pmu;
 
 	if (!event->parent) {
-		atomic_inc(&nr_events);
+		if (event->attach_state & PERF_ATTACH_TASK)
+			jump_label_inc(&perf_task_events);
 		if (event->attr.mmap || event->attr.mmap_data)
 			atomic_inc(&nr_mmap_events);
 		if (event->attr.comm)
@@ -5849,7 +5843,7 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
 	 * our context.
 	 */
 	child_ctx = child->perf_event_ctxp[ctxn];
-	__perf_event_task_sched_out(child_ctx);
+	task_ctx_sched_out(child_ctx, EVENT_ALL);
 
 	/*
 	 * Take the context lock here so that if find_get_context is