[PATCHSET] tracing: Make event triggers affect correct instance

[PATCHSET] tracing: Make event triggers affect correct instance

[Howard Cochran]

Although all of the trace event triggers can be used in instances, most
of them affect the top-level instance even when triggered from within an
instance. This patch set makes them affect the instance in which the
trigger occurred. The enable_event & disable_event triggers already work
within the correct instance, so these are unchanged.

The same problem also exists with the function probe triggers, but they
will be fixed in a later patch set.

Patch 1 is preparatory.
Patch 2 fixes the stacktrace event trigger
Patch 3 fixes the traceon & traceoff event triggers
Patch 4 fixes the snapshot event trigger
Patch 5 Makes related warning messages also go to the correct instance

 kernel/trace/trace.c                | 96 +++++++++++++++++++++++++------------
 kernel/trace/trace.h                | 14 +++++-
 kernel/trace/trace_events_trigger.c | 62 ++++++++++++++----------
 3 files changed, 113 insertions(+), 59 deletions(-)

[PATCH 1/5] tracing: Add param to event_trigger_ops.func() for instances

[Howard Cochran]

(Non-functional change)
Currently, the traceon, traceoff, stacktrace, and snapshot event
triggers always affect the top level trace buffer, even when the
trigger is enabled in a buffer instance. In order to fix this, the
trigger's .func() needs to receive the struct trace_event_file * so
that it can find the correct trace_array for the instance.

This change adds the neeeded parameter and changes the declarations of
each impementation to match. Changes to actually make use of the
parameter will follow in subsequent commits.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 kernel/trace/trace.h                |  3 ++-
 kernel/trace/trace_events_trigger.c | 40 +++++++++++++++++++++++--------------
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 3fff4ad..ca4915f 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1230,7 +1230,8 @@ extern int register_event_command(struct event_command *cmd);
  */
 struct event_trigger_ops {
 	void			(*func)(struct event_trigger_data *data,
-					void *rec);
+					void *rec,
+					struct trace_event_file *file);
 	int			(*init)(struct event_trigger_ops *ops,
 					struct event_trigger_data *data);
 	void			(*free)(struct event_trigger_ops *ops,
diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
index d67992f..b69f2a2 100644
--- a/kernel/trace/trace_events_trigger.c
+++ b/kernel/trace/trace_events_trigger.c
@@ -75,7 +75,7 @@ event_triggers_call(struct trace_event_file *file, void *rec)
 		if (data->paused)
 			continue;
 		if (!rec) {
-			data->ops->func(data, rec);
+			data->ops->func(data, rec, file);
 			continue;
 		}
 		filter = rcu_dereference_sched(data->filter);
@@ -85,7 +85,7 @@ event_triggers_call(struct trace_event_file *file, void *rec)
 			tt |= data->cmd_ops->trigger_type;
 			continue;
 		}
-		data->ops->func(data, rec);
+		data->ops->func(data, rec, file);
 	}
 	return tt;
 }
@@ -115,7 +115,7 @@ event_triggers_post_call(struct trace_event_file *file,
 		if (data->paused)
 			continue;
 		if (data->cmd_ops->trigger_type & tt)
-			data->ops->func(data, rec);
+			data->ops->func(data, rec, file);
 	}
 }
 EXPORT_SYMBOL_GPL(event_triggers_post_call);
@@ -765,7 +765,8 @@ int set_trigger_filter(char *filter_str,
 }
 
 static void
-traceon_trigger(struct event_trigger_data *data, void *rec)
+traceon_trigger(struct event_trigger_data *data, void *rec,
+		struct trace_event_file *file)
 {
 	if (tracing_is_on())
 		return;
@@ -774,7 +775,8 @@ traceon_trigger(struct event_trigger_data *data, void *rec)
 }
 
 static void
-traceon_count_trigger(struct event_trigger_data *data, void *rec)
+traceon_count_trigger(struct event_trigger_data *data, void *rec,
+		      struct trace_event_file *file)
 {
 	if (tracing_is_on())
 		return;
@@ -789,7 +791,8 @@ traceon_count_trigger(struct event_trigger_data *data, void *rec)
 }
 
 static void
-traceoff_trigger(struct event_trigger_data *data, void *rec)
+traceoff_trigger(struct event_trigger_data *data, void *rec,
+		 struct trace_event_file *file)
 {
 	if (!tracing_is_on())
 		return;
@@ -798,7 +801,8 @@ traceoff_trigger(struct event_trigger_data *data, void *rec)
 }
 
 static void
-traceoff_count_trigger(struct event_trigger_data *data, void *rec)
+traceoff_count_trigger(struct event_trigger_data *data, void *rec,
+		       struct trace_event_file *file)
 {
 	if (!tracing_is_on())
 		return;
@@ -894,13 +898,15 @@ static struct event_command trigger_traceoff_cmd = {
 
 #ifdef CONFIG_TRACER_SNAPSHOT
 static void
-snapshot_trigger(struct event_trigger_data *data, void *rec)
+snapshot_trigger(struct event_trigger_data *data, void *rec,
+		 struct trace_event_file *file)
 {
 	tracing_snapshot();
 }
 
 static void
-snapshot_count_trigger(struct event_trigger_data *data, void *rec)
+snapshot_count_trigger(struct event_trigger_data *data, void *rec,
+		       struct trace_event_file *file)
 {
 	if (!data->count)
 		return;
@@ -987,13 +993,15 @@ static __init int register_trigger_snapshot_cmd(void) { return 0; }
 #define STACK_SKIP 3
 
 static void
-stacktrace_trigger(struct event_trigger_data *data, void *rec)
+stacktrace_trigger(struct event_trigger_data *data, void *rec,
+		   struct trace_event_file *file)
 {
 	trace_dump_stack(STACK_SKIP);
 }
 
 static void
-stacktrace_count_trigger(struct event_trigger_data *data, void *rec)
+stacktrace_count_trigger(struct event_trigger_data *data, void *rec,
+			 struct trace_event_file *file)
 {
 	if (!data->count)
 		return;
@@ -1001,7 +1009,7 @@ stacktrace_count_trigger(struct event_trigger_data *data, void *rec)
 	if (data->count != -1)
 		(data->count)--;
 
-	stacktrace_trigger(data, rec);
+	stacktrace_trigger(data, rec, file);
 }
 
 static int
@@ -1072,7 +1080,8 @@ struct enable_trigger_data {
 };
 
 static void
-event_enable_trigger(struct event_trigger_data *data, void *rec)
+event_enable_trigger(struct event_trigger_data *data, void *rec,
+		     struct trace_event_file *file)
 {
 	struct enable_trigger_data *enable_data = data->private_data;
 
@@ -1083,7 +1092,8 @@ event_enable_trigger(struct event_trigger_data *data, void *rec)
 }
 
 static void
-event_enable_count_trigger(struct event_trigger_data *data, void *rec)
+event_enable_count_trigger(struct event_trigger_data *data, void *rec,
+			   struct trace_event_file *file)
 {
 	struct enable_trigger_data *enable_data = data->private_data;
 
@@ -1097,7 +1107,7 @@ event_enable_count_trigger(struct event_trigger_data *data, void *rec)
 	if (data->count != -1)
 		(data->count)--;
 
-	event_enable_trigger(data, rec);
+	event_enable_trigger(data, rec, file);
 }
 
 static int
-- 
1.9.1

Re: [PATCH 1/5] tracing: Add param to event_trigger_ops.func() for instances

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 2086 bytes --]

Hi Howard,

[auto build test ERROR on tip/perf/core]
[also build test ERROR on v4.6-rc3 next-20160414]
[if your patch is applied to the wrong git tree, please drop us a note to help improving the system]

url:    https://github.com/0day-ci/linux/commits/Howard-Cochran/tracing-Add-param-to-event_trigger_ops-func-for-instances/20160415-145801
config: i386-randconfig-x000-201615 (attached as .config)
reproduce:
        # save the attached .config to linux build tree
        make ARCH=i386 

Note: the linux-review/Howard-Cochran/tracing-Add-param-to-event_trigger_ops-func-for-instances/20160415-145801 HEAD 30aa67c47e19ac73897f7683d98b7c49a08cf635 builds fine.
      It only hurts bisectibility.

All errors (new ones prefixed by >>):

   kernel/trace/trace_events_trigger.c: In function 'snapshot_count_trigger':
>> kernel/trace/trace_events_trigger.c:917:2: error: too few arguments to function 'snapshot_trigger'
     snapshot_trigger(data, rec);
     ^
   kernel/trace/trace_events_trigger.c:901:1: note: declared here
    snapshot_trigger(struct event_trigger_data *data, void *rec,
    ^

vim +/snapshot_trigger +917 kernel/trace/trace_events_trigger.c

93e31ffb Tom Zanussi 2013-10-24  911  	if (!data->count)
93e31ffb Tom Zanussi 2013-10-24  912  		return;
93e31ffb Tom Zanussi 2013-10-24  913  
93e31ffb Tom Zanussi 2013-10-24  914  	if (data->count != -1)
93e31ffb Tom Zanussi 2013-10-24  915  		(data->count)--;
93e31ffb Tom Zanussi 2013-10-24  916  
c4a59230 Tom Zanussi 2015-12-10 @917  	snapshot_trigger(data, rec);
93e31ffb Tom Zanussi 2013-10-24  918  }
93e31ffb Tom Zanussi 2013-10-24  919  
93e31ffb Tom Zanussi 2013-10-24  920  static int

:::::: The code at line 917 was first introduced by commit
:::::: c4a5923055c9e0c87dfc0387f7cda5ee2bbac3c1 tracing: Add event record param to trigger_ops.func()

:::::: TO: Tom Zanussi <tom.zanussi@linux.intel.com>
:::::: CC: Steven Rostedt <rostedt@goodmis.org>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 21675 bytes --]

[PATCH 2/5] tracing: Make stacktrace trigger affect the correct instance

[Howard Cochran]

Currently, the stacktrace trigger always dumps the stack into the top
level buffer, even if it was triggered within an instance. Fixed
by using the trace_array associated with the trace_event_file which
fired the trigger.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 kernel/trace/trace.c                | 21 +++++++++++++++++----
 kernel/trace/trace.h                |  2 ++
 kernel/trace/trace_events_trigger.c |  2 +-
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index a2f0b9f..5541517 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1900,14 +1900,15 @@ void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
 }
 
 /**
- * trace_dump_stack - record a stack back trace in the trace buffer
+ * __trace_dump_stack - record a stack back trace in the given trace buffer
  * @skip: Number of functions to skip (helper handlers)
+ * @tr:   Which trace buffer (instance) to write to
  */
-void trace_dump_stack(int skip)
+void __trace_dump_stack(int skip, struct trace_array *tr)
 {
 	unsigned long flags;
 
-	if (tracing_disabled || tracing_selftest_running)
+	if (tracing_disabled || tr->buffer_disabled || tracing_selftest_running)
 		return;
 
 	local_save_flags(flags);
@@ -1917,10 +1918,22 @@ void trace_dump_stack(int skip)
 	 * this function.
 	 */
 	skip += 3;
-	__ftrace_trace_stack(global_trace.trace_buffer.buffer,
+	__ftrace_trace_stack(tr->trace_buffer.buffer,
 			     flags, skip, preempt_count(), NULL);
 }
 
+/**
+ * trace_dump_stack - record a stack back trace in the top-level trace buffer
+ * @skip: Number of functions to skip (helper handlers)
+ */
+void trace_dump_stack(int skip)
+{
+	/* This wrapper function deepens the stack one level */
+	skip += 1;
+
+	__trace_dump_stack(skip, &global_trace);
+}
+
 static DEFINE_PER_CPU(int, user_stack_count);
 
 void
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index ca4915f..00a5419 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -716,6 +716,8 @@ enum print_line_t print_trace_line(struct trace_iterator *iter);
 
 extern char trace_find_mark(unsigned long long duration);
 
+void __trace_dump_stack(int skip, struct trace_array *tr);
+
 /* Standard output formatting function used for function return traces */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 
diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
index b69f2a2..adfcd53 100644
--- a/kernel/trace/trace_events_trigger.c
+++ b/kernel/trace/trace_events_trigger.c
@@ -996,7 +996,7 @@ static void
 stacktrace_trigger(struct event_trigger_data *data, void *rec,
 		   struct trace_event_file *file)
 {
-	trace_dump_stack(STACK_SKIP);
+	__trace_dump_stack(STACK_SKIP, file->tr);
 }
 
 static void
-- 
1.9.1

[PATCH 3/5] tracing: Make traceon, traceoff trigger affect the correct instance

[Howard Cochran]

Currently, the traceon & traceoff triggers always enable or disable
tracing in the top level buffer, even if it was triggered within an
instance. Fixed by using the trace_array associated with the
trace_event_file which fired the trigger.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 kernel/trace/trace.c                |  6 +++---
 kernel/trace/trace.h                |  3 +++
 kernel/trace/trace_events_trigger.c | 16 ++++++++--------
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 5541517..c78fe42 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -498,7 +498,7 @@ static inline void ftrace_trace_stack(struct trace_array *tr,
 
 #endif
 
-static void tracer_tracing_on(struct trace_array *tr)
+void tracer_tracing_on(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		ring_buffer_record_on(tr->trace_buffer.buffer);
@@ -764,7 +764,7 @@ void tracing_snapshot_alloc(void)
 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
 #endif /* CONFIG_TRACER_SNAPSHOT */
 
-static void tracer_tracing_off(struct trace_array *tr)
+void tracer_tracing_off(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		ring_buffer_record_off(tr->trace_buffer.buffer);
@@ -807,7 +807,7 @@ void disable_trace_on_warning(void)
  *
  * Shows real state of the ring buffer if it is enabled or not.
  */
-static int tracer_tracing_is_on(struct trace_array *tr)
+int tracer_tracing_is_on(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
 		return ring_buffer_record_is_on(tr->trace_buffer.buffer);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 00a5419..d3bc0e8 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -717,6 +717,9 @@ enum print_line_t print_trace_line(struct trace_iterator *iter);
 extern char trace_find_mark(unsigned long long duration);
 
 void __trace_dump_stack(int skip, struct trace_array *tr);
+int tracer_tracing_is_on(struct trace_array *tr);
+void tracer_tracing_on(struct trace_array *tr);
+void tracer_tracing_off(struct trace_array *tr);
 
 /* Standard output formatting function used for function return traces */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
index adfcd53..a73b823 100644
--- a/kernel/trace/trace_events_trigger.c
+++ b/kernel/trace/trace_events_trigger.c
@@ -768,17 +768,17 @@ static void
 traceon_trigger(struct event_trigger_data *data, void *rec,
 		struct trace_event_file *file)
 {
-	if (tracing_is_on())
+	if (tracer_tracing_is_on(file->tr))
 		return;
 
-	tracing_on();
+	tracer_tracing_on(file->tr);
 }
 
 static void
 traceon_count_trigger(struct event_trigger_data *data, void *rec,
 		      struct trace_event_file *file)
 {
-	if (tracing_is_on())
+	if (tracer_tracing_is_on(file->tr))
 		return;
 
 	if (!data->count)
@@ -787,24 +787,24 @@ traceon_count_trigger(struct event_trigger_data *data, void *rec,
 	if (data->count != -1)
 		(data->count)--;
 
-	tracing_on();
+	tracer_tracing_on(file->tr);
 }
 
 static void
 traceoff_trigger(struct event_trigger_data *data, void *rec,
 		 struct trace_event_file *file)
 {
-	if (!tracing_is_on())
+	if (!tracer_tracing_is_on(file->tr))
 		return;
 
-	tracing_off();
+	tracer_tracing_off(file->tr);
 }
 
 static void
 traceoff_count_trigger(struct event_trigger_data *data, void *rec,
 		       struct trace_event_file *file)
 {
-	if (!tracing_is_on())
+	if (!tracer_tracing_is_on(file->tr))
 		return;
 
 	if (!data->count)
@@ -813,7 +813,7 @@ traceoff_count_trigger(struct event_trigger_data *data, void *rec,
 	if (data->count != -1)
 		(data->count)--;
 
-	tracing_off();
+	tracer_tracing_off(file->tr);
 }
 
 static int
-- 
1.9.1

[PATCH 4/5] tracing: Make snapshot trigger affect the correct instance

[Howard Cochran]

Currently, the snapshot trigger always snapshots the top level instance,
even if it was triggered within an instance. Fixed by using the trace_array
associated with the trace_event_file which fired the trigger.

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 kernel/trace/trace.c                | 38 ++++++++++++++++++++++---------------
 kernel/trace/trace.h                |  1 +
 kernel/trace/trace_events_trigger.c |  4 ++--
 3 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index c78fe42..e98770b 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -620,22 +620,11 @@ EXPORT_SYMBOL_GPL(__trace_bputs);
 
 #ifdef CONFIG_TRACER_SNAPSHOT
 /**
- * trace_snapshot - take a snapshot of the current buffer.
- *
- * This causes a swap between the snapshot buffer and the current live
- * tracing buffer. You can use this to take snapshots of the live
- * trace when some condition is triggered, but continue to trace.
- *
- * Note, make sure to allocate the snapshot with either
- * a tracing_snapshot_alloc(), or by doing it manually
- * with: echo 1 > /sys/kernel/debug/tracing/snapshot
- *
- * If the snapshot buffer is not allocated, it will stop tracing.
- * Basically making a permanent snapshot.
+ * trace_snapshot - take a snapshot of a buffer.
+ * @tr:    Which tracing buffer (instance) to take a snapshot of
  */
-void tracing_snapshot(void)
+void __tracing_snapshot(struct trace_array *tr)
 {
-	struct trace_array *tr = &global_trace;
 	struct tracer *tracer = tr->current_trace;
 	unsigned long flags;
 
@@ -648,7 +637,7 @@ void tracing_snapshot(void)
 	if (!tr->allocated_snapshot) {
 		internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
 		internal_trace_puts("*** stopping trace here!   ***\n");
-		tracing_off();
+		tracer_tracing_off(tr);
 		return;
 	}
 
@@ -663,6 +652,25 @@ void tracing_snapshot(void)
 	update_max_tr(tr, current, smp_processor_id());
 	local_irq_restore(flags);
 }
+
+/**
+ * trace_snapshot - take a snapshot of the top level buffer.
+ *
+ * This causes a swap between the snapshot buffer and the current live
+ * tracing buffer. You can use this to take snapshots of the live
+ * trace when some condition is triggered, but continue to trace.
+ *
+ * Note, make sure to allocate the snapshot with either
+ * a tracing_snapshot_alloc(), or by doing it manually
+ * with: echo 1 > /sys/kernel/debug/tracing/snapshot
+ *
+ * If the snapshot buffer is not allocated, it will stop tracing.
+ * Basically making a permanent snapshot.
+ */
+void tracing_snapshot(void)
+{
+	__tracing_snapshot(&global_trace);
+}
 EXPORT_SYMBOL_GPL(tracing_snapshot);
 
 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index d3bc0e8..d98d54c 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -717,6 +717,7 @@ enum print_line_t print_trace_line(struct trace_iterator *iter);
 extern char trace_find_mark(unsigned long long duration);
 
 void __trace_dump_stack(int skip, struct trace_array *tr);
+void __tracing_snapshot(struct trace_array *tr);
 int tracer_tracing_is_on(struct trace_array *tr);
 void tracer_tracing_on(struct trace_array *tr);
 void tracer_tracing_off(struct trace_array *tr);
diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c
index a73b823..2f7dc19 100644
--- a/kernel/trace/trace_events_trigger.c
+++ b/kernel/trace/trace_events_trigger.c
@@ -901,7 +901,7 @@ static void
 snapshot_trigger(struct event_trigger_data *data, void *rec,
 		 struct trace_event_file *file)
 {
-	tracing_snapshot();
+	__tracing_snapshot(file->tr);
 }
 
 static void
@@ -914,7 +914,7 @@ snapshot_count_trigger(struct event_trigger_data *data, void *rec,
 	if (data->count != -1)
 		(data->count)--;
 
-	snapshot_trigger(data, rec);
+	snapshot_trigger(data, rec, file);
 }
 
 static int
-- 
1.9.1

[PATCH 5/5] tracing: Make internal_trace_puts() write to correct instance

[Howard Cochran]

Currently, internal_trace_puts() always writes to the top level
instance, but callers use this to write warnings which may apply
to any instance. Fixed by adding a struct trace_array * parameter
and changing current callers to use it.

Note that this still queries the trace_printk option for the top level
to decide whether to actually write to the buffer because that option is
documented to only apply to the top level buffer. This behavior also
matches that of the existing error message in update_max_tr_single().

Signed-off-by: Howard Cochran <hcochran@kernelspring.com>
---
 kernel/trace/trace.c | 31 ++++++++++++++++++++++---------
 kernel/trace/trace.h |  5 ++++-
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index e98770b..b03dda4 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -529,11 +529,13 @@ EXPORT_SYMBOL_GPL(tracing_on);
 
 /**
  * __trace_puts - write a constant string into the trace buffer.
+ * @tr:    Which buffer (instance) to write to
  * @ip:	   The address of the caller
  * @str:   The constant string to write
  * @size:  The size of the string.
  */
-int __trace_puts(unsigned long ip, const char *str, int size)
+int __trace_puts_tr(struct trace_array *tr, unsigned long ip,
+		    const char *str, int size)
 {
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
@@ -553,7 +555,7 @@ int __trace_puts(unsigned long ip, const char *str, int size)
 	alloc = sizeof(*entry) + size + 2; /* possible \n added */
 
 	local_save_flags(irq_flags);
-	buffer = global_trace.trace_buffer.buffer;
+	buffer = tr->trace_buffer.buffer;
 	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
 					  irq_flags, pc);
 	if (!event)
@@ -572,10 +574,21 @@ int __trace_puts(unsigned long ip, const char *str, int size)
 		entry->buf[size] = '\0';
 
 	__buffer_unlock_commit(buffer, event);
-	ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
+	ftrace_trace_stack(tr, buffer, irq_flags, 4, pc, NULL);
 
 	return size;
 }
+
+/**
+ * __trace_puts - write a constant string into the top level trace buffer.
+ * @ip:	   The address of the caller
+ * @str:   The constant string to write
+ * @size:  The size of the string.
+ */
+int __trace_puts(unsigned long ip, const char *str, int size)
+{
+	return __trace_puts_tr(&global_trace, ip, str, size);
+}
 EXPORT_SYMBOL_GPL(__trace_puts);
 
 /**
@@ -629,22 +642,22 @@ void __tracing_snapshot(struct trace_array *tr)
 	unsigned long flags;
 
 	if (in_nmi()) {
-		internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
-		internal_trace_puts("*** snapshot is being ignored        ***\n");
+		internal_trace_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
+		internal_trace_puts(tr, "*** snapshot is being ignored        ***\n");
 		return;
 	}
 
 	if (!tr->allocated_snapshot) {
-		internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
-		internal_trace_puts("*** stopping trace here!   ***\n");
+		internal_trace_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n");
+		internal_trace_puts(tr, "*** stopping trace here!   ***\n");
 		tracer_tracing_off(tr);
 		return;
 	}
 
 	/* Note, snapshot can not be used when the tracer uses it */
 	if (tracer->use_max_tr) {
-		internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
-		internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
+		internal_trace_puts(tr, "*** LATENCY TRACER ACTIVE ***\n");
+		internal_trace_puts(tr, "*** Can not use snapshot (sorry) ***\n");
 		return;
 	}
 
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index d98d54c..d23e6ac 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -1404,6 +1404,8 @@ void trace_printk_start_comm(void);
 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
 
+int __trace_puts_tr(struct trace_array *tr, unsigned long ip,
+		    const char *str, int size);
 /*
  * Normal trace_printk() and friends allocates special buffers
  * to do the manipulation, as well as saves the print formats
@@ -1413,7 +1415,8 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
  * about performance). The internal_trace_puts() is for such
  * a purpose.
  */
-#define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
+#define internal_trace_puts(tr, str)	\
+	__trace_puts_tr(tr, _THIS_IP_, str, strlen(str))
 
 #undef FTRACE_ENTRY
 #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter)	\
-- 
1.9.1