From: Peter Zijlstra <peterz@infradead.org>
To: x86@kernel.org
Cc: linux-kernel@vger.kernel.org,
Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Andy Lutomirski <luto@kernel.org>,
Steven Rostedt <rostedt@goodmis.org>,
Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
Linus Torvalds <torvalds@linux-foundation.org>,
Masami Hiramatsu <mhiramat@kernel.org>,
Jason Baron <jbaron@akamai.com>, Jiri Kosina <jkosina@suse.cz>,
David Laight <David.Laight@ACULAB.COM>,
Borislav Petkov <bp@alien8.de>, Julia Cartwright <julia@ni.com>,
Jessica Yu <jeyu@kernel.org>, "H. Peter Anvin" <hpa@zytor.com>,
Nadav Amit <namit@vmware.com>,
Rasmus Villemoes <linux@rasmusvillemoes.dk>,
Edward Cree <ecree@solarflare.com>,
Daniel Bristot de Oliveira <bristot@redhat.com>
Subject: [PATCH 15/15] tracepoints: Use static_call
Date: Wed, 05 Jun 2019 15:08:08 +0200 [thread overview]
Message-ID: <20190605131945.431564484@infradead.org> (raw)
In-Reply-To: 20190605130753.327195108@infradead.org
From: Steven Rostedt (VMware) <rostedt@goodmis.org>
... Changelog goes here ...
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
include/linux/tracepoint-defs.h | 4 ++
include/linux/tracepoint.h | 73 ++++++++++++++++++++++++++--------------
include/trace/define_trace.h | 14 +++----
kernel/tracepoint.c | 25 +++++++++++--
4 files changed, 80 insertions(+), 36 deletions(-)
--- a/include/linux/tracepoint-defs.h
+++ b/include/linux/tracepoint-defs.h
@@ -11,6 +11,8 @@
#include <linux/atomic.h>
#include <linux/static_key.h>
+struct static_call_key;
+
struct trace_print_flags {
unsigned long mask;
const char *name;
@@ -30,6 +32,8 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
struct static_key key;
+ struct static_call_key *static_call_key;
+ void *iterator;
int (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -21,6 +21,7 @@
#include <linux/cpumask.h>
#include <linux/rcupdate.h>
#include <linux/tracepoint-defs.h>
+#include <linux/static_call.h>
struct module;
struct tracepoint;
@@ -94,7 +95,9 @@ extern int syscall_regfunc(void);
extern void syscall_unregfunc(void);
#endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
+#ifndef PARAMS
#define PARAMS(args...) args
+#endif
#define TRACE_DEFINE_ENUM(x)
#define TRACE_DEFINE_SIZEOF(x)
@@ -161,12 +164,11 @@ static inline struct tracepoint *tracepo
* as "(void *, void)". The DECLARE_TRACE_NOARGS() will pass in just
* "void *data", where as the DECLARE_TRACE() will pass in "void *data, proto".
*/
-#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
+#define __DO_TRACE(name, proto, args, cond, rcuidle) \
do { \
struct tracepoint_func *it_func_ptr; \
- void *it_func; \
- void *__data; \
int __maybe_unused __idx = 0; \
+ void *__data; \
\
if (!(cond)) \
return; \
@@ -186,14 +188,11 @@ static inline struct tracepoint *tracepo
rcu_irq_enter_irqson(); \
} \
\
- it_func_ptr = rcu_dereference_raw((tp)->funcs); \
- \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
if (it_func_ptr) { \
- do { \
- it_func = (it_func_ptr)->func; \
- __data = (it_func_ptr)->data; \
- ((void(*)(proto))(it_func))(args); \
- } while ((++it_func_ptr)->func); \
+ __data = (it_func_ptr)->data; \
+ static_call(tp_func_##name, args); \
} \
\
if (rcuidle) { \
@@ -209,7 +208,7 @@ static inline struct tracepoint *tracepo
static inline void trace_##name##_rcuidle(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 1); \
@@ -231,11 +230,13 @@ static inline struct tracepoint *tracepo
* poking RCU a bit.
*/
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
+ extern int __tracepoint_iter_##name(data_proto); \
+ DECLARE_STATIC_CALL(tp_func_##name, __tracepoint_iter_##name); \
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 0); \
@@ -281,21 +282,43 @@ static inline struct tracepoint *tracepo
* structures, so we create an array of pointers that will be used for iteration
* on the tracepoints.
*/
-#define DEFINE_TRACE_FN(name, reg, unreg) \
- static const char __tpstrtab_##name[] \
- __attribute__((section("__tracepoints_strings"))) = #name; \
- struct tracepoint __tracepoint_##name \
- __attribute__((section("__tracepoints"), used)) = \
- { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
- __TRACEPOINT_ENTRY(name);
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args) \
+ static const char __tpstrtab_##name[] \
+ __attribute__((section("__tracepoints_strings"))) = #name; \
+ extern struct static_call_key tp_func_##name; \
+ int __tracepoint_iter_##name(void *__data, proto); \
+ struct tracepoint __tracepoint_##name \
+ __attribute__((section("__tracepoints"), used)) = \
+ { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, \
+ &tp_func_##name, __tracepoint_iter_##name, \
+ reg, unreg, NULL }; \
+ __TRACEPOINT_ENTRY(name); \
+ int __tracepoint_iter_##name(void *__data, proto) \
+ { \
+ struct tracepoint_func *it_func_ptr; \
+ void *it_func; \
+ \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
+ do { \
+ it_func = (it_func_ptr)->func; \
+ __data = (it_func_ptr)->data; \
+ ((void(*)(void *, proto))(it_func))(__data, args); \
+ } while ((++it_func_ptr)->func); \
+ return 0; \
+ } \
+ DEFINE_STATIC_CALL(tp_func_##name, __tracepoint_iter_##name);
-#define DEFINE_TRACE(name) \
- DEFINE_TRACE_FN(name, NULL, NULL);
+#define DEFINE_TRACE(name, proto, args) \
+ DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
- EXPORT_SYMBOL_GPL(__tracepoint_##name)
+ EXPORT_SYMBOL_GPL(__tracepoint_##name); \
+ EXPORT_STATIC_CALL_GPL(tp_func_##name)
#define EXPORT_TRACEPOINT_SYMBOL(name) \
- EXPORT_SYMBOL(__tracepoint_##name)
+ EXPORT_SYMBOL(__tracepoint_##name); \
+ EXPORT_STATIC_CALL(tp_func_##name)
+
#else /* !TRACEPOINTS_ENABLED */
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
@@ -324,8 +347,8 @@ static inline struct tracepoint *tracepo
return false; \
}
-#define DEFINE_TRACE_FN(name, reg, unreg)
-#define DEFINE_TRACE(name)
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
+#define DEFINE_TRACE(name, proto, args)
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
#define EXPORT_TRACEPOINT_SYMBOL(name)
--- a/include/trace/define_trace.h
+++ b/include/trace/define_trace.h
@@ -25,7 +25,7 @@
#undef TRACE_EVENT
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_CONDITION
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@
#undef TRACE_EVENT_FN
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_FN_COND
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_NOP
#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
@@ -54,15 +54,15 @@
#undef DEFINE_EVENT
#define DEFINE_EVENT(template, name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_FN
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_PRINT
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_CONDITION
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
@@ -70,7 +70,7 @@
#undef DECLARE_TRACE
#define DECLARE_TRACE(name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef TRACE_INCLUDE
#undef __TRACE_INCLUDE
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -127,7 +127,7 @@ static void debug_print_probes(struct tr
static struct tracepoint_func *
func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
- int prio)
+ int prio, int *tot_probes)
{
struct tracepoint_func *old, *new;
int nr_probes = 0;
@@ -170,11 +170,12 @@ func_add(struct tracepoint_func **funcs,
new[nr_probes + 1].func = NULL;
*funcs = new;
debug_print_probes(*funcs);
+ *tot_probes = nr_probes + 1;
return old;
}
static void *func_remove(struct tracepoint_func **funcs,
- struct tracepoint_func *tp_func)
+ struct tracepoint_func *tp_func, int *left)
{
int nr_probes = 0, nr_del = 0, i;
struct tracepoint_func *old, *new;
@@ -228,6 +229,7 @@ static int tracepoint_add_func(struct tr
struct tracepoint_func *func, int prio)
{
struct tracepoint_func *old, *tp_funcs;
+ int probes = 0;
int ret;
if (tp->regfunc && !static_key_enabled(&tp->key)) {
@@ -238,7 +240,7 @@ static int tracepoint_add_func(struct tr
tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
- old = func_add(&tp_funcs, func, prio);
+ old = func_add(&tp_funcs, func, prio, &probes);
if (IS_ERR(old)) {
WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
return PTR_ERR(old);
@@ -253,6 +255,13 @@ static int tracepoint_add_func(struct tr
rcu_assign_pointer(tp->funcs, tp_funcs);
if (!static_key_enabled(&tp->key))
static_key_slow_inc(&tp->key);
+
+ if (probes == 1) {
+ __static_call_update(tp->static_call_key, tp_funcs->func);
+ } else {
+ __static_call_update(tp->static_call_key, tp->iterator);
+ }
+
release_probes(old);
return 0;
}
@@ -267,10 +276,11 @@ static int tracepoint_remove_func(struct
struct tracepoint_func *func)
{
struct tracepoint_func *old, *tp_funcs;
+ int probes_left = 0;
tp_funcs = rcu_dereference_protected(tp->funcs,
lockdep_is_held(&tracepoints_mutex));
- old = func_remove(&tp_funcs, func);
+ old = func_remove(&tp_funcs, func, &probes_left);
if (IS_ERR(old)) {
WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM);
return PTR_ERR(old);
@@ -284,6 +294,13 @@ static int tracepoint_remove_func(struct
if (static_key_enabled(&tp->key))
static_key_slow_dec(&tp->key);
}
+
+ if (probes_left == 1) {
+ __static_call_update(tp->static_call_key, tp_funcs->func);
+ } else {
+ __static_call_update(tp->static_call_key, tp->iterator);
+ }
+
rcu_assign_pointer(tp->funcs, tp_funcs);
release_probes(old);
return 0;
prev parent reply other threads:[~2019-06-05 13:23 UTC|newest]
Thread overview: 87+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-05 13:07 [PATCH 00/15] x86 cleanups and static_call() Peter Zijlstra
2019-06-05 13:07 ` [PATCH 01/15] x86/entry/32: Clean up return from interrupt preemption path Peter Zijlstra
2019-06-07 14:21 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 02/15] x86: Move ENCODE_FRAME_POINTER to asm/frame.h Peter Zijlstra
2019-06-07 14:24 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 03/15] x86/kprobes: Fix frame pointer annotations Peter Zijlstra
2019-06-07 13:02 ` Masami Hiramatsu
2019-06-07 13:36 ` Josh Poimboeuf
2019-06-07 15:21 ` Masami Hiramatsu
2019-06-11 8:12 ` Peter Zijlstra
2019-06-05 13:07 ` [PATCH 04/15] x86/ftrace: Add pt_regs frame annotations Peter Zijlstra
2019-06-07 14:45 ` Josh Poimboeuf
2019-06-05 13:07 ` [PATCH 05/15] x86_32: Provide consistent pt_regs Peter Zijlstra
2019-06-07 13:13 ` Masami Hiramatsu
2019-06-07 19:32 ` Josh Poimboeuf
2019-06-11 8:14 ` Peter Zijlstra
2019-06-05 13:07 ` [PATCH 06/15] x86_32: Allow int3_emulate_push() Peter Zijlstra
2019-06-05 13:08 ` [PATCH 07/15] x86: Add int3_emulate_call() selftest Peter Zijlstra
2019-06-10 16:52 ` Josh Poimboeuf
2019-06-10 16:57 ` Andy Lutomirski
2019-06-11 8:17 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 08/15] x86/alternatives: Teach text_poke_bp() to emulate instructions Peter Zijlstra
2019-06-07 5:41 ` Nadav Amit
2019-06-07 8:20 ` Peter Zijlstra
2019-06-07 14:27 ` Masami Hiramatsu
2019-06-07 15:47 ` Masami Hiramatsu
2019-06-07 17:34 ` Peter Zijlstra
2019-06-07 17:48 ` Linus Torvalds
2019-06-11 10:44 ` Peter Zijlstra
2019-06-07 18:10 ` Andy Lutomirski
2019-06-07 20:22 ` hpa
2019-06-11 8:03 ` Peter Zijlstra
2019-06-11 12:08 ` Peter Zijlstra
2019-06-11 12:34 ` Peter Zijlstra
2019-06-11 12:42 ` Peter Zijlstra
2019-06-11 15:22 ` Steven Rostedt
2019-06-11 15:52 ` Steven Rostedt
2019-06-11 15:55 ` Peter Zijlstra
2019-06-12 19:44 ` Nadav Amit
2019-06-17 14:42 ` Peter Zijlstra
2019-06-17 17:06 ` Nadav Amit
2019-06-17 17:25 ` Andy Lutomirski
2019-06-17 19:26 ` Peter Zijlstra
2019-06-11 15:54 ` Andy Lutomirski
2019-06-11 16:11 ` Steven Rostedt
2019-06-17 14:31 ` Peter Zijlstra
2019-06-12 17:09 ` Peter Zijlstra
2019-06-10 16:57 ` Josh Poimboeuf
2019-06-11 15:14 ` Steven Rostedt
2019-06-11 15:52 ` Peter Zijlstra
2019-06-11 16:21 ` Peter Zijlstra
2019-06-12 14:44 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 09/15] compiler.h: Make __ADDRESSABLE() symbol truly unique Peter Zijlstra
2019-06-05 13:08 ` [PATCH 10/15] static_call: Add basic static call infrastructure Peter Zijlstra
2019-06-06 22:44 ` Nadav Amit
2019-06-07 8:28 ` Peter Zijlstra
2019-06-07 8:49 ` Ard Biesheuvel
2019-06-07 16:33 ` Andy Lutomirski
2019-06-07 16:58 ` Nadav Amit
2019-10-02 13:54 ` Peter Zijlstra
2019-10-02 20:48 ` Josh Poimboeuf
2019-06-05 13:08 ` [PATCH 11/15] static_call: Add inline " Peter Zijlstra
2019-06-06 22:24 ` Nadav Amit
2019-06-07 8:37 ` Peter Zijlstra
2019-06-07 16:35 ` Nadav Amit
2019-06-07 17:41 ` Peter Zijlstra
2019-06-10 17:19 ` Josh Poimboeuf
2019-06-10 18:33 ` Nadav Amit
2019-06-10 18:42 ` Josh Poimboeuf
2019-10-01 12:00 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 12/15] x86/static_call: Add out-of-line static call implementation Peter Zijlstra
2019-06-07 6:13 ` Nadav Amit
2019-06-07 7:51 ` Steven Rostedt
2019-06-07 8:38 ` Peter Zijlstra
2019-06-07 8:52 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 13/15] x86/static_call: Add inline static call implementation for x86-64 Peter Zijlstra
2019-06-07 5:50 ` Nadav Amit
2019-06-10 18:33 ` Josh Poimboeuf
2019-06-10 18:45 ` Nadav Amit
2019-06-10 18:55 ` Josh Poimboeuf
2019-06-10 19:20 ` Nadav Amit
2019-10-01 14:43 ` Peter Zijlstra
2019-06-05 13:08 ` [PATCH 14/15] static_call: Simple self-test module Peter Zijlstra
2019-06-10 17:24 ` Josh Poimboeuf
2019-06-11 8:29 ` Peter Zijlstra
2019-06-11 13:02 ` Josh Poimboeuf
2019-06-05 13:08 ` Peter Zijlstra [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190605131945.431564484@infradead.org \
--to=peterz@infradead.org \
--cc=David.Laight@ACULAB.COM \
--cc=ard.biesheuvel@linaro.org \
--cc=bp@alien8.de \
--cc=bristot@redhat.com \
--cc=ecree@solarflare.com \
--cc=hpa@zytor.com \
--cc=jbaron@akamai.com \
--cc=jeyu@kernel.org \
--cc=jkosina@suse.cz \
--cc=julia@ni.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux@rasmusvillemoes.dk \
--cc=luto@kernel.org \
--cc=mhiramat@kernel.org \
--cc=mingo@kernel.org \
--cc=namit@vmware.com \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=torvalds@linux-foundation.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).