From: "tip-bot2 for Peter Zijlstra" <tip-bot2@linutronix.de>
To: linux-tip-commits@vger.kernel.org
Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org>,
Ingo Molnar <mingo@kernel.org>,
Linus Torvalds <torvalds@linux-foundation.org>,
x86 <x86@kernel.org>, LKML <linux-kernel@vger.kernel.org>
Subject: [tip: core/static_call] static_call: Handle tail-calls
Date: Tue, 01 Sep 2020 11:48:03 -0000 [thread overview]
Message-ID: <159896088374.20229.4926395571390516128.tip-bot2@tip-bot2> (raw)
In-Reply-To: <20200818135805.101186767@infradead.org>
The following commit has been merged into the core/static_call branch of tip:
Commit-ID: 5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997
Gitweb: https://git.kernel.org/tip/5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997
Author: Peter Zijlstra <peterz@infradead.org>
AuthorDate: Tue, 18 Aug 2020 15:57:49 +02:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Tue, 01 Sep 2020 09:58:06 +02:00
static_call: Handle tail-calls
GCC can turn our static_call(name)(args...) into a tail call, in which
case we get a JMP.d32 into the trampoline (which then does a further
tail-call).
Teach objtool to recognise and mark these in .static_call_sites and
adjust the code patching to deal with this.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135805.101186767@infradead.org
---
arch/x86/kernel/static_call.c | 21 ++++++++++++++++++---
include/linux/static_call.h | 4 ++--
include/linux/static_call_types.h | 7 +++++++
kernel/static_call.c | 21 +++++++++++++--------
tools/include/linux/static_call_types.h | 7 +++++++
tools/objtool/check.c | 18 +++++++++++++-----
6 files changed, 60 insertions(+), 18 deletions(-)
diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
index ead6726..60a325c 100644
--- a/arch/x86/kernel/static_call.c
+++ b/arch/x86/kernel/static_call.c
@@ -41,15 +41,30 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func)
text_poke_bp(insn, code, size, NULL);
}
-void arch_static_call_transform(void *site, void *tramp, void *func)
+static inline enum insn_type __sc_insn(bool null, bool tail)
+{
+ /*
+ * Encode the following table without branches:
+ *
+ * tail null insn
+ * -----+-------+------
+ * 0 | 0 | CALL
+ * 0 | 1 | NOP
+ * 1 | 0 | JMP
+ * 1 | 1 | RET
+ */
+ return 2*tail + null;
+}
+
+void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
{
mutex_lock(&text_mutex);
if (tramp)
- __static_call_transform(tramp, func ? JMP : RET, func);
+ __static_call_transform(tramp, __sc_insn(!func, true), func);
if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site)
- __static_call_transform(site, func ? CALL : NOP, func);
+ __static_call_transform(site, __sc_insn(!func, tail), func);
mutex_unlock(&text_mutex);
}
diff --git a/include/linux/static_call.h b/include/linux/static_call.h
index 0f74581..519bd66 100644
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -103,7 +103,7 @@
/*
* Either @site or @tramp can be NULL.
*/
-extern void arch_static_call_transform(void *site, void *tramp, void *func);
+extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
@@ -206,7 +206,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
{
cpus_read_lock();
WRITE_ONCE(key->func, func);
- arch_static_call_transform(NULL, tramp, func);
+ arch_static_call_transform(NULL, tramp, func, false);
cpus_read_unlock();
}
diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h
index 408d345..89135bb 100644
--- a/include/linux/static_call_types.h
+++ b/include/linux/static_call_types.h
@@ -17,6 +17,13 @@
#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
/*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL /* tail call */
+#define STATIC_CALL_SITE_INIT 2UL /* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
* The static call site table needs to be created by external tooling (objtool
* or a compiler plugin).
*/
diff --git a/kernel/static_call.c b/kernel/static_call.c
index 97142cb..d98e0e4 100644
--- a/kernel/static_call.c
+++ b/kernel/static_call.c
@@ -15,8 +15,6 @@ extern struct static_call_site __start_static_call_sites[],
static bool static_call_initialized;
-#define STATIC_CALL_INIT 1UL
-
/* mutex to protect key modules/sites */
static DEFINE_MUTEX(static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(struct static_call_site *site)
static inline struct static_call_key *static_call_key(const struct static_call_site *site)
{
return (struct static_call_key *)
- (((long)site->key + (long)&site->key) & ~STATIC_CALL_INIT);
+ (((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);
}
/* These assume the key is word-aligned. */
static inline bool static_call_is_init(struct static_call_site *site)
{
- return ((long)site->key + (long)&site->key) & STATIC_CALL_INIT;
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;
+}
+
+static inline bool static_call_is_tail(struct static_call_site *site)
+{
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;
}
static inline void static_call_set_init(struct static_call_site *site)
{
- site->key = ((long)static_call_key(site) | STATIC_CALL_INIT) -
+ site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -
(long)&site->key;
}
@@ -104,7 +107,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
key->func = func;
- arch_static_call_transform(NULL, tramp, func);
+ arch_static_call_transform(NULL, tramp, func, false);
/*
* If uninitialized, we'll not update the callsites, but they still
@@ -154,7 +157,8 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
continue;
}
- arch_static_call_transform(site_addr, NULL, func);
+ arch_static_call_transform(site_addr, NULL, func,
+ static_call_is_tail(site));
}
}
@@ -198,7 +202,8 @@ static int __static_call_init(struct module *mod,
key->mods = site_mod;
}
- arch_static_call_transform(site_addr, NULL, key->func);
+ arch_static_call_transform(site_addr, NULL, key->func,
+ static_call_is_tail(site));
}
return 0;
diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h
index 408d345..89135bb 100644
--- a/tools/include/linux/static_call_types.h
+++ b/tools/include/linux/static_call_types.h
@@ -17,6 +17,13 @@
#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
/*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL /* tail call */
+#define STATIC_CALL_SITE_INIT 2UL /* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
* The static call site table needs to be created by external tooling (objtool
* or a compiler plugin).
*/
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index f8f7a40..75d0cd2 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -516,7 +516,7 @@ static int create_static_call_sections(struct objtool_file *file)
}
memset(reloc, 0, sizeof(*reloc));
reloc->sym = key_sym;
- reloc->addend = 0;
+ reloc->addend = is_sibling_call(insn) ? STATIC_CALL_SITE_TAIL : 0;
reloc->type = R_X86_64_PC32;
reloc->offset = idx * sizeof(struct static_call_site) + 4;
reloc->sec = reloc_sec;
@@ -747,6 +747,10 @@ static int add_jump_destinations(struct objtool_file *file)
} else {
/* external sibling call */
insn->call_dest = reloc->sym;
+ if (insn->call_dest->static_call_tramp) {
+ list_add_tail(&insn->static_call_node,
+ &file->static_call_list);
+ }
continue;
}
@@ -798,6 +802,10 @@ static int add_jump_destinations(struct objtool_file *file)
/* internal sibling call */
insn->call_dest = insn->jump_dest->func;
+ if (insn->call_dest->static_call_tramp) {
+ list_add_tail(&insn->static_call_node,
+ &file->static_call_list);
+ }
}
}
}
@@ -1684,6 +1692,10 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
+ ret = read_static_call_tramps(file);
+ if (ret)
+ return ret;
+
ret = add_jump_destinations(file);
if (ret)
return ret;
@@ -1716,10 +1728,6 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
- ret = read_static_call_tramps(file);
- if (ret)
- return ret;
-
return 0;
}
next prev parent reply other threads:[~2020-09-01 11:57 UTC|newest]
Thread overview: 41+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-08-18 13:57 [PATCH v7 00/18] Add static_call Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 01/18] notifier: Fix broken error handling pattern Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 02/18] module: Fix up module_notifier return values Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 03/18] module: Properly propagate MODULE_STATE_COMING failure Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 04/18] jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] jump_label,module: Fix module lifetime for __jump_label_mod_text_reserved() tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 05/18] compiler.h: Make __ADDRESSABLE() symbol truly unique Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Josh Poimboeuf
2020-08-18 13:57 ` [PATCH v7 06/18] static_call: Add basic static call infrastructure Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Josh Poimboeuf
2020-08-18 13:57 ` [PATCH v7 07/18] static_call: Add inline " Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Josh Poimboeuf
2020-08-18 13:57 ` [PATCH v7 08/18] static_call: Avoid kprobes on inline static_call()s Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-09-02 1:35 ` [PATCH v7 08/18] " Masami Hiramatsu
2020-09-02 9:48 ` peterz
2020-09-02 10:16 ` Masami Hiramatsu
2020-09-02 12:01 ` peterz
2020-08-18 13:57 ` [PATCH v7 09/18] x86/static_call: Add out-of-line static call implementation Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Josh Poimboeuf
2020-08-18 13:57 ` [PATCH v7 10/18] x86/static_call: Add inline static call implementation for x86-64 Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Josh Poimboeuf
2020-08-18 13:57 ` [PATCH v7 11/18] static_call: Simple self-test Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] static_call: Add simple self-test for static calls tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 12/18] x86/alternatives: Teach text_poke_bp() to emulate RET Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 13/18] static_call: Add static_call_cond() Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 14/18] static_call: Handle tail-calls Peter Zijlstra
2020-09-01 11:48 ` tip-bot2 for Peter Zijlstra [this message]
2020-08-18 13:57 ` [PATCH v7 15/18] static_call: Add some validation Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 16/18] static_call: Allow early init Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
2020-08-18 13:57 ` [PATCH v7 17/18] tracepoint: Optimize using static_call() Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Steven Rostedt (VMware)
2020-08-18 13:57 ` [PATCH v7 18/18] x86/perf, static_call: Optimize x86_pmu methods Peter Zijlstra
2020-09-01 11:48 ` [tip: core/static_call] " tip-bot2 for Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=159896088374.20229.4926395571390516128.tip-bot2@tip-bot2 \
--to=tip-bot2@linutronix.de \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=torvalds@linux-foundation.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.