From: Josh Poimboeuf <jpoimboe@redhat.com>
To: Peter Zijlstra <peterz@infradead.org>
Cc: Frederic Weisbecker <frederic@kernel.org>,
LKML <linux-kernel@vger.kernel.org>, Mel Gorman <mgorman@suse.de>,
Michal Hocko <mhocko@kernel.org>,
Thomas Gleixner <tglx@linutronix.de>,
"Paul E . McKenney" <paulmck@kernel.org>,
Ingo Molnar <mingo@redhat.com>, Michal Hocko <mhocko@suse.com>,
rostedt@goodmis.org, jbaron@akamai.com, ardb@kernel.org
Subject: Re: [RFC PATCH 6/8] preempt/dynamic: Provide preempt_schedule[_notrace]() static calls
Date: Wed, 27 Jan 2021 17:18:37 -0600 [thread overview]
Message-ID: <20210127231837.ifddpn7rhwdaepiu@treble> (raw)
In-Reply-To: <20210127190218.hoztl7eidujqarkt@treble>
On Wed, Jan 27, 2021 at 01:02:18PM -0600, Josh Poimboeuf wrote:
> On Wed, Jan 27, 2021 at 01:00:07PM -0600, Josh Poimboeuf wrote:
> > On Wed, Jan 27, 2021 at 07:44:01PM +0100, Peter Zijlstra wrote:
> > > On Wed, Jan 27, 2021 at 10:33:08AM -0600, Josh Poimboeuf wrote:
> > >
> > > > What did you think about .static_call_tramp_key? I could whip up a
> > > > patch later unless you beat me to it.
> > >
> > > Yeah, I'm not sure.. why duplicate information already present in
> > > kallsyms?
> >
> > Well, but it's not exactly duplicating kallsyms. No need to store
> > symbol names, just the pointer relationships. And kallsyms is
> > presumably slow.
> >
> > > There's a fair number of features that already require KALLSYMS, I can't
> > > really be bothered about adding one more (kprobes, function_tracer,
> > > stack_tracer, ftrace_syscalls).
Here ya go. It builds... And the tramp_key section is nice and small.
Relocation section [1497] '.rela.static_call_tramp_key' for section [1496] '.static_call_tramp_key' at offset 0x179ab818 contains 8 entries:
Offset Type Value Addend Name
000000000000000000 X86_64_PC32 0x00000000000004c0 +0 __SCT__preempt_schedule
0x0000000000000004 X86_64_PC32 0x000000000005ee10 +0 __SCK__preempt_schedule
0x0000000000000008 X86_64_PC32 0x00000000000004c8 +0 __SCT__preempt_schedule_notrace
0x000000000000000c X86_64_PC32 0x000000000005ee00 +0 __SCK__preempt_schedule_notrace
0x0000000000000010 X86_64_PC32 0x00000000000004d0 +0 __SCT__cond_resched
0x0000000000000014 X86_64_PC32 0x000000000005dd20 +0 __SCK__cond_resched
0x0000000000000018 X86_64_PC32 0x00000000000004d8 +0 __SCT__might_resched
0x000000000000001c X86_64_PC32 0x000000000005dd10 +0 __SCK__might_resched
diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h
index c37f11999d0c..cbb67b6030f9 100644
--- a/arch/x86/include/asm/static_call.h
+++ b/arch/x86/include/asm/static_call.h
@@ -37,4 +37,11 @@
#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \
__ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop")
+
+#define ARCH_ADD_TRAMP_KEY(name) \
+ asm(".pushsection .static_call_tramp_key, \"a\" \n" \
+ ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \
+ ".long " STATIC_CALL_KEY_STR(name) " - . \n" \
+ ".popsection \n")
+
#endif /* _ASM_STATIC_CALL_H */
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index b2b3d81b1535..b0871e282c4f 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -393,7 +393,10 @@
. = ALIGN(8); \
__start_static_call_sites = .; \
KEEP(*(.static_call_sites)) \
- __stop_static_call_sites = .;
+ __stop_static_call_sites = .; \
+ __start_static_call_tramp_key = .; \
+ KEEP(*(.static_call_tramp_key)) \
+ __stop_static_call_tramp_key = .;
/*
* Allow architectures to handle ro_after_init data on their
diff --git a/include/linux/static_call.h b/include/linux/static_call.h
index 077330874c60..16bcd5af3d35 100644
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -138,6 +138,12 @@ struct static_call_key {
};
};
+/* For finding the key associated with a trampoline */
+struct static_call_tramp_key {
+ s32 tramp;
+ s32 key;
+};
+
extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
extern int static_call_mod_init(struct module *mod);
extern int static_call_text_reserved(void *start, void *end);
@@ -165,11 +171,18 @@ extern long __static_call_return0(void);
#define EXPORT_STATIC_CALL(name) \
EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
-
#define EXPORT_STATIC_CALL_GPL(name) \
EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+/* Leave the key unexported, so modules can't change static call targets: */
+#define EXPORT_STATIC_CALL_TRAMP(name) \
+ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
+ ARCH_ADD_TRAMP_KEY(name)
+#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
+ ARCH_ADD_TRAMP_KEY(name)
+
#elif defined(CONFIG_HAVE_STATIC_CALL)
static inline int static_call_init(void) { return 0; }
@@ -216,11 +229,16 @@ static inline long __static_call_return0(void)
#define EXPORT_STATIC_CALL(name) \
EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
-
#define EXPORT_STATIC_CALL_GPL(name) \
EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+/* Leave the key unexported, so modules can't change static call targets: */
+#define EXPORT_STATIC_CALL_TRAMP(name) \
+ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+
#else /* Generic implementation */
static inline int static_call_init(void) { return 0; }
diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h
index 08f78b1b88b4..ae5662d368b9 100644
--- a/include/linux/static_call_types.h
+++ b/include/linux/static_call_types.h
@@ -10,6 +10,7 @@
#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX)
#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)
#define STATIC_CALL_KEY(name) __PASTE(STATIC_CALL_KEY_PREFIX, name)
+#define STATIC_CALL_KEY_STR(name) __stringify(STATIC_CALL_KEY(name))
#define STATIC_CALL_TRAMP_PREFIX __SCT__
#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX)
@@ -39,17 +40,39 @@ struct static_call_site {
#ifdef CONFIG_HAVE_STATIC_CALL
+#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name))
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
/*
* __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
* the symbol table so that objtool can reference it when it generates the
* .static_call_sites section.
*/
+#define __STATIC_CALL_ADDRESSABLE(name) \
+ __ADDRESSABLE(STATIC_CALL_KEY(name))
+
#define __static_call(name) \
({ \
- __ADDRESSABLE(STATIC_CALL_KEY(name)); \
- &STATIC_CALL_TRAMP(name); \
+ __STATIC_CALL_ADDRESSABLE(name); \
+ __raw_static_call(name); \
})
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
+#define __STATIC_CALL_ADDRESSABLE(name)
+#define __static_call(name) __raw_static_call(name)
+
+#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */
+
+#ifdef MODULE
+#define __STATIC_CALL_MOD_ADDRESSABLE(name)
+#define static_call_mod(name) __raw_static_call(name)
+#else
+#define __STATIC_CALL_MOD_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name)
+#define static_call_mod(name) __static_call(name)
+#endif
+
#define static_call(name) __static_call(name)
#else
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 66129245b6a0..9f4564b89e9f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5268,7 +5268,7 @@ EXPORT_SYMBOL(preempt_schedule);
#ifdef CONFIG_PREEMPT_DYNAMIC
DEFINE_STATIC_CALL(preempt_schedule, __preempt_schedule_func());
-EXPORT_STATIC_CALL(preempt_schedule);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
#endif
@@ -5326,7 +5326,7 @@ EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
#ifdef CONFIG_PREEMPT_DYNAMIC
DEFINE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func());
-EXPORT_STATIC_CALL(preempt_schedule_notrace);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
#endif
#endif /* CONFIG_PREEMPTION */
@@ -6993,10 +6993,10 @@ EXPORT_SYMBOL(__cond_resched);
#ifdef CONFIG_PREEMPT_DYNAMIC
DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
-EXPORT_STATIC_CALL(cond_resched);
+EXPORT_STATIC_CALL_TRAMP(cond_resched);
DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
-EXPORT_STATIC_CALL(might_resched);
+EXPORT_STATIC_CALL_TRAMP(might_resched);
#endif
/*
diff --git a/kernel/static_call.c b/kernel/static_call.c
index 0bc11b5ce681..5e6f567976c1 100644
--- a/kernel/static_call.c
+++ b/kernel/static_call.c
@@ -12,6 +12,8 @@
extern struct static_call_site __start_static_call_sites[],
__stop_static_call_sites[];
+extern struct static_call_tramp_key __start_static_call_tramp_key[],
+ __stop_static_call_tramp_key[];
static bool static_call_initialized;
@@ -323,10 +325,59 @@ static int __static_call_mod_text_reserved(void *start, void *end)
return ret;
}
+static struct static_call_tramp_key *tramp_key_lookup(unsigned long addr)
+{
+ struct static_call_tramp_key *start = __start_static_call_tramp_key;
+ struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
+ struct static_call_tramp_key *tramp_key;
+
+ for (tramp_key = start; tramp_key != stop; tramp_key++) {
+ unsigned long tramp;
+
+ tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
+ if (tramp == addr)
+ return tramp_key;
+ }
+
+ return NULL;
+}
+
static int static_call_add_module(struct module *mod)
{
- return __static_call_init(mod, mod->static_call_sites,
- mod->static_call_sites + mod->num_static_call_sites);
+ struct static_call_site *start = mod->static_call_sites;
+ struct static_call_site *stop = start + mod->num_static_call_sites;
+ struct static_call_site *site;
+
+ for (site = start; site != stop; site++) {
+ unsigned long addr = (unsigned long)static_call_key(site);
+ struct static_call_tramp_key *tramp_key;
+
+ /*
+ * Is the key is exported, 'addr' points to the key, which
+ * means modules are allowed to call static_call_update() on
+ * it.
+ *
+ * Otherwise, the key isn't exported, and 'addr' points to the
+ * trampoline so we need to lookup the key.
+ *
+ * We go through this dance to prevent crazy modules from
+ * abusing sensitive static calls.
+ */
+ if (!kernel_text_address(addr))
+ continue;
+
+ tramp_key = tramp_key_lookup(addr);
+ if (!tramp_key) {
+ pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
+ static_call_addr(site));
+ return -EINVAL;
+ }
+
+ site->key = ((long)tramp_key->key - (long)&tramp_key->key) |
+ (site->key & STATIC_CALL_SITE_FLAGS);
+ }
+
+ return __static_call_init(mod, start, stop);
}
static void static_call_del_module(struct module *mod)
diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h
index 08f78b1b88b4..2a3afb6ebf49 100644
--- a/tools/include/linux/static_call_types.h
+++ b/tools/include/linux/static_call_types.h
@@ -39,17 +39,39 @@ struct static_call_site {
#ifdef CONFIG_HAVE_STATIC_CALL
+#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name))
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
/*
* __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
* the symbol table so that objtool can reference it when it generates the
* .static_call_sites section.
*/
+#define __STATIC_CALL_ADDRESSABLE(name) \
+ __ADDRESSABLE(STATIC_CALL_KEY(name))
+
#define __static_call(name) \
({ \
- __ADDRESSABLE(STATIC_CALL_KEY(name)); \
- &STATIC_CALL_TRAMP(name); \
+ __STATIC_CALL_ADDRESSABLE(name); \
+ __raw_static_call(name); \
})
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
+
+#define __STATIC_CALL_ADDRESSABLE(name)
+#define __static_call(name) __raw_static_call(name)
+
+#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */
+
+#ifdef MODULE
+#define __STATIC_CALL_MOD_ADDRESSABLE(name)
+#define static_call_mod(name) __raw_static_call(name)
+#else
+#define __STATIC_CALL_MOD_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name)
+#define static_call_mod(name) __static_call(name)
+#endif
+
#define static_call(name) __static_call(name)
#else
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 4bd30315eb62..f2e5e5ce1a05 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -502,8 +502,21 @@ static int create_static_call_sections(struct objtool_file *file)
key_sym = find_symbol_by_name(file->elf, tmp);
if (!key_sym) {
- WARN("static_call: can't find static_call_key symbol: %s", tmp);
- return -1;
+ if (!module) {
+ WARN("static_call: can't find static_call_key symbol: %s", tmp);
+ return -1;
+ }
+
+ /*
+ * For modules(), the key might not be exported, which
+ * means the module can make static calls but isn't
+ * allowed to change them.
+ *
+ * In that case we temporarily set the key to be the
+ * trampoline address. This is fixed up in
+ * static_call_add_module().
+ */
+ key_sym = insn->call_dest;
}
free(key_name);
next prev parent reply other threads:[~2021-01-27 23:21 UTC|newest]
Thread overview: 59+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-18 14:12 [RFC PATCH 0/7] preempt: Tune preemption flavour on boot v4 Frederic Weisbecker
2021-01-18 14:12 ` [RFC PATCH 1/8] static_call/x86: Add __static_call_return0() Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Peter Zijlstra
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra
2021-01-18 14:12 ` [RFC PATCH 2/8] static_call: Provide DEFINE_STATIC_CALL_RET0() Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Frederic Weisbecker
2021-02-17 13:17 ` tip-bot2 for Frederic Weisbecker
2021-01-18 14:12 ` [RFC PATCH 3/8] static_call: Pull some static_call declarations to the type headers Frederic Weisbecker
2021-01-19 10:46 ` Jürgen Groß
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Peter Zijlstra
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra
2021-01-18 14:12 ` [RFC PATCH 4/8] preempt: Introduce CONFIG_PREEMPT_DYNAMIC Frederic Weisbecker
2021-01-22 16:53 ` Peter Zijlstra
2021-01-28 12:17 ` Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Michal Hocko
2021-02-17 13:17 ` tip-bot2 for Michal Hocko
2021-01-18 14:12 ` [RFC PATCH 5/8] preempt/dynamic: Provide cond_resched() and might_resched() static calls Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Peter Zijlstra (Intel)
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra (Intel)
2021-01-18 14:12 ` [RFC PATCH 6/8] preempt/dynamic: Provide preempt_schedule[_notrace]() " Frederic Weisbecker
2021-01-21 21:58 ` Peter Zijlstra
2021-01-21 22:25 ` Peter Zijlstra
2021-01-22 16:52 ` Peter Zijlstra
2021-01-22 16:57 ` Ard Biesheuvel
2021-01-22 17:08 ` Peter Zijlstra
2021-02-08 12:00 ` [tip: sched/core] sched: Add /debug/sched_preempt tip-bot2 for Peter Zijlstra
2021-02-09 15:45 ` tip-bot2 for Peter Zijlstra
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra
2021-01-25 23:40 ` [RFC PATCH 6/8] preempt/dynamic: Provide preempt_schedule[_notrace]() static calls Josh Poimboeuf
2021-01-26 9:24 ` Peter Zijlstra
2021-01-26 23:57 ` Josh Poimboeuf
2021-01-27 9:13 ` Peter Zijlstra
2021-01-27 11:27 ` Peter Zijlstra
2021-01-27 15:59 ` Josh Poimboeuf
2021-01-27 16:19 ` Peter Zijlstra
2021-01-27 16:33 ` Josh Poimboeuf
2021-01-27 18:44 ` Peter Zijlstra
2021-01-27 19:00 ` Josh Poimboeuf
2021-01-27 19:02 ` Josh Poimboeuf
2021-01-27 23:18 ` Josh Poimboeuf [this message]
2021-02-03 14:04 ` Peter Zijlstra
2021-02-05 15:30 ` Peter Zijlstra
2021-02-06 2:31 ` Josh Poimboeuf
2021-02-06 9:03 ` Peter Zijlstra
2021-02-05 15:22 ` Peter Zijlstra
2021-02-08 12:00 ` [tip: sched/core] static_call: Allow module use without exposing static_call_key tip-bot2 for Josh Poimboeuf
2021-02-09 15:45 ` tip-bot2 for Josh Poimboeuf
2021-02-17 13:17 ` tip-bot2 for Josh Poimboeuf
2021-02-08 12:00 ` [tip: sched/core] preempt/dynamic: Provide preempt_schedule[_notrace]() static calls tip-bot2 for Peter Zijlstra (Intel)
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra (Intel)
2021-01-18 14:12 ` [RFC PATCH 7/8] preempt/dynamic: Provide irqentry_exit_cond_resched() static call Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Peter Zijlstra (Intel)
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra (Intel)
2021-01-18 14:12 ` [RFC PATCH 8/8] preempt/dynamic: Support dynamic preempt with preempt= boot option Frederic Weisbecker
2021-02-08 12:00 ` [tip: sched/core] " tip-bot2 for Peter Zijlstra (Intel)
2021-02-09 15:45 ` tip-bot2 for Peter Zijlstra (Intel)
2021-02-17 13:17 ` tip-bot2 for Peter Zijlstra (Intel)
2021-01-21 21:22 ` [RFC PATCH 0/7] preempt: Tune preemption flavour on boot v4 Peter Zijlstra
2021-01-22 15:02 ` Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210127231837.ifddpn7rhwdaepiu@treble \
--to=jpoimboe@redhat.com \
--cc=ardb@kernel.org \
--cc=frederic@kernel.org \
--cc=jbaron@akamai.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mgorman@suse.de \
--cc=mhocko@kernel.org \
--cc=mhocko@suse.com \
--cc=mingo@redhat.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).