From: Valentin Schneider <vschneid@redhat.com>
To: linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org,
linux-doc@vger.kernel.org, kvm@vger.kernel.org,
linux-mm@kvack.org, bpf@vger.kernel.org, x86@kernel.org,
rcu@vger.kernel.org, linux-kselftest@vger.kernel.org
Cc: "Peter Zijlstra" <peterz@infradead.org>,
"Steven Rostedt" <rostedt@goodmis.org>,
"Masami Hiramatsu" <mhiramat@kernel.org>,
"Jonathan Corbet" <corbet@lwn.net>,
"Thomas Gleixner" <tglx@linutronix.de>,
"Ingo Molnar" <mingo@redhat.com>,
"Borislav Petkov" <bp@alien8.de>,
"Dave Hansen" <dave.hansen@linux.intel.com>,
"H. Peter Anvin" <hpa@zytor.com>,
"Paolo Bonzini" <pbonzini@redhat.com>,
"Wanpeng Li" <wanpengli@tencent.com>,
"Vitaly Kuznetsov" <vkuznets@redhat.com>,
"Andy Lutomirski" <luto@kernel.org>,
"Frederic Weisbecker" <frederic@kernel.org>,
"Paul E. McKenney" <paulmck@kernel.org>,
"Neeraj Upadhyay" <quic_neeraju@quicinc.com>,
"Joel Fernandes" <joel@joelfernandes.org>,
"Josh Triplett" <josh@joshtriplett.org>,
"Boqun Feng" <boqun.feng@gmail.com>,
"Mathieu Desnoyers" <mathieu.desnoyers@efficios.com>,
"Lai Jiangshan" <jiangshanlai@gmail.com>,
Zqiang <qiang.zhang1211@gmail.com>,
"Andrew Morton" <akpm@linux-foundation.org>,
"Uladzislau Rezki" <urezki@gmail.com>,
"Christoph Hellwig" <hch@infradead.org>,
"Lorenzo Stoakes" <lstoakes@gmail.com>,
"Josh Poimboeuf" <jpoimboe@kernel.org>,
"Jason Baron" <jbaron@akamai.com>,
"Kees Cook" <keescook@chromium.org>,
"Sami Tolvanen" <samitolvanen@google.com>,
"Ard Biesheuvel" <ardb@kernel.org>,
"Nicholas Piggin" <npiggin@gmail.com>,
"Juerg Haefliger" <juerg.haefliger@canonical.com>,
"Nicolas Saenz Julienne" <nsaenz@kernel.org>,
"Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>,
"Nadav Amit" <namit@vmware.com>,
"Dan Carpenter" <error27@gmail.com>,
"Chuang Wang" <nashuiliang@gmail.com>,
"Yang Jihong" <yangjihong1@huawei.com>,
"Petr Mladek" <pmladek@suse.com>,
"Jason A. Donenfeld" <Jason@zx2c4.com>,
"Song Liu" <song@kernel.org>,
"Julian Pidancet" <julian.pidancet@oracle.com>,
"Tom Lendacky" <thomas.lendacky@amd.com>,
"Dionna Glaze" <dionnaglaze@google.com>,
"Thomas Weißschuh" <linux@weissschuh.net>,
"Juri Lelli" <juri.lelli@redhat.com>,
"Daniel Bristot de Oliveira" <bristot@redhat.com>,
"Marcelo Tosatti" <mtosatti@redhat.com>,
"Yair Podemsky" <ypodemsk@redhat.com>
Subject: [RFC PATCH v2 10/20] jump_label,module: Don't alloc static_key_mod for __ro_after_init keys
Date: Thu, 20 Jul 2023 17:30:46 +0100 [thread overview]
Message-ID: <20230720163056.2564824-11-vschneid@redhat.com> (raw)
In-Reply-To: <20230720163056.2564824-1-vschneid@redhat.com>
From: Peter Zijlstra <peterz@infradead.org>
When a static_key is marked ro_after_init, its state will never change
(after init), therefore jump_label_update() will never need to iterate
the entries, and thus module load won't actually need to track this --
avoiding the static_key::next write.
Therefore, mark these keys such that jump_label_add_module() might
recognise them and avoid the modification.
Use the special state: 'static_key_linked(key) && !static_key_mod(key)'
to denote such keys.
Link: http://lore.kernel.org/r/20230705204142.GB2813335@hirez.programming.kicks-ass.net
NOT-Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Valentin Schneider <vschneid@redhat.com>
---
@Peter: I've barely touched this patch, it's just been writing a comment
and fixing benign compilation issues, so credit's all yours really!
---
include/asm-generic/sections.h | 5 ++++
include/linux/jump_label.h | 1 +
init/main.c | 1 +
kernel/jump_label.c | 49 ++++++++++++++++++++++++++++++++++
4 files changed, 56 insertions(+)
diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h
index db13bb620f527..c768de6f19a9a 100644
--- a/include/asm-generic/sections.h
+++ b/include/asm-generic/sections.h
@@ -180,6 +180,11 @@ static inline bool is_kernel_rodata(unsigned long addr)
addr < (unsigned long)__end_rodata;
}
+static inline bool is_kernel_ro_after_init(unsigned long addr)
+{
+ return addr >= (unsigned long)__start_ro_after_init &&
+ addr < (unsigned long)__end_ro_after_init;
+}
/**
* is_kernel_inittext - checks if the pointer address is located in the
* .init.text section
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f0a949b7c9733..88ef9e776af8d 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -216,6 +216,7 @@ extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];
extern void jump_label_init(void);
+extern void jump_label_ro(void);
extern void jump_label_lock(void);
extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
diff --git a/init/main.c b/init/main.c
index ad920fac325c3..cb5304ca18f4d 100644
--- a/init/main.c
+++ b/init/main.c
@@ -1403,6 +1403,7 @@ static void mark_readonly(void)
* insecure pages which are W+X.
*/
rcu_barrier();
+ jump_label_ro();
mark_rodata_ro();
rodata_test();
} else
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index d9c822bbffb8d..661ef74dee9b7 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -530,6 +530,45 @@ void __init jump_label_init(void)
cpus_read_unlock();
}
+static inline bool static_key_sealed(struct static_key *key)
+{
+ return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK);
+}
+
+static inline void static_key_seal(struct static_key *key)
+{
+ unsigned long type = key->type & JUMP_TYPE_TRUE;
+ key->type = JUMP_TYPE_LINKED | type;
+}
+
+void jump_label_ro(void)
+{
+ struct jump_entry *iter_start = __start___jump_table;
+ struct jump_entry *iter_stop = __stop___jump_table;
+ struct jump_entry *iter;
+
+ if (WARN_ON_ONCE(!static_key_initialized))
+ return;
+
+ cpus_read_lock();
+ jump_label_lock();
+
+ for (iter = iter_start; iter < iter_stop; iter++) {
+ struct static_key *iterk = jump_entry_key(iter);
+
+ if (!is_kernel_ro_after_init((unsigned long)iterk))
+ continue;
+
+ if (static_key_sealed(iterk))
+ continue;
+
+ static_key_seal(iterk);
+ }
+
+ jump_label_unlock();
+ cpus_read_unlock();
+}
+
#ifdef CONFIG_MODULES
enum jump_label_type jump_label_init_type(struct jump_entry *entry)
@@ -650,6 +689,15 @@ static int jump_label_add_module(struct module *mod)
static_key_set_entries(key, iter);
continue;
}
+
+ /*
+ * If the key was sealed at init, then there's no need to keep a
+ * a reference to its module entries - just patch them now and
+ * be done with it.
+ */
+ if (static_key_sealed(key))
+ goto do_poke;
+
jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
@@ -675,6 +723,7 @@ static int jump_label_add_module(struct module *mod)
static_key_set_linked(key);
/* Only update if we've changed from our initial state */
+do_poke:
if (jump_label_type(iter) != jump_label_init_type(iter))
__jump_label_update(key, iter, iter_stop, true);
}
--
2.31.1
next prev parent reply other threads:[~2023-07-20 16:33 UTC|newest]
Thread overview: 76+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-07-20 16:30 [RFC PATCH v2 00/20] context_tracking,x86: Defer some IPIs until a user->kernel transition Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 01/20] tracing/filters: Dynamically allocate filter_pred.regex Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 02/20] tracing/filters: Enable filtering a cpumask field by another cpumask Valentin Schneider
2023-07-26 19:41 ` Josh Poimboeuf
2023-07-27 9:46 ` Valentin Schneider
2023-07-29 19:09 ` Steven Rostedt
2023-07-31 11:19 ` Valentin Schneider
2023-07-31 15:48 ` Steven Rostedt
2023-07-20 16:30 ` [RFC PATCH v2 03/20] tracing/filters: Enable filtering a scalar field by a cpumask Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 04/20] tracing/filters: Enable filtering the CPU common " Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 05/20] tracing/filters: Optimise cpumask vs cpumask filtering when user mask is a single CPU Valentin Schneider
2023-07-29 19:34 ` Steven Rostedt
2023-07-31 11:20 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 06/20] tracing/filters: Optimise scalar vs cpumask filtering when the " Valentin Schneider
2023-07-29 19:55 ` Steven Rostedt
2023-07-31 11:20 ` Valentin Schneider
2023-07-31 12:07 ` Dan Carpenter
2023-07-31 15:54 ` Steven Rostedt
2023-07-31 16:03 ` Dan Carpenter
2023-07-31 17:20 ` Valentin Schneider
2023-07-31 18:16 ` Steven Rostedt
2023-07-20 16:30 ` [RFC PATCH v2 07/20] tracing/filters: Optimise CPU " Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 08/20] tracing/filters: Further optimise scalar vs cpumask comparison Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 09/20] tracing/filters: Document cpumask filtering Valentin Schneider
2023-07-20 16:30 ` Valentin Schneider [this message]
2023-07-28 22:04 ` [RFC PATCH v2 10/20] jump_label,module: Don't alloc static_key_mod for __ro_after_init keys Peter Zijlstra
2023-07-20 16:30 ` [RFC PATCH v2 11/20] objtool: Flesh out warning related to pv_ops[] calls Valentin Schneider
2023-07-28 15:33 ` Josh Poimboeuf
2023-07-31 11:16 ` Valentin Schneider
2023-07-31 21:36 ` Josh Poimboeuf
2023-07-31 21:46 ` Peter Zijlstra
2023-08-01 16:06 ` Josh Poimboeuf
2023-08-01 18:12 ` Peter Zijlstra
2023-07-20 16:30 ` [RFC PATCH v2 12/20] objtool: Warn about non __ro_after_init static key usage in .noinstr Valentin Schneider
2023-07-28 15:35 ` Josh Poimboeuf
2023-07-31 11:18 ` Valentin Schneider
2023-07-28 16:02 ` Josh Poimboeuf
2023-07-31 11:18 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 13/20] context_tracking: Make context_tracking_key __ro_after_init Valentin Schneider
2023-07-28 16:00 ` Josh Poimboeuf
2023-07-31 11:16 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 14/20] x86/kvm: Make kvm_async_pf_enabled __ro_after_init Valentin Schneider
2023-10-09 16:40 ` Maxim Levitsky
2023-07-20 16:30 ` [RFC PATCH v2 15/20] context-tracking: Introduce work deferral infrastructure Valentin Schneider
2023-07-24 14:52 ` Frederic Weisbecker
2023-07-24 16:55 ` Valentin Schneider
2023-07-24 19:18 ` Frederic Weisbecker
2023-07-25 10:10 ` Valentin Schneider
2023-07-25 11:22 ` Frederic Weisbecker
2023-07-25 13:05 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 16/20] rcu: Make RCU dynticks counter size configurable Valentin Schneider
2023-07-21 8:17 ` Valentin Schneider
2023-07-21 14:10 ` Paul E. McKenney
2023-07-21 15:08 ` Valentin Schneider
2023-07-21 16:09 ` Paul E. McKenney
2023-07-20 16:30 ` [RFC PATCH v2 17/20] rcutorture: Add a test config to torture test low RCU_DYNTICKS width Valentin Schneider
2023-07-20 19:53 ` Paul E. McKenney
2023-07-21 4:00 ` Paul E. McKenney
2023-07-21 7:58 ` Valentin Schneider
2023-07-21 14:07 ` Paul E. McKenney
2023-07-21 15:08 ` Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 18/20] context_tracking,x86: Defer kernel text patching IPIs Valentin Schneider
2023-07-25 10:49 ` Joel Fernandes
2023-07-25 13:36 ` Valentin Schneider
2023-07-25 17:41 ` Joel Fernandes
2023-07-25 13:39 ` Peter Zijlstra
2023-07-25 17:47 ` Joel Fernandes
2023-07-20 16:30 ` [RFC PATCH v2 19/20] context_tracking,x86: Add infrastructure to defer kernel TLBI Valentin Schneider
2023-07-20 16:30 ` [RFC PATCH v2 20/20] x86/mm, mm/vmalloc: Defer flush_tlb_kernel_range() targeting NOHZ_FULL CPUs Valentin Schneider
2023-07-21 18:15 ` Nadav Amit
2023-07-24 11:32 ` Valentin Schneider
2023-07-24 17:40 ` Dave Hansen
2023-07-25 13:21 ` Peter Zijlstra
2023-07-25 14:03 ` Valentin Schneider
2023-07-25 16:37 ` Marcelo Tosatti
2023-07-25 17:12 ` Dave Hansen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230720163056.2564824-11-vschneid@redhat.com \
--to=vschneid@redhat.com \
--cc=Jason@zx2c4.com \
--cc=akpm@linux-foundation.org \
--cc=ardb@kernel.org \
--cc=boqun.feng@gmail.com \
--cc=bp@alien8.de \
--cc=bpf@vger.kernel.org \
--cc=bristot@redhat.com \
--cc=corbet@lwn.net \
--cc=dave.hansen@linux.intel.com \
--cc=dionnaglaze@google.com \
--cc=error27@gmail.com \
--cc=frederic@kernel.org \
--cc=hch@infradead.org \
--cc=hpa@zytor.com \
--cc=jbaron@akamai.com \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=jpoimboe@kernel.org \
--cc=juerg.haefliger@canonical.com \
--cc=julian.pidancet@oracle.com \
--cc=juri.lelli@redhat.com \
--cc=keescook@chromium.org \
--cc=kirill.shutemov@linux.intel.com \
--cc=kvm@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-trace-kernel@vger.kernel.org \
--cc=linux@weissschuh.net \
--cc=lstoakes@gmail.com \
--cc=luto@kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mhiramat@kernel.org \
--cc=mingo@redhat.com \
--cc=mtosatti@redhat.com \
--cc=namit@vmware.com \
--cc=nashuiliang@gmail.com \
--cc=npiggin@gmail.com \
--cc=nsaenz@kernel.org \
--cc=paulmck@kernel.org \
--cc=pbonzini@redhat.com \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=qiang.zhang1211@gmail.com \
--cc=quic_neeraju@quicinc.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=samitolvanen@google.com \
--cc=song@kernel.org \
--cc=tglx@linutronix.de \
--cc=thomas.lendacky@amd.com \
--cc=urezki@gmail.com \
--cc=vkuznets@redhat.com \
--cc=wanpengli@tencent.com \
--cc=x86@kernel.org \
--cc=yangjihong1@huawei.com \
--cc=ypodemsk@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).