From: Ard Biesheuvel <ard.biesheuvel@linaro.org> To: linux-kernel@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org, linux-s390@vger.kernel.org, Ard Biesheuvel <ard.biesheuvel@linaro.org>, Arnd Bergmann <arnd@arndb.de>, Heiko Carstens <heiko.carstens@de.ibm.com>, Kees Cook <keescook@chromium.org>, Will Deacon <will.deacon@arm.com>, Thomas Gleixner <tglx@linutronix.de>, Catalin Marinas <catalin.marinas@arm.com>, Ingo Molnar <mingo@redhat.com>, Steven Rostedt <rostedt@goodmis.org>, Martin Schwidefsky <schwidefsky@de.ibm.com>, Jessica Yu <jeyu@kernel.org>, Peter Zijlstra <peterz@infradead.org> Subject: [PATCH v3 1/9] kernel/jump_label: abstract jump_entry member accessors Date: Tue, 18 Sep 2018 23:51:36 -0700 [thread overview] Message-ID: <20180919065144.25010-2-ard.biesheuvel@linaro.org> (raw) In-Reply-To: <20180919065144.25010-1-ard.biesheuvel@linaro.org> In preparation of allowing architectures to use relative references in jump_label entries [which can dramatically reduce the memory footprint], introduce abstractions for references to the 'code' and 'key' members of struct jump_entry. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> --- include/linux/jump_label.h | 34 +++++++++++++++++ kernel/jump_label.c | 40 ++++++++------------ 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 1a0b6f17a5d6..2eadff9b3b90 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -119,6 +119,40 @@ struct static_key { #ifdef HAVE_JUMP_LABEL #include <asm/jump_label.h> + +#ifndef __ASSEMBLY__ + +static inline unsigned long jump_entry_code(const struct jump_entry *entry) +{ + return entry->code; +} + +static inline unsigned long jump_entry_target(const struct jump_entry *entry) +{ + return entry->target; +} + +static inline struct static_key *jump_entry_key(const struct jump_entry *entry) +{ + return (struct static_key *)((unsigned long)entry->key & ~1UL); +} + +static inline bool jump_entry_is_branch(const struct jump_entry *entry) +{ + return (unsigned long)entry->key & 1UL; +} + +static inline bool jump_entry_is_init(const struct jump_entry *entry) +{ + return entry->code == 0; +} + +static inline void jump_entry_set_init(struct jump_entry *entry) +{ + entry->code = 0; +} + +#endif #endif #ifndef __ASSEMBLY__ diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 2e62503bea0d..834e43de0daf 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -38,10 +38,10 @@ static int jump_label_cmp(const void *a, const void *b) const struct jump_entry *jea = a; const struct jump_entry *jeb = b; - if (jea->key < jeb->key) + if (jump_entry_key(jea) < jump_entry_key(jeb)) return -1; - if (jea->key > jeb->key) + if (jump_entry_key(jea) > jump_entry_key(jeb)) return 1; return 0; @@ -261,8 +261,8 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit); static int addr_conflict(struct jump_entry *entry, void *start, void *end) { - if (entry->code <= (unsigned long)end && - entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) + if (jump_entry_code(entry) <= (unsigned long)end && + jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start) return 1; return 0; @@ -321,16 +321,6 @@ static inline void static_key_set_linked(struct static_key *key) key->type |= JUMP_TYPE_LINKED; } -static inline struct static_key *jump_entry_key(struct jump_entry *entry) -{ - return (struct static_key *)((unsigned long)entry->key & ~1UL); -} - -static bool jump_entry_branch(struct jump_entry *entry) -{ - return (unsigned long)entry->key & 1UL; -} - /*** * A 'struct static_key' uses a union such that it either points directly * to a table of 'struct jump_entry' or to a linked list of modules which in @@ -355,7 +345,7 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry) { struct static_key *key = jump_entry_key(entry); bool enabled = static_key_enabled(key); - bool branch = jump_entry_branch(entry); + bool branch = jump_entry_is_branch(entry); /* See the comment in linux/jump_label.h */ return enabled ^ branch; @@ -370,12 +360,12 @@ static void __jump_label_update(struct static_key *key, * An entry->code of 0 indicates an entry which has been * disabled because it was in an init text area. */ - if (entry->code) { - if (kernel_text_address(entry->code)) + if (!jump_entry_is_init(entry)) { + if (kernel_text_address(jump_entry_code(entry))) arch_jump_label_transform(entry, jump_label_type(entry)); else WARN_ONCE(1, "can't patch jump_label at %pS", - (void *)(unsigned long)entry->code); + (void *)jump_entry_code(entry)); } } } @@ -430,8 +420,8 @@ void __init jump_label_invalidate_initmem(void) struct jump_entry *iter; for (iter = iter_start; iter < iter_stop; iter++) { - if (init_section_contains((void *)(unsigned long)iter->code, 1)) - iter->code = 0; + if (init_section_contains((void *)jump_entry_code(iter), 1)) + jump_entry_set_init(iter); } } @@ -441,7 +431,7 @@ static enum jump_label_type jump_label_init_type(struct jump_entry *entry) { struct static_key *key = jump_entry_key(entry); bool type = static_key_type(key); - bool branch = jump_entry_branch(entry); + bool branch = jump_entry_is_branch(entry); /* See the comment in linux/jump_label.h */ return type ^ branch; @@ -565,7 +555,7 @@ static int jump_label_add_module(struct module *mod) continue; key = iterk; - if (within_module(iter->key, mod)) { + if (within_module((unsigned long)key, mod)) { static_key_set_entries(key, iter); continue; } @@ -615,7 +605,7 @@ static void jump_label_del_module(struct module *mod) key = jump_entry_key(iter); - if (within_module(iter->key, mod)) + if (within_module((unsigned long)key, mod)) continue; /* No memory during module load */ @@ -659,8 +649,8 @@ static void jump_label_invalidate_module_init(struct module *mod) struct jump_entry *iter; for (iter = iter_start; iter < iter_stop; iter++) { - if (within_module_init(iter->code, mod)) - iter->code = 0; + if (within_module_init(jump_entry_code(iter), mod)) + jump_entry_set_init(iter); } } -- 2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: ard.biesheuvel@linaro.org (Ard Biesheuvel) To: linux-arm-kernel@lists.infradead.org Subject: [PATCH v3 1/9] kernel/jump_label: abstract jump_entry member accessors Date: Tue, 18 Sep 2018 23:51:36 -0700 [thread overview] Message-ID: <20180919065144.25010-2-ard.biesheuvel@linaro.org> (raw) In-Reply-To: <20180919065144.25010-1-ard.biesheuvel@linaro.org> In preparation of allowing architectures to use relative references in jump_label entries [which can dramatically reduce the memory footprint], introduce abstractions for references to the 'code' and 'key' members of struct jump_entry. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> --- include/linux/jump_label.h | 34 +++++++++++++++++ kernel/jump_label.c | 40 ++++++++------------ 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 1a0b6f17a5d6..2eadff9b3b90 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -119,6 +119,40 @@ struct static_key { #ifdef HAVE_JUMP_LABEL #include <asm/jump_label.h> + +#ifndef __ASSEMBLY__ + +static inline unsigned long jump_entry_code(const struct jump_entry *entry) +{ + return entry->code; +} + +static inline unsigned long jump_entry_target(const struct jump_entry *entry) +{ + return entry->target; +} + +static inline struct static_key *jump_entry_key(const struct jump_entry *entry) +{ + return (struct static_key *)((unsigned long)entry->key & ~1UL); +} + +static inline bool jump_entry_is_branch(const struct jump_entry *entry) +{ + return (unsigned long)entry->key & 1UL; +} + +static inline bool jump_entry_is_init(const struct jump_entry *entry) +{ + return entry->code == 0; +} + +static inline void jump_entry_set_init(struct jump_entry *entry) +{ + entry->code = 0; +} + +#endif #endif #ifndef __ASSEMBLY__ diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 2e62503bea0d..834e43de0daf 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -38,10 +38,10 @@ static int jump_label_cmp(const void *a, const void *b) const struct jump_entry *jea = a; const struct jump_entry *jeb = b; - if (jea->key < jeb->key) + if (jump_entry_key(jea) < jump_entry_key(jeb)) return -1; - if (jea->key > jeb->key) + if (jump_entry_key(jea) > jump_entry_key(jeb)) return 1; return 0; @@ -261,8 +261,8 @@ EXPORT_SYMBOL_GPL(jump_label_rate_limit); static int addr_conflict(struct jump_entry *entry, void *start, void *end) { - if (entry->code <= (unsigned long)end && - entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start) + if (jump_entry_code(entry) <= (unsigned long)end && + jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start) return 1; return 0; @@ -321,16 +321,6 @@ static inline void static_key_set_linked(struct static_key *key) key->type |= JUMP_TYPE_LINKED; } -static inline struct static_key *jump_entry_key(struct jump_entry *entry) -{ - return (struct static_key *)((unsigned long)entry->key & ~1UL); -} - -static bool jump_entry_branch(struct jump_entry *entry) -{ - return (unsigned long)entry->key & 1UL; -} - /*** * A 'struct static_key' uses a union such that it either points directly * to a table of 'struct jump_entry' or to a linked list of modules which in @@ -355,7 +345,7 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry) { struct static_key *key = jump_entry_key(entry); bool enabled = static_key_enabled(key); - bool branch = jump_entry_branch(entry); + bool branch = jump_entry_is_branch(entry); /* See the comment in linux/jump_label.h */ return enabled ^ branch; @@ -370,12 +360,12 @@ static void __jump_label_update(struct static_key *key, * An entry->code of 0 indicates an entry which has been * disabled because it was in an init text area. */ - if (entry->code) { - if (kernel_text_address(entry->code)) + if (!jump_entry_is_init(entry)) { + if (kernel_text_address(jump_entry_code(entry))) arch_jump_label_transform(entry, jump_label_type(entry)); else WARN_ONCE(1, "can't patch jump_label at %pS", - (void *)(unsigned long)entry->code); + (void *)jump_entry_code(entry)); } } } @@ -430,8 +420,8 @@ void __init jump_label_invalidate_initmem(void) struct jump_entry *iter; for (iter = iter_start; iter < iter_stop; iter++) { - if (init_section_contains((void *)(unsigned long)iter->code, 1)) - iter->code = 0; + if (init_section_contains((void *)jump_entry_code(iter), 1)) + jump_entry_set_init(iter); } } @@ -441,7 +431,7 @@ static enum jump_label_type jump_label_init_type(struct jump_entry *entry) { struct static_key *key = jump_entry_key(entry); bool type = static_key_type(key); - bool branch = jump_entry_branch(entry); + bool branch = jump_entry_is_branch(entry); /* See the comment in linux/jump_label.h */ return type ^ branch; @@ -565,7 +555,7 @@ static int jump_label_add_module(struct module *mod) continue; key = iterk; - if (within_module(iter->key, mod)) { + if (within_module((unsigned long)key, mod)) { static_key_set_entries(key, iter); continue; } @@ -615,7 +605,7 @@ static void jump_label_del_module(struct module *mod) key = jump_entry_key(iter); - if (within_module(iter->key, mod)) + if (within_module((unsigned long)key, mod)) continue; /* No memory during module load */ @@ -659,8 +649,8 @@ static void jump_label_invalidate_module_init(struct module *mod) struct jump_entry *iter; for (iter = iter_start; iter < iter_stop; iter++) { - if (within_module_init(iter->code, mod)) - iter->code = 0; + if (within_module_init(jump_entry_code(iter), mod)) + jump_entry_set_init(iter); } } -- 2.17.1
next prev parent reply other threads:[~2018-09-19 6:52 UTC|newest] Thread overview: 37+ messages / expand[flat|nested] mbox.gz Atom feed top 2018-09-19 6:51 [PATCH v3 0/9] add support for relative references in jump tables Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel [this message] 2018-09-19 6:51 ` [PATCH v3 1/9] kernel/jump_label: abstract jump_entry member accessors Ard Biesheuvel 2018-09-27 16:00 ` [tip:core/core] jump_label: Abstract " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 2/9] kernel/jump_label: implement generic support for relative references Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:01 ` [tip:core/core] jump_label: Implement " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 3/9] arm64/kernel: jump_label: switch to " Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:02 ` [tip:core/core] arm64/kernel: jump_label: Switch " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 4/9] x86: add support for 64-bit place relative relocations Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:02 ` [tip:core/core] x86: Add " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 5/9] x86: jump_label: switch to jump_entry accessors Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:03 ` [tip:core/core] x86/jump_label: Switch " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 6/9] x86/kernel: jump_table: use relative references Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:03 ` [tip:core/core] x86/jump_table: Use " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 7/9] jump_label: annotate entries that operate on __init code earlier Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:04 ` [tip:core/core] jump_label: Annotate " tip-bot for Ard Biesheuvel 2018-09-19 6:51 ` [PATCH v3 8/9] jump_table: move entries into ro_after_init region Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:04 ` [tip:core/core] jump_table: Move " tip-bot for Ard Biesheuvel 2018-09-30 15:42 ` [PATCH v3 8/9] jump_table: move " Guenter Roeck 2018-09-30 15:42 ` Guenter Roeck 2018-09-19 6:51 ` [PATCH v3 9/9] s390/jump_label: switch to relative references Ard Biesheuvel 2018-09-19 6:51 ` Ard Biesheuvel 2018-09-27 16:05 ` [tip:core/core] s390/jump_label: Switch " tip-bot for Heiko Carstens 2018-09-19 13:08 ` [PATCH v3 0/9] add support for relative references in jump tables Peter Zijlstra 2018-09-19 13:08 ` Peter Zijlstra 2018-09-27 15:22 ` Ard Biesheuvel 2018-09-27 15:22 ` Ard Biesheuvel 2018-09-27 15:46 ` Kees Cook 2018-09-27 15:46 ` Kees Cook
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20180919065144.25010-2-ard.biesheuvel@linaro.org \ --to=ard.biesheuvel@linaro.org \ --cc=arnd@arndb.de \ --cc=catalin.marinas@arm.com \ --cc=heiko.carstens@de.ibm.com \ --cc=jeyu@kernel.org \ --cc=keescook@chromium.org \ --cc=linux-arm-kernel@lists.infradead.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-s390@vger.kernel.org \ --cc=mingo@redhat.com \ --cc=peterz@infradead.org \ --cc=rostedt@goodmis.org \ --cc=schwidefsky@de.ibm.com \ --cc=tglx@linutronix.de \ --cc=will.deacon@arm.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.