From: "tip-bot2 for Thomas Gleixner" <tip-bot2@linutronix.de>
To: linux-tip-commits@vger.kernel.org
Cc: Thomas Gleixner <tglx@linutronix.de>,
Kees Cook <keescook@chromium.org>,
x86@kernel.org, linux-kernel@vger.kernel.org
Subject: [tip: x86/entry] x86/entry: Convert system vectors to irq stack macro
Date: Thu, 11 Feb 2021 00:50:25 -0000 [thread overview]
Message-ID: <161300462559.23325.15667527831927013522.tip-bot2@tip-bot2> (raw)
In-Reply-To: <20210210002512.676197354@linutronix.de>
The following commit has been merged into the x86/entry branch of tip:
Commit-ID: 569dd8b4eb7ef666b467c41b8e8e4f2820d07f67
Gitweb: https://git.kernel.org/tip/569dd8b4eb7ef666b467c41b8e8e4f2820d07f67
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 10 Feb 2021 00:40:47 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Wed, 10 Feb 2021 23:34:15 +01:00
x86/entry: Convert system vectors to irq stack macro
To inline the stack switching and to prepare for enabling
CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK provide a macro template for system
vectors and device interrupts and convert the system vectors over to it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20210210002512.676197354@linutronix.de
---
arch/x86/entry/entry_64.S | 1 +-
arch/x86/include/asm/idtentry.h | 2 +-
arch/x86/include/asm/irq_stack.h | 93 ++++++++++++++++++++++---------
3 files changed, 66 insertions(+), 30 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index cad0870..68643d3 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -762,7 +762,6 @@ SYM_CODE_END(.Lbad_gs)
* rdx: Function argument (can be NULL if none)
*/
SYM_FUNC_START(asm_call_on_stack)
-SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL)
SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL)
/*
* Save the frame pointer unconditionally. This allows the ORC
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 247a60a..712b3c8 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -237,10 +237,8 @@ __visible noinstr void func(struct pt_regs *regs) \
irqentry_state_t state = irqentry_enter(regs); \
\
instrumentation_begin(); \
- irq_enter_rcu(); \
kvm_set_cpu_l1tf_flush_l1d(); \
run_sysvec_on_irqstack_cond(__##func, regs); \
- irq_exit_rcu(); \
instrumentation_end(); \
irqentry_exit(regs, state); \
} \
diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
index d5b7bc6..05c37e7 100644
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -105,14 +105,69 @@
#define assert_arg_type(arg, proto) \
static_assert(__builtin_types_compatible_p(typeof(arg), proto))
+/*
+ * Macro to invoke system vector and device interrupt C handlers.
+ */
+#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \
+{ \
+ /* \
+ * User mode entry and interrupt on the irq stack do not \
+ * switch stacks. If from user mode the task stack is empty. \
+ */ \
+ if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \
+ irq_enter_rcu(); \
+ func(c_args); \
+ irq_exit_rcu(); \
+ } else { \
+ /* \
+ * Mark the irq stack inuse _before_ and unmark _after_ \
+ * switching stacks. Interrupts are disabled in both \
+ * places. Invoke the stack switch macro with the call \
+ * sequence which matches the above direct invocation. \
+ */ \
+ __this_cpu_write(hardirq_stack_inuse, true); \
+ call_on_irqstack(func, asm_call, constr); \
+ __this_cpu_write(hardirq_stack_inuse, false); \
+ } \
+}
+
+/*
+ * Function call sequence for __call_on_irqstack() for system vectors.
+ *
+ * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
+ * mechanism because these functions are global and cannot be optimized out
+ * when compiling a particular source file which uses one of these macros.
+ *
+ * The argument (regs) does not need to be pushed or stashed in a callee
+ * saved register to be safe vs. the irq_enter_rcu() call because the
+ * clobbers already prevent the compiler from storing it in a callee
+ * clobbered register. As the compiler has to preserve @regs for the final
+ * call to idtentry_exit() anyway, it's likely that it does not cause extra
+ * effort for this asm magic.
+ */
+#define ASM_CALL_SYSVEC \
+ "call irq_enter_rcu \n" \
+ "movq %[arg1], %%rdi \n" \
+ "call %P[__func] \n" \
+ "call irq_exit_rcu \n"
+
+#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs)
+
+#define run_sysvec_on_irqstack_cond(func, regs) \
+{ \
+ assert_function_type(func, void (*)(struct pt_regs *)); \
+ assert_arg_type(regs, struct pt_regs *); \
+ \
+ call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \
+ SYSVEC_CONSTRAINTS, regs); \
+}
+
static __always_inline bool irqstack_active(void)
{
return __this_cpu_read(hardirq_stack_inuse);
}
void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
-void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
- struct pt_regs *regs);
void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
struct irq_desc *desc);
@@ -126,17 +181,6 @@ static __always_inline void __run_on_irqstack(void (*func)(void))
}
static __always_inline void
-__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
- struct pt_regs *regs)
-{
- void *tos = __this_cpu_read(hardirq_stack_ptr);
-
- __this_cpu_write(hardirq_stack_inuse, true);
- asm_call_sysvec_on_stack(tos, func, regs);
- __this_cpu_write(hardirq_stack_inuse, false);
-}
-
-static __always_inline void
__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
struct irq_desc *desc)
{
@@ -148,10 +192,17 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
}
#else /* CONFIG_X86_64 */
+
+/* System vector handlers always run on the stack they interrupted. */
+#define run_sysvec_on_irqstack_cond(func, regs) \
+{ \
+ irq_enter_rcu(); \
+ func(regs); \
+ irq_exit_rcu(); \
+}
+
static inline bool irqstack_active(void) { return false; }
static inline void __run_on_irqstack(void (*func)(void)) { }
-static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
- struct pt_regs *regs) { }
static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
struct irq_desc *desc) { }
#endif /* !CONFIG_X86_64 */
@@ -178,18 +229,6 @@ static __always_inline void run_on_irqstack_cond(void (*func)(void),
}
static __always_inline void
-run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
- struct pt_regs *regs)
-{
- lockdep_assert_irqs_disabled();
-
- if (irq_needs_irq_stack(regs))
- __run_sysvec_on_irqstack(func, regs);
- else
- func(regs);
-}
-
-static __always_inline void
run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
struct pt_regs *regs)
{
next prev parent reply other threads:[~2021-02-11 0:52 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-09 23:40 [patch V2 00/13] x86/irq/64: Inline irq stack switching Thomas Gleixner
2021-02-09 23:40 ` [patch V2 01/13] x86/entry: Fix instrumentation annotation Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 02/13] x86/irq: Sanitize irq stack tracking Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 03/13] x86/irq/64: Adjust the per CPU irq stack pointer by 8 Thomas Gleixner
2021-02-10 11:41 ` David Laight
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 04/13] x86/apic: Split out spurious handling code Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 05/13] x86/irq: Provide macro for inlining irq stack switching Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 06/13] x86/entry: Convert system vectors to irq stack macro Thomas Gleixner
2021-02-11 0:50 ` tip-bot2 for Thomas Gleixner [this message]
2021-02-09 23:40 ` [patch V2 07/13] x86/entry: Convert device interrupts to inline stack switching Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 08/13] x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 09/13] x86/softirq: Remove indirection in do_softirq_own_stack() Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 10/13] x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 11/13] softirq: Move __ARCH_HAS_DO_SOFTIRQ to Kconfig Thomas Gleixner
2021-02-10 7:03 ` Kees Cook
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 12/13] softirq: Move do_softirq_own_stack() to generic asm header Thomas Gleixner
2021-02-10 7:04 ` Kees Cook
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 13/13] x86/softirq/64: Inline do_softirq_own_stack() Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-15 16:39 ` [patch V2 13/13] " Guenter Roeck
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=161300462559.23325.15667527831927013522.tip-bot2@tip-bot2 \
--to=tip-bot2@linutronix.de \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-tip-commits@vger.kernel.org \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).