From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, Josh Poimboeuf <jpoimboe@redhat.com>,
Helge Deller <deller@gmx.de>,
"David S. Miller" <davem@davemloft.net>,
Michael Ellerman <mpe@ellerman.id.au>,
Rich Felker <dalias@libc.org>, Heiko Carstens <hca@linux.ibm.com>,
Kees Cook <keescook@chromium.org>,
Lai Jiangshan <jiangshanlai+lkml@gmail.com>
Subject: [patch V2 09/13] x86/softirq: Remove indirection in do_softirq_own_stack()
Date: Wed, 10 Feb 2021 00:40:50 +0100 [thread overview]
Message-ID: <20210210002512.972714001@linutronix.de> (raw)
In-Reply-To: 20210209234041.127454039@linutronix.de
From: Thomas Gleixner <tglx@linutronix.de>
Use the new inline stack switching and remove the old ASM indirect call
implementation.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
---
arch/x86/entry/entry_64.S | 39 -----------------------------
arch/x86/include/asm/irq_stack.h | 52 ++++++++++++---------------------------
arch/x86/kernel/irq_64.c | 2 -
3 files changed, 17 insertions(+), 76 deletions(-)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -756,45 +756,6 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
SYM_CODE_END(.Lbad_gs)
.previous
-/*
- * rdi: New stack pointer points to the top word of the stack
- * rsi: Function pointer
- * rdx: Function argument (can be NULL if none)
- */
-SYM_FUNC_START(asm_call_on_stack)
- /*
- * Save the frame pointer unconditionally. This allows the ORC
- * unwinder to handle the stack switch.
- */
- pushq %rbp
- mov %rsp, %rbp
-
- /*
- * The unwinder relies on the word at the top of the new stack
- * page linking back to the previous RSP.
- */
- mov %rsp, (%rdi)
- mov %rdi, %rsp
- /* Move the argument to the right place */
- mov %rdx, %rdi
-
-1:
- .pushsection .discard.instr_begin
- .long 1b - .
- .popsection
-
- CALL_NOSPEC rsi
-
-2:
- .pushsection .discard.instr_end
- .long 2b - .
- .popsection
-
- /* Restore the previous stack pointer from RBP. */
- leaveq
- ret
-SYM_FUNC_END(asm_call_on_stack)
-
#ifdef CONFIG_XEN_PV
/*
* A note on the "critical region" in our callback handler.
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -185,20 +185,23 @@
IRQ_CONSTRAINTS, regs, vector); \
}
-static __always_inline bool irqstack_active(void)
-{
- return __this_cpu_read(hardirq_stack_inuse);
-}
-
-void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
+#define ASM_CALL_SOFTIRQ \
+ "call %P[__func] \n"
-static __always_inline void __run_on_irqstack(void (*func)(void))
-{
- void *tos = __this_cpu_read(hardirq_stack_ptr);
-
- __this_cpu_write(hardirq_stack_inuse, true);
- asm_call_on_stack(tos, func, NULL);
- __this_cpu_write(hardirq_stack_inuse, false);
+/*
+ * Macro to invoke __do_softirq on the irq stack. Contrary to the above
+ * the only check which is necessary is whether the interrupt stack is
+ * in use already.
+ */
+#define run_softirq_on_irqstack_cond() \
+{ \
+ if (__this_cpu_read(hardirq_stack_inuse)) { \
+ __do_softirq(); \
+ } else { \
+ __this_cpu_write(hardirq_stack_inuse, true); \
+ call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
+ __this_cpu_write(hardirq_stack_inuse, false); \
+ } \
}
#else /* CONFIG_X86_64 */
@@ -219,29 +222,6 @@ static __always_inline void __run_on_irq
irq_exit_rcu(); \
}
-static inline bool irqstack_active(void) { return false; }
-static inline void __run_on_irqstack(void (*func)(void)) { }
#endif /* !CONFIG_X86_64 */
-static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
-{
- if (IS_ENABLED(CONFIG_X86_32))
- return false;
- if (!regs)
- return !irqstack_active();
- return !user_mode(regs) && !irqstack_active();
-}
-
-
-static __always_inline void run_on_irqstack_cond(void (*func)(void),
- struct pt_regs *regs)
-{
- lockdep_assert_irqs_disabled();
-
- if (irq_needs_irq_stack(regs))
- __run_on_irqstack(func);
- else
- func();
-}
-
#endif
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned in
void do_softirq_own_stack(void)
{
- run_on_irqstack_cond(__do_softirq, NULL);
+ run_softirq_on_irqstack_cond();
}
next prev parent reply other threads:[~2021-02-10 2:08 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-09 23:40 [patch V2 00/13] x86/irq/64: Inline irq stack switching Thomas Gleixner
2021-02-09 23:40 ` [patch V2 01/13] x86/entry: Fix instrumentation annotation Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 02/13] x86/irq: Sanitize irq stack tracking Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 03/13] x86/irq/64: Adjust the per CPU irq stack pointer by 8 Thomas Gleixner
2021-02-10 11:41 ` David Laight
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 04/13] x86/apic: Split out spurious handling code Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 05/13] x86/irq: Provide macro for inlining irq stack switching Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 06/13] x86/entry: Convert system vectors to irq stack macro Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 07/13] x86/entry: Convert device interrupts to inline stack switching Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 08/13] x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` Thomas Gleixner [this message]
2021-02-11 0:50 ` [tip: x86/entry] x86/softirq: Remove indirection in do_softirq_own_stack() tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 10/13] x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 11/13] softirq: Move __ARCH_HAS_DO_SOFTIRQ to Kconfig Thomas Gleixner
2021-02-10 7:03 ` Kees Cook
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 12/13] softirq: Move do_softirq_own_stack() to generic asm header Thomas Gleixner
2021-02-10 7:04 ` Kees Cook
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-09 23:40 ` [patch V2 13/13] x86/softirq/64: Inline do_softirq_own_stack() Thomas Gleixner
2021-02-11 0:50 ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2021-02-15 16:39 ` [patch V2 13/13] " Guenter Roeck
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210210002512.972714001@linutronix.de \
--to=tglx@linutronix.de \
--cc=dalias@libc.org \
--cc=davem@davemloft.net \
--cc=deller@gmx.de \
--cc=hca@linux.ibm.com \
--cc=jiangshanlai+lkml@gmail.com \
--cc=jpoimboe@redhat.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mpe@ellerman.id.au \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).