linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: x86@kernel.org, Josh Poimboeuf <jpoimboe@redhat.com>,
	Kees Cook <keescook@chromium.org>
Subject: [patch 06/12] x86/entry: Convert system vectors to irq stack macro
Date: Thu, 04 Feb 2021 21:49:09 +0100	[thread overview]
Message-ID: <20210204211154.713523041@linutronix.de> (raw)
In-Reply-To: 20210204204903.350275743@linutronix.de

To inline the stack switching and to prepare for enabling
CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK provide a macro template for system
vectors and device interrupts and convert the system vectors over to it.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/entry/entry_64.S        |    1 
 arch/x86/include/asm/idtentry.h  |    2 
 arch/x86/include/asm/irq_stack.h |   93 +++++++++++++++++++++++++++------------
 3 files changed, 66 insertions(+), 30 deletions(-)

--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -762,7 +762,6 @@ SYM_CODE_END(.Lbad_gs)
  * rdx: Function argument (can be NULL if none)
  */
 SYM_FUNC_START(asm_call_on_stack)
-SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL)
 SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL)
 	/*
 	 * Save the frame pointer unconditionally. This allows the ORC
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -237,10 +237,8 @@ static void __##func(struct pt_regs *reg
 	irqentry_state_t state = irqentry_enter(regs);			\
 									\
 	instrumentation_begin();					\
-	irq_enter_rcu();						\
 	kvm_set_cpu_l1tf_flush_l1d();					\
 	run_sysvec_on_irqstack_cond(__##func, regs);			\
-	irq_exit_rcu();							\
 	instrumentation_end();						\
 	irqentry_exit(regs, state);					\
 }									\
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -111,14 +111,69 @@
 #define assert_arg_type(arg, proto)					\
 	static_assert(__builtin_types_compatible_p(typeof(arg), proto))
 
+/*
+ * Macro to invoke system vector and device interrupt C handlers.
+ */
+#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...)	\
+{									\
+	/*								\
+	 * User mode entry and interrupt on the irq stack do not	\
+	 * switch stacks. If from user mode the task stack is empty.	\
+	 */								\
+	if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {	\
+		irq_enter_rcu();					\
+		func(c_args);						\
+		irq_exit_rcu();						\
+	} else {							\
+		/*							\
+		 * Mark the irq stack inuse _before_ and unmark _after_	\
+		 * switching stacks. Interrupts are disabled in both	\
+		 * places. Invoke the stack switch macro with the call	\
+		 * sequence which matches the above direct invocation.	\
+		 */							\
+		__this_cpu_write(hardirq_stack_inuse, true);		\
+		__call_on_irqstack(func, asm_call, constr);		\
+		__this_cpu_write(hardirq_stack_inuse, false);		\
+	}								\
+}
+
+/*
+ * Function call sequence for __call_on_irqstack() for system vectors.
+ *
+ * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
+ * mechanism because these functions are global and cannot be optimized out
+ * when compiling a particular source file which uses one of these macros.
+ *
+ * The argument (regs) does not need to be pushed or stashed in a callee
+ * saved register to be safe vs. the irq_enter_rcu() call because the
+ * clobbers already prevent the compiler from storing it in a callee
+ * clobbered register. As the compiler has to preserve @regs for the final
+ * call to idtentry_exit() anyway, it's likely that it does not cause extra
+ * effort for this asm magic.
+ */
+#define ASM_CALL_SYSVEC							\
+	"call irq_enter_rcu				\n"		\
+	"movq	%[__arg1], %%rdi			\n"		\
+	"call %P[__func]				\n"		\
+	"call irq_exit_rcu				\n"
+
+#define SYSVEC_CONSTRAINTS	, [__arg1] "r" (regs)
+
+#define run_sysvec_on_irqstack_cond(func, regs)				\
+{									\
+	assert_function_type(func, void (*)(struct pt_regs *));		\
+	assert_arg_type(regs, struct pt_regs *);			\
+									\
+	call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC,		\
+			      SYSVEC_CONSTRAINTS, regs);		\
+}
+
 static __always_inline bool irqstack_active(void)
 {
 	return __this_cpu_read(hardirq_stack_inuse);
 }
 
 void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
-void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
-			      struct pt_regs *regs);
 void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
 			   struct irq_desc *desc);
 
@@ -132,17 +187,6 @@ static __always_inline void __run_on_irq
 }
 
 static __always_inline void
-__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
-			 struct pt_regs *regs)
-{
-	void *tos = __this_cpu_read(hardirq_stack_ptr);
-
-	__this_cpu_write(hardirq_stack_inuse, true);
-	asm_call_sysvec_on_stack(tos, func, regs);
-	__this_cpu_write(hardirq_stack_inuse, false);
-}
-
-static __always_inline void
 __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
 		      struct irq_desc *desc)
 {
@@ -154,10 +198,17 @@ static __always_inline void
 }
 
 #else /* CONFIG_X86_64 */
+
+/* System vector handlers always run on the stack they interrupted. */
+#define run_sysvec_on_irqstack_cond(func, regs)				\
+{									\
+	irq_enter_rcu();						\
+	func(regs);							\
+	irq_exit_rcu();							\
+}
+
 static inline bool irqstack_active(void) { return false; }
 static inline void __run_on_irqstack(void (*func)(void)) { }
-static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
-					    struct pt_regs *regs) { }
 static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
 					 struct irq_desc *desc) { }
 #endif /* !CONFIG_X86_64 */
@@ -184,18 +235,6 @@ static __always_inline void run_on_irqst
 }
 
 static __always_inline void
-run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
-			    struct pt_regs *regs)
-{
-	lockdep_assert_irqs_disabled();
-
-	if (irq_needs_irq_stack(regs))
-		__run_sysvec_on_irqstack(func, regs);
-	else
-		func(regs);
-}
-
-static __always_inline void
 run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
 			 struct pt_regs *regs)
 {


  parent reply	other threads:[~2021-02-05  0:59 UTC|newest]

Thread overview: 32+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-04 20:49 [patch 00/12] x86/irq/64: Inline irq stack switching Thomas Gleixner
2021-02-04 20:49 ` [patch 01/12] x86/entry: Fix instrumentation annotation Thomas Gleixner
2021-02-05 12:02   ` Borislav Petkov
2021-02-04 20:49 ` [patch 02/12] x86/irq: Sanitize irq stack tracking Thomas Gleixner
2021-02-04 20:49 ` [patch 03/12] x86/irq/64: Adjust the per CPU irq stack pointer by 8 Thomas Gleixner
2021-02-05 13:53   ` Peter Zijlstra
2021-02-05 15:00     ` Thomas Gleixner
2021-02-04 20:49 ` [patch 04/12] x86/apic: Split out spurious handling code Thomas Gleixner
2021-02-04 20:49 ` [patch 05/12] x86/irq: Provide macro for inlining irq stack switching Thomas Gleixner
2021-02-08 15:06   ` Borislav Petkov
2021-02-08 20:42   ` Josh Poimboeuf
2021-02-09 15:12     ` Thomas Gleixner
2021-02-09 16:22       ` Josh Poimboeuf
2021-02-09 18:14         ` Thomas Gleixner
2021-02-04 20:49 ` Thomas Gleixner [this message]
2021-02-05  3:52   ` [patch 06/12] x86/entry: Convert system vectors to irq stack macro kernel test robot
2021-02-05 14:13     ` Peter Zijlstra
2021-02-07  8:15       ` [kbuild-all] " Rong Chen
2021-02-08 14:19         ` Borislav Petkov
2021-02-09  8:57           ` Rong Chen
2021-02-09 10:46             ` Borislav Petkov
2021-02-04 20:49 ` [patch 07/12] x86/entry: Convert device interrupts to inline stack switching Thomas Gleixner
2021-02-04 20:49 ` [patch 08/12] x86/entry: Use run_sysvec_on_irqstack_cond() for XEN upcall Thomas Gleixner
2021-02-05  7:28   ` kernel test robot
2021-02-04 20:49 ` [patch 09/12] x86/softirq: Remove indirection in do_softirq_own_stack() Thomas Gleixner
2021-02-04 20:49 ` [patch 10/12] x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK Thomas Gleixner
2021-02-04 20:49 ` [patch 11/12] softirq: Allow inlining do_softirq_own_stack() Thomas Gleixner
2021-02-05 10:14   ` Lai Jiangshan
2021-02-05 11:38     ` Thomas Gleixner
2021-02-09 20:43       ` Thomas Gleixner
2021-02-04 20:49 ` [patch 12/12] x86/softirq/64: Inline do_softirq_own_stack() Thomas Gleixner
2021-02-08 21:47 ` [patch 00/12] x86/irq/64: Inline irq stack switching Kees Cook

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210204211154.713523041@linutronix.de \
    --to=tglx@linutronix.de \
    --cc=jpoimboe@redhat.com \
    --cc=keescook@chromium.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).