LKML Archive on lore.kernel.org
 help / color / Atom feed
From: Thomas Gleixner <tglx@linutronix.de>
To: LKML <linux-kernel@vger.kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>,
	Andrew Cooper <andrew.cooper3@citrix.com>,
	X86 ML <x86@kernel.org>, "Paul E. McKenney" <paulmck@kernel.org>,
	Alexandre Chartre <alexandre.chartre@oracle.com>,
	Frederic Weisbecker <frederic@kernel.org>,
	Paolo Bonzini <pbonzini@redhat.com>,
	Sean Christopherson <sean.j.christopherson@intel.com>,
	Masami Hiramatsu <mhiramat@kernel.org>,
	Petr Mladek <pmladek@suse.com>,
	Steven Rostedt <rostedt@goodmis.org>,
	Joel Fernandes <joel@joelfernandes.org>,
	Boris Ostrovsky <boris.ostrovsky@oracle.com>,
	Juergen Gross <jgross@suse.com>, Brian Gerst <brgerst@gmail.com>,
	Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
	Josh Poimboeuf <jpoimboe@redhat.com>,
	Will Deacon <will@kernel.org>,
	Tom Lendacky <thomas.lendacky@amd.com>,
	Wei Liu <wei.liu@kernel.org>,
	Michael Kelley <mikelley@microsoft.com>,
	Jason Chen CJ <jason.cj.chen@intel.com>,
	Zhao Yakui <yakui.zhao@intel.com>,
	"Peter Zijlstra (Intel)" <peterz@infradead.org>
Subject: [patch V9 10/39] x86/entry: Provide helpers for execute on irqstack
Date: Thu, 21 May 2020 22:05:23 +0200
Message-ID: <20200521202117.763775313@linutronix.de> (raw)
In-Reply-To: <20200521200513.656533920@linutronix.de>

From: Thomas Gleixner <tglx@linutronix.de>

Device interrupt handlers and system vector handlers are executed on the
interrupt stack. The stack switch happens in the low level assembly entry
code. This conflicts with the efforts to consolidate the exit code in C to
ensure correctness vs. RCU and tracing.

As there is no way to move #DB away from IST due to the MOV SS issue, the
requirements vs. #DB and NMI for switching to the interrupt stack do not
exist anymore. The only requirement is that interrupts are disabled.

That allows to move the stack switching to C code which simplifies the
entry/exit handling further because it allows to switch stacks after
handling the entry and on exit before handling RCU, return to usermode and
kernel preemption in the same way as for regular exceptions.

The initial attempt of having the stack switching in inline ASM caused too
much headache vs. objtool and the unwinder. After analysing the use cases
it was agreed on that having the stack switch in ASM for the price of an
indirect call is acceptable as the main users are indirect call heavy
anyway and the few system vectors which are empty shells (scheduler IPI and
KVM posted interrupt vectors) can run from the regular stack.

Provide helper functions to check whether the interrupt stack is already
active and whether stack switching is required.

64 bit only for now. 32 bit has a variant of that already. Once this is
cleaned up the two implementations might be consolidated as a cleanup on
top.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
V9: Moved the conditions into an inline to avoid code duplication
---
 arch/x86/entry/entry_64.S        |   39 ++++++++++++++++++++++++++++
 arch/x86/include/asm/irq_stack.h |   53 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)

--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1106,6 +1106,45 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
 SYM_CODE_END(.Lbad_gs)
 	.previous
 
+/*
+ * rdi: New stack pointer points to the top word of the stack
+ * rsi: Function pointer
+ * rdx: Function argument (can be NULL if none)
+ */
+SYM_FUNC_START(asm_call_on_stack)
+	/*
+	 * Save the frame pointer unconditionally. This allows the ORC
+	 * unwinder to handle the stack switch.
+	 */
+	pushq		%rbp
+	mov		%rsp, %rbp
+
+	/*
+	 * The unwinder relies on the word at the top of the new stack
+	 * page linking back to the previous RSP.
+	 */
+	mov		%rsp, (%rdi)
+	mov		%rdi, %rsp
+	/* Move the argument to the right place */
+	mov		%rdx, %rdi
+
+1:
+	.pushsection .discard.instr_begin
+	.long 1b - .
+	.popsection
+
+	CALL_NOSPEC	rsi
+
+2:
+	.pushsection .discard.instr_end
+	.long 2b - .
+	.popsection
+
+	/* Restore the previous stack pointer from RBP. */
+	leaveq
+	ret
+SYM_FUNC_END(asm_call_on_stack)
+
 /* Call softirq on interrupt stack. Interrupts are off. */
 .pushsection .text, "ax"
 SYM_FUNC_START(do_softirq_own_stack)
--- /dev/null
+++ b/arch/x86/include/asm/irq_stack.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_IRQ_STACK_H
+#define _ASM_X86_IRQ_STACK_H
+
+#include <linux/ptrace.h>
+
+#include <asm/processor.h>
+
+#ifdef CONFIG_X86_64
+static __always_inline bool irqstack_active(void)
+{
+	return __this_cpu_read(irq_count) != -1;
+}
+
+void asm_call_on_stack(void *sp, void *func, void *arg);
+
+static __always_inline void __run_on_irqstack(void *func, void *arg)
+{
+	void *tos = __this_cpu_read(hardirq_stack_ptr);
+
+	__this_cpu_add(irq_count, 1);
+	asm_call_on_stack(tos - 8, func, arg);
+	__this_cpu_sub(irq_count, 1);
+}
+
+#else /* CONFIG_X86_64 */
+static inline bool irqstack_active(void) { return false; }
+static inline void __run_on_irqstack(void *func, void *arg) { }
+#endif /* !CONFIG_X86_64 */
+
+static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
+{
+	if (IS_ENABLED(CONFIG_X86_32))
+		return false;
+	if (!regs)
+		return !irqstack_active();
+	return !user_mode(regs) && !irqstack_active();
+}
+
+static __always_inline void run_on_irqstack_cond(void *func, void *arg,
+						 struct pt_regs *regs)
+{
+	void (*__func)(void *arg) = func;
+
+	lockdep_assert_irqs_disabled();
+
+	if (irq_needs_irq_stack(regs))
+		__run_on_irqstack(__func, arg);
+	else
+		__func(arg);
+}
+
+#endif


  parent reply index

Thread overview: 132+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-05-21 20:05 [patch V9 00/39] x86/entry: Rework leftovers (was part V) Thomas Gleixner
2020-05-21 20:05 ` [patch V9 01/39] nmi, tracing: Make hardware latency tracing noinstr safe Thomas Gleixner
2020-05-27  8:12   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 02/39] rcu: Abstract out rcu_irq_enter_check_tick() from rcu_nmi_enter() Thomas Gleixner
2020-05-21 21:03   ` Paul E. McKenney
2020-05-21 21:25     ` Thomas Gleixner
2020-05-26  8:14     ` Ingo Molnar
2020-05-26 15:34       ` Paul E. McKenney
2020-05-27  8:12   ` [tip: x86/entry] " tip-bot2 for Paul E. McKenney
2020-05-21 20:05 ` [patch V9 03/39] rcu: Provide rcu_irq_exit_check_preempt() Thomas Gleixner
2020-05-27  8:12   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 04/39] x86/entry: Provide idtentry_entry/exit_cond_rcu() Thomas Gleixner
2020-05-21 21:06   ` Paul E. McKenney
2020-05-26  8:23   ` Ingo Molnar
2020-05-26  8:58     ` Thomas Gleixner
2020-05-21 20:05 ` [patch V9 05/39] x86/entry: Provide idtentry_enter/exit_user() Thomas Gleixner
2020-05-27  8:12   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 06/39] x86/idtentry: Switch to conditional RCU handling Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 07/39] x86/entry: Cleanup idtentry_enter/exit() leftovers Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] x86/entry: Clean up " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 08/39] genirq: Provide irq_enter/exit_rcu() Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 09/39] genirq: Provide __irq_enter/exit_raw() Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` Thomas Gleixner [this message]
2020-05-27  8:11   ` [tip: x86/entry] x86/entry: Provide helpers for executing on the irqstack tip-bot2 for Thomas Gleixner
2020-06-05 17:18   ` [patch V9 10/39] x86/entry: Provide helpers for execute on irqstack Qian Cai
2020-06-05 17:36     ` Peter Zijlstra
2020-06-05 17:52       ` Qian Cai
2020-06-07 11:59         ` Thomas Gleixner
2020-06-07 18:27           ` Qian Cai
2020-06-08 16:01       ` Qian Cai
2020-06-08 22:20         ` Thomas Gleixner
2020-06-09  2:32           ` Qian Cai
2020-06-09 20:33             ` Thomas Gleixner
2020-06-09 20:50               ` Thomas Gleixner
2020-06-10 12:38                 ` Qian Cai
2020-06-10 19:38                   ` Thomas Gleixner
2020-06-13 13:55                     ` Qian Cai
2020-06-13 14:03                       ` Thomas Gleixner
2020-06-13 21:41                         ` Qian Cai
2020-06-14  8:59                           ` Thomas Gleixner
2020-05-21 20:05 ` [patch V9 11/39] x86/entry/64: Move do_softirq_own_stack() to C Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 12/39] x86/entry: Split out idtentry_exit_cond_resched() Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 13/39] x86/entry: Switch XEN/PV hypercall entry to IDTENTRY Thomas Gleixner
2020-05-22 18:32   ` [patch V9-1 " Thomas Gleixner
2020-05-26  7:44     ` Jürgen Groß
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 14/39] x86/entry/64: Simplify idtentry_body Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 15/39] x86/entry: Switch page fault exception to IDTENTRY_RAW Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 16/39] x86/entry: Remove the transition leftovers Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 17/39] x86/entry: Change exit path of xen_failsafe_callback Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 18/39] x86/entry/64: Remove error_exit Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] x86/entry/64: Remove error_exit() tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 19/39] x86/entry/32: Remove common_exception Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] x86/entry/32: Remove common_exception() tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 20/39] x86/irq: Use generic irq_regs implementation Thomas Gleixner
2020-05-26 18:39   ` damian
2020-05-28  9:50     ` Thomas Gleixner
2020-05-28 20:20       ` damian
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 21/39] x86/irq: Convey vector as argument and not in ptregs Thomas Gleixner
2020-05-22 19:34   ` Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-08-24 17:29   ` [patch V9 21/39] " Alexander Graf
2020-08-25 10:28     ` Thomas Gleixner
2020-08-25 23:17       ` Alexander Graf
2020-08-25 23:41         ` Andy Lutomirski
2020-08-26  0:04           ` Alexander Graf
2020-08-26  1:03             ` Brian Gerst
2020-08-26  0:55           ` Thomas Gleixner
2020-05-21 20:05 ` [patch V9 22/39] x86/irq: Rework handle_irq() for 64bit Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] x86/irq: Rework handle_irq() for 64-bit tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 23/39] x86/entry: Add IRQENTRY_IRQ macro Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 24/39] x86/entry: Use idtentry for interrupts Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 25/39] x86/entry: Provide IDTENTRY_SYSVEC Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 26/39] x86/entry: Convert APIC interrupts to IDTENTRY_SYSVEC Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 27/39] x86/entry: Convert SMP system vectors " Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 28/39] x86/entry: Convert various system vectors Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 29/39] x86/entry: Convert KVM vectors to IDTENTRY_SYSVEC* Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 30/39] x86/entry: Convert various hypervisor vectors to IDTENTRY_SYSVEC Thomas Gleixner
2020-05-26  9:29   ` Wei Liu
2020-05-27  1:46   ` Boqun Feng
2020-05-27  8:38     ` Wei Liu
2020-05-27 12:09       ` Wei Liu
2020-05-27 23:06         ` Boqun Feng
2020-05-27 12:30       ` Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 31/39] x86/entry: Convert XEN hypercall vector " Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 32/39] x86/entry: Convert reschedule interrupt to IDTENTRY_SYSVEC_SIMPLE Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 33/39] x86/entry: Remove the apic/BUILD interrupt leftovers Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 34/39] x86/entry/64: Remove IRQ stack switching ASM Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 35/39] x86/entry: Make enter_from_user_mode() static Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 36/39] x86/entry/32: Remove redundant irq disable code Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 37/39] x86/entry/64: Remove TRACE_IRQS_*_DEBUG Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 38/39] x86/entry: Move paranoid irq tracing out of ASM code Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-21 20:05 ` [patch V9 39/39] x86/entry: Remove the TRACE_IRQS cruft Thomas Gleixner
2020-05-27  8:11   ` [tip: x86/entry] " tip-bot2 for Thomas Gleixner
2020-05-22  7:20 ` [patch V9 00/39] x86/entry: Rework leftovers (was part V) Andrew Cooper
2020-05-22 21:17   ` Peter Zijlstra
2020-06-03 19:18     ` Andrew Cooper
2020-06-04 13:25       ` Peter Zijlstra
2020-06-04 13:29         ` Paolo Bonzini
2020-06-04 13:35           ` Peter Zijlstra
2020-06-04 15:42             ` Andy Lutomirski
2020-06-04 15:55               ` Peter Zijlstra
2020-05-22 14:26 ` Boris Ostrovsky
2020-05-22 17:47   ` Thomas Gleixner
2020-05-22 18:08     ` Thomas Gleixner
2020-05-26  4:33 ` Andy Lutomirski

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200521202117.763775313@linutronix.de \
    --to=tglx@linutronix.de \
    --cc=alexandre.chartre@oracle.com \
    --cc=andrew.cooper3@citrix.com \
    --cc=boris.ostrovsky@oracle.com \
    --cc=brgerst@gmail.com \
    --cc=frederic@kernel.org \
    --cc=jason.cj.chen@intel.com \
    --cc=jgross@suse.com \
    --cc=joel@joelfernandes.org \
    --cc=jpoimboe@redhat.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=luto@kernel.org \
    --cc=mathieu.desnoyers@efficios.com \
    --cc=mhiramat@kernel.org \
    --cc=mikelley@microsoft.com \
    --cc=paulmck@kernel.org \
    --cc=pbonzini@redhat.com \
    --cc=peterz@infradead.org \
    --cc=pmladek@suse.com \
    --cc=rostedt@goodmis.org \
    --cc=sean.j.christopherson@intel.com \
    --cc=thomas.lendacky@amd.com \
    --cc=wei.liu@kernel.org \
    --cc=will@kernel.org \
    --cc=x86@kernel.org \
    --cc=yakui.zhao@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

LKML Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/lkml/0 lkml/git/0.git
	git clone --mirror https://lore.kernel.org/lkml/1 lkml/git/1.git
	git clone --mirror https://lore.kernel.org/lkml/2 lkml/git/2.git
	git clone --mirror https://lore.kernel.org/lkml/3 lkml/git/3.git
	git clone --mirror https://lore.kernel.org/lkml/4 lkml/git/4.git
	git clone --mirror https://lore.kernel.org/lkml/5 lkml/git/5.git
	git clone --mirror https://lore.kernel.org/lkml/6 lkml/git/6.git
	git clone --mirror https://lore.kernel.org/lkml/7 lkml/git/7.git
	git clone --mirror https://lore.kernel.org/lkml/8 lkml/git/8.git
	git clone --mirror https://lore.kernel.org/lkml/9 lkml/git/9.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 lkml lkml/ https://lore.kernel.org/lkml \
		linux-kernel@vger.kernel.org
	public-inbox-index lkml

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-kernel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git