From: Alexandre Chartre <alexandre.chartre@oracle.com>
To: "tglx@linutronix.de"@userv0122.oracle.com,
"mingo@redhat.com"@userv0122.oracle.com,
"bp@alien8.de"@userv0122.oracle.com,
"hpa@zytor.com"@userv0122.oracle.com,
"x86@kernel.org"@userv0122.oracle.com,
"dave.hansen@linux.intel.com"@userv0122.oracle.com,
"luto@kernel.org"@userv0122.oracle.com,
"peterz@infradead.org"@userv0122.oracle.com,
"linux-kernel@vger.kernel.org"@userv0122.oracle.com,
"thomas.lendacky@amd.com"@userv0122.oracle.com,
"jroedel@suse.de"@userv0122.oracle.com
Cc: "konrad.wilk@oracle.com"@userv0122.oracle.com,
"jan.setjeeilers@oracle.com"@userv0122.oracle.com,
"junaids@google.com"@userv0122.oracle.com,
"oweisse@google.com"@userv0122.oracle.com,
"rppt@linux.vnet.ibm.com"@userv0122.oracle.com,
"graf@amazon.de"@userv0122.oracle.com,
"mgross@linux.intel.com"@userv0122.oracle.com,
"kuzuno@gmail.com"@userv0122.oracle.com,
"alexandre.chartre@oracle.com"@userv0122.oracle.com
Subject: [RFC][PATCH 03/24] x86/entry: Consolidate IST entry from userspace
Date: Mon, 9 Nov 2020 12:22:58 +0100 [thread overview]
Message-ID: <20201109112319.264511-4-alexandre.chartre@oracle.com> (raw)
In-Reply-To: <20201109112319.264511-1-alexandre.chartre@oracle.com>
Most IST entries (NMI, MCE, DEBUG, VC but not DF) handle an entry from
userspace the same way: they switch from the IST stack to the kernel
stack, call the handler and then return to userspace. However, NMI,
MCE/DEBUG and VC implement this same behavior using different code paths,
so consolidate this code into a single assembly macro.
Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
---
arch/x86/entry/entry_64.S | 137 +++++++++++++++++++++-----------------
1 file changed, 75 insertions(+), 62 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index c42948aca0a8..51df9f1871c6 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -316,6 +316,72 @@ SYM_CODE_END(ret_from_fork)
#endif
.endm
+/*
+ * Macro to handle an IDT entry defined with the IST mechanism. It should
+ * be invoked at the beginning of the IDT handler with a pointer to the C
+ * function (cfunc_user) to invoke if the IDT was entered from userspace.
+ *
+ * If the IDT was entered from userspace, the macro will switch from the
+ * IST stack to the regular task stack, call the provided function and
+ * return to userland.
+ *
+ * If IDT was entered from the kernel, the macro will just return.
+ */
+.macro ist_entry_user cfunc_user has_error_code=0
+ UNWIND_HINT_IRET_REGS
+ ASM_CLAC
+
+ /* only process entry from userspace */
+ .if \has_error_code == 1
+ testb $3, CS-ORIG_RAX(%rsp)
+ jz .List_entry_from_kernel_\@
+ .else
+ testb $3, CS-RIP(%rsp)
+ jz .List_entry_from_kernel_\@
+ pushq $-1 /* ORIG_RAX: no syscall to restart */
+ .endif
+
+ /* Use %rdx as a temp variable */
+ pushq %rdx
+
+ /*
+ * Switch from the IST stack to the regular task stack and
+ * use the provided entry point.
+ */
+ swapgs
+ cld
+ FENCE_SWAPGS_USER_ENTRY
+ SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
+ movq %rsp, %rdx
+ movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+ UNWIND_HINT_IRET_REGS base=%rdx offset=8
+ pushq 6*8(%rdx) /* pt_regs->ss */
+ pushq 5*8(%rdx) /* pt_regs->rsp */
+ pushq 4*8(%rdx) /* pt_regs->flags */
+ pushq 3*8(%rdx) /* pt_regs->cs */
+ pushq 2*8(%rdx) /* pt_regs->rip */
+ UNWIND_HINT_IRET_REGS
+ pushq 1*8(%rdx) /* pt_regs->orig_ax */
+ PUSH_AND_CLEAR_REGS rdx=(%rdx)
+ ENCODE_FRAME_POINTER
+
+ /*
+ * At this point we no longer need to worry about stack damage
+ * due to nesting -- we're on the normal thread stack and we're
+ * done with the IST stack.
+ */
+
+ mov %rsp, %rdi
+ .if \has_error_code == 1
+ movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
+ movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
+ .endif
+ call \cfunc_user
+ jmp swapgs_restore_regs_and_return_to_usermode
+
+.List_entry_from_kernel_\@:
+.endm
+
/**
* idtentry_body - Macro to emit code calling the C function
* @cfunc: C function to be called
@@ -417,18 +483,15 @@ SYM_CODE_END(\asmsym)
*/
.macro idtentry_mce_db vector asmsym cfunc
SYM_CODE_START(\asmsym)
- UNWIND_HINT_IRET_REGS
- ASM_CLAC
-
- pushq $-1 /* ORIG_RAX: no syscall to restart */
-
/*
* If the entry is from userspace, switch stacks and treat it as
* a normal entry.
*/
- testb $3, CS-ORIG_RAX(%rsp)
- jnz .Lfrom_usermode_switch_stack_\@
+ ist_entry_user noist_\cfunc
+ /* Entry from kernel */
+
+ pushq $-1 /* ORIG_RAX: no syscall to restart */
/* paranoid_entry returns GS information for paranoid_exit in EBX. */
call paranoid_entry
@@ -440,10 +503,6 @@ SYM_CODE_START(\asmsym)
jmp paranoid_exit
- /* Switch to the regular task stack and use the noist entry point */
-.Lfrom_usermode_switch_stack_\@:
- idtentry_body noist_\cfunc, has_error_code=0
-
_ASM_NOKPROBE(\asmsym)
SYM_CODE_END(\asmsym)
.endm
@@ -472,15 +531,11 @@ SYM_CODE_END(\asmsym)
*/
.macro idtentry_vc vector asmsym cfunc
SYM_CODE_START(\asmsym)
- UNWIND_HINT_IRET_REGS
- ASM_CLAC
-
/*
* If the entry is from userspace, switch stacks and treat it as
* a normal entry.
*/
- testb $3, CS-ORIG_RAX(%rsp)
- jnz .Lfrom_usermode_switch_stack_\@
+ ist_entry_user safe_stack_\cfunc, has_error_code=1
/*
* paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
@@ -517,10 +572,6 @@ SYM_CODE_START(\asmsym)
*/
jmp paranoid_exit
- /* Switch to the regular task stack */
-.Lfrom_usermode_switch_stack_\@:
- idtentry_body safe_stack_\cfunc, has_error_code=1
-
_ASM_NOKPROBE(\asmsym)
SYM_CODE_END(\asmsym)
.endm
@@ -1113,8 +1164,6 @@ SYM_CODE_END(error_return)
* when PAGE_TABLE_ISOLATION is in use. Do not clobber.
*/
SYM_CODE_START(asm_exc_nmi)
- UNWIND_HINT_IRET_REGS
-
/*
* We allow breakpoints in NMIs. If a breakpoint occurs, then
* the iretq it performs will take us out of NMI context.
@@ -1153,14 +1202,6 @@ SYM_CODE_START(asm_exc_nmi)
* other IST entries.
*/
- ASM_CLAC
-
- /* Use %rdx as our temp variable throughout */
- pushq %rdx
-
- testb $3, CS-RIP+8(%rsp)
- jz .Lnmi_from_kernel
-
/*
* NMI from user mode. We need to run on the thread stack, but we
* can't go through the normal entry paths: NMIs are masked, and
@@ -1171,41 +1212,13 @@ SYM_CODE_START(asm_exc_nmi)
* We also must not push anything to the stack before switching
* stacks lest we corrupt the "NMI executing" variable.
*/
+ ist_entry_user exc_nmi
- swapgs
- cld
- FENCE_SWAPGS_USER_ENTRY
- SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
- movq %rsp, %rdx
- movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
- UNWIND_HINT_IRET_REGS base=%rdx offset=8
- pushq 5*8(%rdx) /* pt_regs->ss */
- pushq 4*8(%rdx) /* pt_regs->rsp */
- pushq 3*8(%rdx) /* pt_regs->flags */
- pushq 2*8(%rdx) /* pt_regs->cs */
- pushq 1*8(%rdx) /* pt_regs->rip */
- UNWIND_HINT_IRET_REGS
- pushq $-1 /* pt_regs->orig_ax */
- PUSH_AND_CLEAR_REGS rdx=(%rdx)
- ENCODE_FRAME_POINTER
-
- /*
- * At this point we no longer need to worry about stack damage
- * due to nesting -- we're on the normal thread stack and we're
- * done with the NMI stack.
- */
-
- movq %rsp, %rdi
- movq $-1, %rsi
- call exc_nmi
+ /* NMI from kernel */
- /*
- * Return back to user mode. We must *not* do the normal exit
- * work, because we don't want to enable interrupts.
- */
- jmp swapgs_restore_regs_and_return_to_usermode
+ /* Use %rdx as our temp variable throughout */
+ pushq %rdx
-.Lnmi_from_kernel:
/*
* Here's what our stack frame will look like:
* +---------------------------------------------------------+
--
2.18.4
next prev parent reply other threads:[~2020-11-09 11:24 UTC|newest]
Thread overview: 33+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-09 11:22 [RFC][PATCH 00/24] x86/pti: Defer CR3 switch to C code Alexandre Chartre
2020-11-09 11:22 ` [RFC][PATCH 01/24] x86/syscall: Add wrapper for invoking syscall function Alexandre Chartre
2020-11-09 17:25 ` Andy Lutomirski
2020-11-09 17:45 ` Alexandre Chartre
2020-11-09 11:22 ` [RFC][PATCH 02/24] x86/entry: Update asm_call_on_stack to support more function arguments Alexandre Chartre
2020-11-09 11:22 ` Alexandre Chartre [this message]
2020-11-09 11:22 ` [RFC][PATCH 04/24] x86/sev-es: Define a setup stack function for the VC idtentry Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 05/24] x86/entry: Implement ret_from_fork body with C code Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 06/24] x86/pti: Provide C variants of PTI switch CR3 macros Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 07/24] x86/entry: Fill ESPFIX stack using C code Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 08/24] x86/entry: Add C version of SWAPGS and SWAPGS_UNSAFE_STACK Alexandre Chartre
2020-11-09 17:38 ` Andy Lutomirski
2020-11-09 18:04 ` Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 09/24] x86/entry: Add C version of paranoid_entry/exit Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 10/24] x86/pti: Introduce per-task PTI trampoline stack Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 11/24] x86/pti: Function to clone page-table entries from a specified mm Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 12/24] x86/pti: Function to map per-cpu page-table entry Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 13/24] x86/pti: Extend PTI user mappings Alexandre Chartre
2020-11-09 17:28 ` Andy Lutomirski
2020-11-09 17:52 ` Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 14/24] x86/pti: Use PTI stack instead of trampoline stack Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 15/24] x86/pti: Execute syscall functions on the kernel stack Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 16/24] x86/pti: Execute IDT handlers " Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 17/24] x86/pti: Execute IDT handlers with error code " Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 18/24] x86/pti: Execute system vector handlers " Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 19/24] x86/pti: Execute page fault handler " Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 20/24] x86/pti: Execute NMI " Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 21/24] x86/entry: Disable stack-protector for IST entry C handlers Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 22/24] x86/entry: Defer paranoid entry/exit to C code Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 23/24] x86/entry: Remove paranoid_entry and paranoid_exit Alexandre Chartre
2020-11-09 11:23 ` [RFC][PATCH 24/24] x86/pti: Defer CR3 switch to C code for non-IST and syscall entries Alexandre Chartre
2020-11-09 14:00 ` [RFC][PATCH 00/24] x86/pti: Defer CR3 switch to C code Alexandre Chartre
2020-11-09 14:44 Alexandre Chartre
2020-11-09 14:44 ` [RFC][PATCH 03/24] x86/entry: Consolidate IST entry from userspace Alexandre Chartre
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201109112319.264511-4-alexandre.chartre@oracle.com \
--to=alexandre.chartre@oracle.com \
--cc="alexandre.chartre@oracle.com"@userv0122.oracle.com \
--cc="bp@alien8.de"@userv0122.oracle.com \
--cc="dave.hansen@linux.intel.com"@userv0122.oracle.com \
--cc="graf@amazon.de"@userv0122.oracle.com \
--cc="hpa@zytor.com"@userv0122.oracle.com \
--cc="jan.setjeeilers@oracle.com"@userv0122.oracle.com \
--cc="jroedel@suse.de"@userv0122.oracle.com \
--cc="junaids@google.com"@userv0122.oracle.com \
--cc="konrad.wilk@oracle.com"@userv0122.oracle.com \
--cc="kuzuno@gmail.com"@userv0122.oracle.com \
--cc="linux-kernel@vger.kernel.org"@userv0122.oracle.com \
--cc="luto@kernel.org"@userv0122.oracle.com \
--cc="mgross@linux.intel.com"@userv0122.oracle.com \
--cc="mingo@redhat.com"@userv0122.oracle.com \
--cc="oweisse@google.com"@userv0122.oracle.com \
--cc="peterz@infradead.org"@userv0122.oracle.com \
--cc="rppt@linux.vnet.ibm.com"@userv0122.oracle.com \
--cc="tglx@linutronix.de"@userv0122.oracle.com \
--cc="thomas.lendacky@amd.com"@userv0122.oracle.com \
--cc="x86@kernel.org"@userv0122.oracle.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).