From: James Morse <james.morse@arm.com> To: linux-arm-kernel@lists.infradead.org Cc: Jonathan.Zhang@cavium.com, Marc Zyngier <marc.zyngier@arm.com>, Catalin Marinas <catalin.marinas@arm.com>, Julien Thierry <julien.thierry@arm.com>, Will Deacon <will.deacon@arm.com>, wangxiongfeng2@huawei.com, Dongjiu Geng <gengdongjiu@huawei.com>, kvmarm@lists.cs.columbia.edu Subject: [PATCH v4 04/21] arm64: Mask all exceptions during kernel_exit Date: Thu, 19 Oct 2017 15:57:50 +0100 [thread overview] Message-ID: <20171019145807.23251-5-james.morse@arm.com> (raw) In-Reply-To: <20171019145807.23251-1-james.morse@arm.com> To take RAS Exceptions as quickly as possible we need to keep SError unmasked as much as possible. We need to mask it during kernel_exit as taking an error from this code will overwrite the exception-registers. Adding a naked 'disable_daif' to kernel_exit causes a performance problem for micro-benchmarks that do no real work, (e.g. calling getpid() in a loop). This is because the ret_to_user loop has already masked IRQs so that the TIF_WORK_MASK thread flags can't change underneath it, adding disable_daif is an additional self-synchronising operation. In the future, the RAS APEI code may need to modify the TIF_WORK_MASK flags from an SError, in which case the ret_to_user loop must mask SError while it examines the flags. Disable all exceptions for return to EL1. For return to EL0 get the ret_to_user loop to leave all exceptions masked once it has done its work, this avoids an extra pstate-write. Signed-off-by: James Morse <james.morse@arm.com> Reviewed-by: Julien Thierry <julien.thierry@arm.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> --- Changes since v3: * swapped verb/daif word-order. arch/arm64/kernel/entry.S | 10 +++++----- arch/arm64/kernel/signal.c | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e1c59d4008a8..f7d7bf9d76e7 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -221,6 +221,8 @@ alternative_else_nop_endif .macro kernel_exit, el .if \el != 0 + disable_daif + /* Restore the task's original addr_limit. */ ldr x20, [sp, #S_ORIG_ADDR_LIMIT] str x20, [tsk, #TSK_TI_ADDR_LIMIT] @@ -517,8 +519,6 @@ el1_da: mov x2, sp // struct pt_regs bl do_mem_abort - // disable interrupts before pulling preserved data off the stack - disable_irq kernel_exit 1 el1_sp_pc: /* @@ -793,7 +793,7 @@ ENDPROC(el0_irq) * and this includes saving x0 back into the kernel stack. */ ret_fast_syscall: - disable_irq // disable interrupts + disable_daif str x0, [sp, #S_X0] // returned x0 ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing and x2, x1, #_TIF_SYSCALL_WORK @@ -803,7 +803,7 @@ ret_fast_syscall: enable_step_tsk x1, x2 kernel_exit 0 ret_fast_syscall_trace: - enable_irq // enable interrupts + enable_daif b __sys_trace_return_skipped // we already saved x0 /* @@ -821,7 +821,7 @@ work_pending: * "slow" syscall return path. */ ret_to_user: - disable_irq // disable interrupts + disable_daif ldr x1, [tsk, #TSK_TI_FLAGS] and x2, x1, #_TIF_WORK_MASK cbnz x2, work_pending diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 0bdc96c61bc0..8e6500c9471b 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -31,6 +31,7 @@ #include <linux/ratelimit.h> #include <linux/syscalls.h> +#include <asm/daifflags.h> #include <asm/debug-monitors.h> #include <asm/elf.h> #include <asm/cacheflush.h> @@ -756,9 +757,12 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, addr_limit_user_check(); if (thread_flags & _TIF_NEED_RESCHED) { + /* Unmask Debug and SError for the next task */ + local_daif_restore(DAIF_PROCCTX_NOIRQ); + schedule(); } else { - local_irq_enable(); + local_daif_restore(DAIF_PROCCTX); if (thread_flags & _TIF_UPROBE) uprobe_notify_resume(regs); @@ -775,7 +779,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, fpsimd_restore_current_state(); } - local_irq_disable(); + local_daif_mask(); thread_flags = READ_ONCE(current_thread_info()->flags); } while (thread_flags & _TIF_WORK_MASK); } -- 2.13.3
WARNING: multiple messages have this Message-ID (diff)
From: james.morse@arm.com (James Morse) To: linux-arm-kernel@lists.infradead.org Subject: [PATCH v4 04/21] arm64: Mask all exceptions during kernel_exit Date: Thu, 19 Oct 2017 15:57:50 +0100 [thread overview] Message-ID: <20171019145807.23251-5-james.morse@arm.com> (raw) In-Reply-To: <20171019145807.23251-1-james.morse@arm.com> To take RAS Exceptions as quickly as possible we need to keep SError unmasked as much as possible. We need to mask it during kernel_exit as taking an error from this code will overwrite the exception-registers. Adding a naked 'disable_daif' to kernel_exit causes a performance problem for micro-benchmarks that do no real work, (e.g. calling getpid() in a loop). This is because the ret_to_user loop has already masked IRQs so that the TIF_WORK_MASK thread flags can't change underneath it, adding disable_daif is an additional self-synchronising operation. In the future, the RAS APEI code may need to modify the TIF_WORK_MASK flags from an SError, in which case the ret_to_user loop must mask SError while it examines the flags. Disable all exceptions for return to EL1. For return to EL0 get the ret_to_user loop to leave all exceptions masked once it has done its work, this avoids an extra pstate-write. Signed-off-by: James Morse <james.morse@arm.com> Reviewed-by: Julien Thierry <julien.thierry@arm.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> --- Changes since v3: * swapped verb/daif word-order. arch/arm64/kernel/entry.S | 10 +++++----- arch/arm64/kernel/signal.c | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e1c59d4008a8..f7d7bf9d76e7 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -221,6 +221,8 @@ alternative_else_nop_endif .macro kernel_exit, el .if \el != 0 + disable_daif + /* Restore the task's original addr_limit. */ ldr x20, [sp, #S_ORIG_ADDR_LIMIT] str x20, [tsk, #TSK_TI_ADDR_LIMIT] @@ -517,8 +519,6 @@ el1_da: mov x2, sp // struct pt_regs bl do_mem_abort - // disable interrupts before pulling preserved data off the stack - disable_irq kernel_exit 1 el1_sp_pc: /* @@ -793,7 +793,7 @@ ENDPROC(el0_irq) * and this includes saving x0 back into the kernel stack. */ ret_fast_syscall: - disable_irq // disable interrupts + disable_daif str x0, [sp, #S_X0] // returned x0 ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing and x2, x1, #_TIF_SYSCALL_WORK @@ -803,7 +803,7 @@ ret_fast_syscall: enable_step_tsk x1, x2 kernel_exit 0 ret_fast_syscall_trace: - enable_irq // enable interrupts + enable_daif b __sys_trace_return_skipped // we already saved x0 /* @@ -821,7 +821,7 @@ work_pending: * "slow" syscall return path. */ ret_to_user: - disable_irq // disable interrupts + disable_daif ldr x1, [tsk, #TSK_TI_FLAGS] and x2, x1, #_TIF_WORK_MASK cbnz x2, work_pending diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 0bdc96c61bc0..8e6500c9471b 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -31,6 +31,7 @@ #include <linux/ratelimit.h> #include <linux/syscalls.h> +#include <asm/daifflags.h> #include <asm/debug-monitors.h> #include <asm/elf.h> #include <asm/cacheflush.h> @@ -756,9 +757,12 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, addr_limit_user_check(); if (thread_flags & _TIF_NEED_RESCHED) { + /* Unmask Debug and SError for the next task */ + local_daif_restore(DAIF_PROCCTX_NOIRQ); + schedule(); } else { - local_irq_enable(); + local_daif_restore(DAIF_PROCCTX); if (thread_flags & _TIF_UPROBE) uprobe_notify_resume(regs); @@ -775,7 +779,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, fpsimd_restore_current_state(); } - local_irq_disable(); + local_daif_mask(); thread_flags = READ_ONCE(current_thread_info()->flags); } while (thread_flags & _TIF_WORK_MASK); } -- 2.13.3
next prev parent reply other threads:[~2017-10-19 14:59 UTC|newest] Thread overview: 160+ messages / expand[flat|nested] mbox.gz Atom feed top 2017-10-19 14:57 [PATCH v4 00/21] SError rework + RAS&IESB for firmware first support James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 01/21] arm64: explicitly mask all exceptions James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 02/21] arm64: introduce an order for exceptions James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 03/21] arm64: Move the async/fiq helpers to explicitly set process context flags James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` James Morse [this message] 2017-10-19 14:57 ` [PATCH v4 04/21] arm64: Mask all exceptions during kernel_exit James Morse 2017-10-19 14:57 ` [PATCH v4 05/21] arm64: entry.S: Remove disable_dbg James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 06/21] arm64: entry.S: convert el1_sync James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 07/21] arm64: entry.S convert el0_sync James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 08/21] arm64: entry.S: convert elX_irq James Morse 2017-10-19 14:57 ` James Morse 2017-10-19 14:57 ` [PATCH v4 09/21] KVM: arm/arm64: mask/unmask daif around VHE guests James Morse 2017-10-19 14:57 ` James Morse 2017-10-30 7:40 ` Christoffer Dall 2017-10-30 7:40 ` Christoffer Dall 2017-11-02 12:14 ` James Morse 2017-11-02 12:14 ` James Morse 2017-11-03 12:45 ` Christoffer Dall 2017-11-03 12:45 ` Christoffer Dall 2017-11-03 17:19 ` James Morse 2017-11-03 17:19 ` James Morse 2017-11-06 12:42 ` Christoffer Dall 2017-11-06 12:42 ` Christoffer Dall 2017-10-19 14:57 ` [PATCH v4 10/21] arm64: entry.S: move SError handling into a C function for future expansion James Morse 2017-10-19 14:57 ` James Morse 2018-01-02 21:07 ` Adam Wallis 2018-01-02 21:07 ` Adam Wallis 2018-01-03 16:00 ` James Morse 2018-01-03 16:00 ` James Morse 2017-10-19 14:57 ` [PATCH v4 11/21] arm64: cpufeature: Detect CPU RAS Extentions James Morse 2017-10-19 14:57 ` James Morse 2017-10-31 13:14 ` Will Deacon 2017-10-31 13:14 ` Will Deacon 2017-11-02 12:15 ` James Morse 2017-11-02 12:15 ` James Morse 2017-10-19 14:57 ` [PATCH v4 12/21] arm64: kernel: Survive corrected RAS errors notified by SError James Morse 2017-10-19 14:57 ` James Morse 2017-10-31 13:50 ` Will Deacon 2017-10-31 13:50 ` Will Deacon 2017-11-02 12:15 ` James Morse 2017-11-02 12:15 ` James Morse 2017-10-19 14:57 ` [PATCH v4 13/21] arm64: cpufeature: Enable IESB on exception entry/return for firmware-first James Morse 2017-10-19 14:57 ` James Morse 2017-10-31 13:56 ` Will Deacon 2017-10-31 13:56 ` Will Deacon 2017-10-19 14:58 ` [PATCH v4 14/21] arm64: kernel: Prepare for a DISR user James Morse 2017-10-19 14:58 ` James Morse 2017-10-19 14:58 ` [PATCH v4 15/21] KVM: arm64: Set an impdef ESR for Virtual-SError using VSESR_EL2 James Morse 2017-10-19 14:58 ` James Morse 2017-10-20 16:44 ` gengdongjiu 2017-10-20 16:44 ` gengdongjiu 2017-10-23 15:26 ` James Morse 2017-10-23 15:26 ` James Morse 2017-10-24 9:53 ` gengdongjiu 2017-10-24 9:53 ` gengdongjiu 2017-10-30 7:59 ` Christoffer Dall 2017-10-30 7:59 ` Christoffer Dall 2017-10-30 10:51 ` Christoffer Dall 2017-10-30 10:51 ` Christoffer Dall 2017-10-30 15:44 ` James Morse 2017-10-30 15:44 ` James Morse 2017-10-31 5:48 ` Christoffer Dall 2017-10-31 5:48 ` Christoffer Dall 2017-10-31 6:34 ` Marc Zyngier 2017-10-31 6:34 ` Marc Zyngier 2017-10-19 14:58 ` [PATCH v4 16/21] KVM: arm64: Save/Restore guest DISR_EL1 James Morse 2017-10-19 14:58 ` James Morse 2017-10-31 4:27 ` Marc Zyngier 2017-10-31 4:27 ` Marc Zyngier 2017-10-31 5:27 ` Christoffer Dall 2017-10-31 5:27 ` Christoffer Dall 2017-10-19 14:58 ` [PATCH v4 17/21] KVM: arm64: Save ESR_EL2 on guest SError James Morse 2017-10-19 14:58 ` James Morse 2017-10-31 4:26 ` Marc Zyngier 2017-10-31 4:26 ` Marc Zyngier 2017-10-31 5:47 ` Marc Zyngier 2017-10-31 5:47 ` Marc Zyngier 2017-11-01 17:42 ` James Morse 2017-11-01 17:42 ` James Morse 2017-10-19 14:58 ` [PATCH v4 18/21] KVM: arm64: Handle RAS SErrors from EL1 on guest exit James Morse 2017-10-19 14:58 ` James Morse 2017-10-31 5:55 ` Marc Zyngier 2017-10-31 5:55 ` Marc Zyngier 2017-10-31 5:56 ` Christoffer Dall 2017-10-31 5:56 ` Christoffer Dall 2017-10-19 14:58 ` [PATCH v4 19/21] KVM: arm64: Handle RAS SErrors from EL2 " James Morse 2017-10-19 14:58 ` James Morse 2017-10-27 6:26 ` gengdongjiu 2017-10-27 6:26 ` gengdongjiu 2017-10-27 17:38 ` James Morse 2017-10-27 17:38 ` James Morse 2017-10-31 6:13 ` Marc Zyngier 2017-10-31 6:13 ` Marc Zyngier 2017-10-31 6:13 ` Christoffer Dall 2017-10-31 6:13 ` Christoffer Dall 2017-10-19 14:58 ` [PATCH v4 20/21] KVM: arm64: Take any host SError before entering the guest James Morse 2017-10-19 14:58 ` James Morse 2017-10-31 6:23 ` Christoffer Dall 2017-10-31 6:23 ` Christoffer Dall 2017-10-31 11:43 ` James Morse 2017-10-31 11:43 ` James Morse 2017-11-01 4:55 ` Christoffer Dall 2017-11-01 4:55 ` Christoffer Dall 2017-11-02 12:18 ` James Morse 2017-11-02 12:18 ` James Morse 2017-11-03 12:49 ` Christoffer Dall 2017-11-03 12:49 ` Christoffer Dall 2017-11-03 16:14 ` James Morse 2017-11-03 16:14 ` James Morse 2017-11-06 12:45 ` Christoffer Dall 2017-11-06 12:45 ` Christoffer Dall 2017-10-19 14:58 ` [PATCH v4 21/21] KVM: arm64: Trap RAS error registers and set HCR_EL2's TERR & TEA James Morse 2017-10-19 14:58 ` James Morse 2017-10-31 6:32 ` Christoffer Dall 2017-10-31 6:32 ` Christoffer Dall 2017-10-31 6:32 ` Marc Zyngier 2017-10-31 6:32 ` Marc Zyngier 2017-10-31 6:35 ` [PATCH v4 00/21] SError rework + RAS&IESB for firmware first support Christoffer Dall 2017-10-31 6:35 ` Christoffer Dall 2017-10-31 10:08 ` Will Deacon 2017-10-31 10:08 ` Will Deacon 2017-11-01 15:23 ` James Morse 2017-11-01 15:23 ` James Morse 2017-11-02 8:14 ` Christoffer Dall 2017-11-02 8:14 ` Christoffer Dall 2017-11-09 18:14 ` James Morse 2017-11-09 18:14 ` James Morse 2017-11-10 12:03 ` gengdongjiu 2017-11-10 12:03 ` gengdongjiu 2017-11-13 11:29 ` Christoffer Dall 2017-11-13 11:29 ` Christoffer Dall 2017-11-13 13:05 ` Peter Maydell 2017-11-13 13:05 ` Peter Maydell 2017-11-20 8:53 ` Christoffer Dall 2017-11-20 8:53 ` Christoffer Dall 2017-11-13 16:14 ` Andrew Jones 2017-11-13 16:14 ` Andrew Jones 2017-11-13 17:56 ` Peter Maydell 2017-11-13 17:56 ` Peter Maydell 2017-11-14 16:11 ` James Morse 2017-11-14 16:11 ` James Morse 2017-11-15 9:59 ` gengdongjiu 2017-11-15 9:59 ` gengdongjiu 2017-11-14 16:03 ` James Morse 2017-11-14 16:03 ` James Morse 2017-11-15 9:15 ` gengdongjiu 2017-11-15 9:15 ` gengdongjiu 2017-11-15 18:25 ` James Morse 2017-11-15 18:25 ` James Morse 2017-11-21 11:31 ` gengdongjiu 2017-11-21 11:31 ` gengdongjiu 2017-11-20 8:55 ` Christoffer Dall 2017-11-20 8:55 ` Christoffer Dall
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20171019145807.23251-5-james.morse@arm.com \ --to=james.morse@arm.com \ --cc=Jonathan.Zhang@cavium.com \ --cc=catalin.marinas@arm.com \ --cc=gengdongjiu@huawei.com \ --cc=julien.thierry@arm.com \ --cc=kvmarm@lists.cs.columbia.edu \ --cc=linux-arm-kernel@lists.infradead.org \ --cc=marc.zyngier@arm.com \ --cc=wangxiongfeng2@huawei.com \ --cc=will.deacon@arm.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.