From: Rohan McLure <rmclure@linux.ibm.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Rohan McLure <rmclure@linux.ibm.com>
Subject: [PATCH v6 24/25] powerpc/64s: Clear gprs on interrupt routine entry in Book3S
Date: Wed, 21 Sep 2022 16:56:04 +1000 [thread overview]
Message-ID: <20220921065605.1051927-25-rmclure@linux.ibm.com> (raw)
In-Reply-To: <20220921065605.1051927-1-rmclure@linux.ibm.com>
Zero GPRS r0, r2-r11, r14-r31, on entry into the kernel for all
other interrupt sources to limit influence of user-space values
in potential speculation gadgets. The remaining gprs are overwritten by
entry macros to interrupt handlers, irrespective of whether or not a
given handler consumes these register values.
Prior to this commit, r14-r31 are restored on a per-interrupt basis at
exit, but now they are always restored. Remove explicit REST_NVGPRS
invocations as non-volatiles must now always be restored. 32-bit systems
do not clear user registers on interrupt, and continue to depend on the
return value of interrupt_exit_user_prepare to determine whether or not
to restore non-volatiles.
The mmap_bench benchmark in selftests should rapidly invoke pagefaults.
See ~0.8% performance regression with this mitigation, but this
indicates the worst-case performance due to heavier-weight interrupt
handlers. This mitigation is disabled by default, but enabled with
CONFIG_INTERRUPT_SANITIZE_REGISTERS.
Signed-off-by: Rohan McLure <rmclure@linux.ibm.com>
---
V2: Add benchmark data
V3: Use ZEROIZE_GPR{,S} macro renames, clarify
upt_exit_user_prepare changes in summary.
V5: Configurable now with INTERRUPT_SANITIZE_REGISTERS. Zero r12
(containing MSR) from common macro on per-interrupt basis with IOPTION.
V6: Replace ifdefs with invocations of conditionally defined macros.
---
arch/powerpc/kernel/exceptions-64s.S | 47 +++++++++++++++++++++-----
arch/powerpc/kernel/interrupt_64.S | 15 ++++++++
2 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a3b51441b039..b3f5ef1c712f 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -21,6 +21,19 @@
#include <asm/feature-fixups.h>
#include <asm/kup.h>
+/*
+ * macros for handling user register sanitisation
+ */
+#ifdef CONFIG_INTERRUPT_SANITIZE_REGISTERS
+#define SANITIZE_ZEROIZE_NVGPRS() ZEROIZE_NVGPRS()
+#define SANITIZE_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#define HANDLER_RESTORE_NVGPRS()
+#else
+#define SANITIZE_ZEROIZE_NVGPRS()
+#define SANITIZE_RESTORE_NVGPRS()
+#define HANDLER_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#endif /* CONFIG_INTERRUPT_SANITIZE_REGISTERS */
+
/*
* Following are fixed section helper macros.
*
@@ -111,6 +124,7 @@ name:
#define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */
#define __ISTACK(name) .L_ISTACK_ ## name
#define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */
+#define IMSR_R12 .L_IMSR_R12_\name\() /* Assumes MSR saved to r12 */
#define INT_DEFINE_BEGIN(n) \
.macro int_define_ ## n name
@@ -176,6 +190,9 @@ do_define_int n
.ifndef IKUAP
IKUAP=1
.endif
+ .ifndef IMSR_R12
+ IMSR_R12=0
+ .endif
.endm
/*
@@ -502,6 +519,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real, text)
std r10,0(r1) /* make stack chain pointer */
std r0,GPR0(r1) /* save r0 in stackframe */
std r10,GPR1(r1) /* save r1 in stackframe */
+ ZEROIZE_GPR(0)
/* Mark our [H]SRRs valid for return */
li r10,1
@@ -544,8 +562,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
std r9,GPR11(r1)
std r10,GPR12(r1)
std r11,GPR13(r1)
+ .if !IMSR_R12
+ ZEROIZE_GPRS(9, 12)
+ .else
+ ZEROIZE_GPRS(9, 11)
+ .endif
SAVE_NVGPRS(r1)
+ SANITIZE_ZEROIZE_NVGPRS()
.if IDAR
.if IISIDE
@@ -577,8 +601,8 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r10,IAREA+EX_CTR(r13)
std r10,_CTR(r1)
- std r2,GPR2(r1) /* save r2 in stackframe */
- SAVE_GPRS(3, 8, r1) /* save r3 - r8 in stackframe */
+ SAVE_GPRS(2, 8, r1) /* save r2 - r8 in stackframe */
+ ZEROIZE_GPRS(2, 8)
mflr r9 /* Get LR, later save to stack */
ld r2,PACATOC(r13) /* get kernel TOC into r2 */
std r9,_LINK(r1)
@@ -696,6 +720,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
mtlr r9
ld r9,_CCR(r1)
mtcr r9
+ SANITIZE_RESTORE_NVGPRS()
REST_GPRS(2, 13, r1)
REST_GPR(0, r1)
/* restore original r1. */
@@ -1372,7 +1397,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
* do_break() may have changed the NV GPRS while handling a breakpoint.
* If so, we need to restore them with their updated values.
*/
- REST_NVGPRS(r1)
+ HANDLER_RESTORE_NVGPRS()
b interrupt_return_srr
@@ -1598,7 +1623,7 @@ EXC_COMMON_BEGIN(alignment_common)
GEN_COMMON alignment
addi r3,r1,STACK_FRAME_OVERHEAD
bl alignment_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -1708,7 +1733,7 @@ EXC_COMMON_BEGIN(program_check_common)
.Ldo_program_check:
addi r3,r1,STACK_FRAME_OVERHEAD
bl program_check_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -1726,6 +1751,7 @@ INT_DEFINE_BEGIN(fp_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(fp_unavailable)
EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
@@ -2139,7 +2165,7 @@ EXC_COMMON_BEGIN(emulation_assist_common)
GEN_COMMON emulation_assist
addi r3,r1,STACK_FRAME_OVERHEAD
bl emulation_assist_interrupt
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_hsrr
@@ -2347,6 +2373,7 @@ INT_DEFINE_BEGIN(altivec_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(altivec_unavailable)
EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20)
@@ -2396,6 +2423,7 @@ INT_DEFINE_BEGIN(vsx_unavailable)
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
IKVM_REAL=1
#endif
+ IMSR_R12=1
INT_DEFINE_END(vsx_unavailable)
EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
@@ -2457,7 +2485,7 @@ EXC_COMMON_BEGIN(facility_unavailable_common)
GEN_COMMON facility_unavailable
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
b interrupt_return_srr
@@ -2485,7 +2513,8 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common)
GEN_COMMON h_facility_unavailable
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
- REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
+ /* XXX Shouldn't be necessary in practice */
+ HANDLER_RESTORE_NVGPRS()
b interrupt_return_hsrr
@@ -2711,7 +2740,7 @@ EXC_COMMON_BEGIN(altivec_assist_common)
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_ALTIVEC
bl altivec_assist_exception
- REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+ HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
#else
bl unknown_exception
#endif
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index 40147558e1a6..6b60d39f113b 100644
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -13,6 +13,15 @@
#include <asm/ppc_asm.h>
#include <asm/ptrace.h>
+/*
+ * macros for handling user register sanitisation
+ */
+#ifdef CONFIG_INTERRUPT_SANITIZE_REGISTERS
+#define SANITIZE_RESTORE_NVGPRS() REST_NVGPRS(r1)
+#else
+#define SANITIZE_RESTORE_NVGPRS()
+#endif /* CONFIG_INTERRUPT_SANITIZE_REGISTERS */
+
.section ".toc","aw"
SYS_CALL_TABLE:
.tc sys_call_table[TC],sys_call_table
@@ -433,9 +442,11 @@ interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_user_prepare
+#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
cmpdi r3,0
bne- .Lrestore_nvgprs_\srr
.Lrestore_nvgprs_\srr\()_cont:
+#endif
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Linterrupt_return_\srr\()_user_rst_start:
@@ -449,6 +460,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
.Lfast_user_interrupt_return_\srr\():
+ SANITIZE_RESTORE_NVGPRS()
#ifdef CONFIG_PPC_BOOK3S
.ifc \srr,srr
lbz r4,PACASRR_VALID(r13)
@@ -518,9 +530,11 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
b . /* prevent speculative execution */
.Linterrupt_return_\srr\()_user_rst_end:
+#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
.Lrestore_nvgprs_\srr\():
REST_NVGPRS(r1)
b .Lrestore_nvgprs_\srr\()_cont
+#endif
#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_user_restart:
@@ -562,6 +576,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
1:
.Lfast_kernel_interrupt_return_\srr\():
+ SANITIZE_RESTORE_NVGPRS()
cmpdi cr1,r3,0
#ifdef CONFIG_PPC_BOOK3S
.ifc \srr,srr
--
2.34.1
next prev parent reply other threads:[~2022-09-21 7:13 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-21 6:55 [PATCH v6 00/25] powerpc: Syscall wrapper and register clearing Rohan McLure
2022-09-21 6:55 ` [PATCH v6 01/25] powerpc: Remove asmlinkage from syscall handler definitions Rohan McLure
2022-09-21 6:55 ` [PATCH v6 02/25] powerpc: Save caller r3 prior to system_call_exception Rohan McLure
2022-09-21 6:55 ` [PATCH v6 03/25] powerpc: Add ZEROIZE_GPRS macros for register clears Rohan McLure
2022-09-21 6:55 ` [PATCH v6 04/25] powerpc/64s: Use {ZEROIZE,SAVE,REST}_GPRS macros in sc, scv 0 handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 05/25] powerpc/32: Clarify interrupt restores with REST_GPR macro in entry_32.S Rohan McLure
2022-09-21 6:55 ` [PATCH v6 06/25] powerpc/64e: Clarify register saves and clears with {SAVE,ZEROIZE}_GPRS Rohan McLure
2022-09-21 6:55 ` [PATCH v6 07/25] powerpc/64s: Fix comment on interrupt handler prologue Rohan McLure
2022-09-21 6:55 ` [PATCH v6 08/25] powerpc: Fix fallocate and fadvise64_64 compat parameter combination Rohan McLure
2022-09-21 6:55 ` [PATCH v6 09/25] asm-generic: compat: Support BE for long long args in 32-bit ABIs Rohan McLure
2022-10-31 13:23 ` [PATCH] asm-generic: compat: fix compat_arg_u64 and compat_arg_u64_dual Andreas Schwab
2022-11-01 12:25 ` Michael Ellerman
2022-11-03 8:20 ` Christophe Leroy
2022-11-03 8:48 ` Arnd Bergmann
2022-09-21 6:55 ` [PATCH v6 10/25] powerpc: Use generic fallocate compatibility syscall Rohan McLure
2022-09-21 6:55 ` [PATCH v6 11/25] powerpc/32: Remove powerpc select specialisation Rohan McLure
2022-09-21 6:55 ` [PATCH v6 12/25] powerpc: Remove direct call to personality syscall handler Rohan McLure
2022-09-21 6:55 ` [PATCH v6 13/25] powerpc: Remove direct call to mmap2 syscall handlers Rohan McLure
2022-09-28 12:15 ` Michael Ellerman
2022-09-28 13:00 ` Arnd Bergmann
2022-09-30 13:19 ` Michael Ellerman
2022-09-30 14:09 ` Arnd Bergmann
2022-09-21 6:55 ` [PATCH v6 14/25] powerpc: Provide do_ppc64_personality helper Rohan McLure
2022-09-21 6:55 ` [PATCH v6 15/25] powerpc: Adopt SYSCALL_DEFINE for arch-specific syscall handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 16/25] powerpc: Include all arch-specific syscall prototypes Rohan McLure
2022-09-21 6:55 ` [PATCH v6 17/25] powerpc: Enable compile-time check for syscall handlers Rohan McLure
2022-09-21 6:55 ` [PATCH v6 18/25] powerpc: Use common syscall handler type Rohan McLure
2022-09-21 6:55 ` [PATCH v6 19/25] powerpc: Remove high-order word clearing on compat syscall entry Rohan McLure
2022-09-23 7:40 ` Nicholas Piggin
2022-09-28 11:56 ` Michael Ellerman
2022-09-21 6:56 ` [PATCH v6 20/25] powerpc: Change system_call_exception calling convention Rohan McLure
2022-09-23 7:43 ` Nicholas Piggin
2022-09-21 6:56 ` [PATCH v6 21/25] powerpc: Provide syscall wrapper Rohan McLure
2022-09-23 7:50 ` Nicholas Piggin
2022-10-30 15:34 ` Andreas Schwab
2022-10-30 15:50 ` Andreas Schwab
2022-10-30 19:43 ` Arnd Bergmann
2022-10-30 20:05 ` Andreas Schwab
2022-10-31 3:09 ` Michael Ellerman
2022-10-31 14:47 ` [PATCH] powerpc/32: fix syscall wrappers with 64-bit arguments Andreas Schwab
2022-10-31 19:37 ` Arnd Bergmann
2022-11-01 12:25 ` Michael Ellerman
2022-09-21 6:56 ` [PATCH v6 22/25] powerpc/64s: Clear user GPRs in syscall interrupt entry Rohan McLure
2022-09-23 8:02 ` Nicholas Piggin
2022-10-31 23:22 ` Rohan McLure
2022-09-21 6:56 ` [PATCH v6 23/25] powerpc/64: Add INTERRUPT_SANITIZE_REGISTERS Kconfig Rohan McLure
2022-09-21 6:56 ` Rohan McLure [this message]
2022-09-21 6:56 ` [PATCH v6 25/25] powerpc/64e: Clear gprs on interrupt routine entry on Book3E Rohan McLure
2022-10-04 13:24 ` [PATCH v6 00/25] powerpc: Syscall wrapper and register clearing Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220921065605.1051927-25-rmclure@linux.ibm.com \
--to=rmclure@linux.ibm.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.