From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Nicholas Piggin <npiggin@gmail.com>
Subject: [PATCH v4 03/17] powerpc/64s: introduce different functions to return from SRR vs HSRR interrupts
Date: Fri, 18 Jun 2021 01:51:02 +1000 [thread overview]
Message-ID: <20210617155116.2167984-4-npiggin@gmail.com> (raw)
In-Reply-To: <20210617155116.2167984-1-npiggin@gmail.com>
This makes no real difference yet except that HSRR type interrupts will
use hrfid to return. This is important for the next patch.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/entry_64.S | 65 +++++++++++++++++-------
arch/powerpc/kernel/exceptions-64e.S | 4 ++
arch/powerpc/kernel/exceptions-64s.S | 76 +++++++++++++++-------------
arch/powerpc/kernel/vector.S | 2 +-
4 files changed, 92 insertions(+), 55 deletions(-)
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 03727308d8cc..03a45a88b4b8 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -635,51 +635,57 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
* touched, no exit work created, then this can be used.
*/
.balign IFETCH_ALIGN_BYTES
- .globl fast_interrupt_return
-fast_interrupt_return:
-_ASM_NOKPROBE_SYMBOL(fast_interrupt_return)
+ .globl fast_interrupt_return_srr
+fast_interrupt_return_srr:
+_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
kuap_check_amr r3, r4
ld r5,_MSR(r1)
andi. r0,r5,MSR_PR
#ifdef CONFIG_PPC_BOOK3S
- bne .Lfast_user_interrupt_return_amr
+ bne .Lfast_user_interrupt_return_amr_srr
kuap_kernel_restore r3, r4
andi. r0,r5,MSR_RI
li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
- bne+ .Lfast_kernel_interrupt_return
+ bne+ .Lfast_kernel_interrupt_return_srr
addi r3,r1,STACK_FRAME_OVERHEAD
bl unrecoverable_exception
b . /* should not get here */
#else
- bne .Lfast_user_interrupt_return
- b .Lfast_kernel_interrupt_return
+ bne .Lfast_user_interrupt_return_srr
+ b .Lfast_kernel_interrupt_return_srr
#endif
+.macro interrupt_return_macro srr
.balign IFETCH_ALIGN_BYTES
- .globl interrupt_return
-interrupt_return:
-_ASM_NOKPROBE_SYMBOL(interrupt_return)
+ .globl interrupt_return_\srr
+interrupt_return_\srr\():
+_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
ld r4,_MSR(r1)
andi. r0,r4,MSR_PR
- beq .Lkernel_interrupt_return
+ beq .Lkernel_interrupt_return_\srr
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_user_prepare
cmpdi r3,0
- bne- .Lrestore_nvgprs
+ bne- .Lrestore_nvgprs_\srr
#ifdef CONFIG_PPC_BOOK3S
-.Lfast_user_interrupt_return_amr:
+.Lfast_user_interrupt_return_amr_\srr\():
kuap_user_restore r3, r4
#endif
-.Lfast_user_interrupt_return:
+.Lfast_user_interrupt_return_\srr\():
ld r11,_NIP(r1)
ld r12,_MSR(r1)
BEGIN_FTR_SECTION
ld r10,_PPR(r1)
mtspr SPRN_PPR,r10
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+ .ifc \srr,srr
mtspr SPRN_SRR0,r11
mtspr SPRN_SRR1,r12
+ .else
+ mtspr SPRN_HSRR0,r11
+ mtspr SPRN_HSRR1,r12
+ .endif
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
@@ -706,24 +712,33 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
REST_GPR(6, r1)
REST_GPR(0, r1)
REST_GPR(1, r1)
+ .ifc \srr,srr
RFI_TO_USER
+ .else
+ HRFI_TO_USER
+ .endif
b . /* prevent speculative execution */
-.Lrestore_nvgprs:
+.Lrestore_nvgprs_\srr\():
REST_NVGPRS(r1)
- b .Lfast_user_interrupt_return
+ b .Lfast_user_interrupt_return_\srr
.balign IFETCH_ALIGN_BYTES
-.Lkernel_interrupt_return:
+.Lkernel_interrupt_return_\srr\():
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_kernel_prepare
-.Lfast_kernel_interrupt_return:
+.Lfast_kernel_interrupt_return_\srr\():
cmpdi cr1,r3,0
ld r11,_NIP(r1)
ld r12,_MSR(r1)
+ .ifc \srr,srr
mtspr SPRN_SRR0,r11
mtspr SPRN_SRR1,r12
+ .else
+ mtspr SPRN_HSRR0,r11
+ mtspr SPRN_HSRR1,r12
+ .endif
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
@@ -757,7 +772,11 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
REST_GPR(6, r1)
REST_GPR(0, r1)
REST_GPR(1, r1)
+ .ifc \srr,srr
RFI_TO_KERNEL
+ .else
+ HRFI_TO_KERNEL
+ .endif
b . /* prevent speculative execution */
1: /*
@@ -777,8 +796,18 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
std r9,0(r1) /* perform store component of stdu */
ld r9,PACA_EXGEN+0(r13)
+ .ifc \srr,srr
RFI_TO_KERNEL
+ .else
+ HRFI_TO_KERNEL
+ .endif
b . /* prevent speculative execution */
+.endm
+
+interrupt_return_macro srr
+#ifdef CONFIG_PPC_BOOK3S
+interrupt_return_macro hsrr
+#endif
#ifdef CONFIG_PPC_RTAS
/*
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index f1ae710274bc..b35c97c7082f 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -26,6 +26,10 @@
#include <asm/feature-fixups.h>
#include <asm/context_tracking.h>
+/* 64e interrupt returns always use SRR registers */
+#define fast_interrupt_return fast_interrupt_return_srr
+#define interrupt_return interrupt_return_srr
+
/* XXX This will ultimately add space for a special exception save
* structure used to save things like SRR0/SRR1, SPRGs, MAS, etc...
* when taking special interrupts. For now we don't support that,
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index fa8e52a0239e..3a63feedae0b 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1234,7 +1234,7 @@ EXC_COMMON_BEGIN(machine_check_common)
mtmsrd r10,1
addi r3,r1,STACK_FRAME_OVERHEAD
bl machine_check_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM machine_check
@@ -1363,7 +1363,7 @@ BEGIN_MMU_FTR_SECTION
MMU_FTR_SECTION_ELSE
bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
- b interrupt_return
+ b interrupt_return_srr
1: bl do_break
/*
@@ -1371,7 +1371,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
* If so, we need to restore them with their updated values.
*/
REST_NVGPRS(r1)
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM data_access
@@ -1414,7 +1414,7 @@ BEGIN_MMU_FTR_SECTION
bl do_slb_fault
cmpdi r3,0
bne- 1f
- b fast_interrupt_return
+ b fast_interrupt_return_srr
1: /* Error case */
MMU_FTR_SECTION_ELSE
/* Radix case, access is outside page table range */
@@ -1423,7 +1423,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM data_access_slb
@@ -1461,7 +1461,7 @@ BEGIN_MMU_FTR_SECTION
MMU_FTR_SECTION_ELSE
bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM instruction_access
@@ -1498,7 +1498,7 @@ BEGIN_MMU_FTR_SECTION
bl do_slb_fault
cmpdi r3,0
bne- 1f
- b fast_interrupt_return
+ b fast_interrupt_return_srr
1: /* Error case */
MMU_FTR_SECTION_ELSE
/* Radix case, access is outside page table range */
@@ -1507,7 +1507,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
std r3,RESULT(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_bad_slb_fault
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM instruction_access_slb
@@ -1553,7 +1553,11 @@ EXC_COMMON_BEGIN(hardware_interrupt_common)
GEN_COMMON hardware_interrupt
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
- b interrupt_return
+ BEGIN_FTR_SECTION
+ b interrupt_return_hsrr
+ FTR_SECTION_ELSE
+ b interrupt_return_srr
+ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
GEN_KVM hardware_interrupt
@@ -1582,7 +1586,7 @@ EXC_COMMON_BEGIN(alignment_common)
addi r3,r1,STACK_FRAME_OVERHEAD
bl alignment_exception
REST_NVGPRS(r1) /* instruction emulation may change GPRs */
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM alignment
@@ -1691,7 +1695,7 @@ EXC_COMMON_BEGIN(program_check_common)
addi r3,r1,STACK_FRAME_OVERHEAD
bl program_check_exception
REST_NVGPRS(r1) /* instruction emulation may change GPRs */
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM program_check
@@ -1736,12 +1740,12 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_TM)
#endif
bl load_up_fpu
- b fast_interrupt_return
+ b fast_interrupt_return_srr
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2: /* User process was in a transaction */
addi r3,r1,STACK_FRAME_OVERHEAD
bl fp_unavailable_tm
- b interrupt_return
+ b interrupt_return_srr
#endif
GEN_KVM fp_unavailable
@@ -1782,7 +1786,7 @@ EXC_COMMON_BEGIN(decrementer_common)
GEN_COMMON decrementer
addi r3,r1,STACK_FRAME_OVERHEAD
bl timer_interrupt
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM decrementer
@@ -1870,7 +1874,7 @@ EXC_COMMON_BEGIN(doorbell_super_common)
#else
bl unknown_async_exception
#endif
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM doorbell_super
@@ -2041,7 +2045,7 @@ EXC_COMMON_BEGIN(single_step_common)
GEN_COMMON single_step
addi r3,r1,STACK_FRAME_OVERHEAD
bl single_step_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM single_step
@@ -2082,7 +2086,7 @@ BEGIN_MMU_FTR_SECTION
MMU_FTR_SECTION_ELSE
bl unknown_exception
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM h_data_storage
@@ -2109,7 +2113,7 @@ EXC_COMMON_BEGIN(h_instr_storage_common)
GEN_COMMON h_instr_storage
addi r3,r1,STACK_FRAME_OVERHEAD
bl unknown_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM h_instr_storage
@@ -2135,7 +2139,7 @@ EXC_COMMON_BEGIN(emulation_assist_common)
addi r3,r1,STACK_FRAME_OVERHEAD
bl emulation_assist_interrupt
REST_NVGPRS(r1) /* instruction emulation may change GPRs */
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM emulation_assist
@@ -2216,7 +2220,7 @@ EXC_COMMON_BEGIN(hmi_exception_common)
GEN_COMMON hmi_exception
addi r3,r1,STACK_FRAME_OVERHEAD
bl handle_hmi_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM hmi_exception
@@ -2248,7 +2252,7 @@ EXC_COMMON_BEGIN(h_doorbell_common)
#else
bl unknown_async_exception
#endif
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM h_doorbell
@@ -2276,7 +2280,7 @@ EXC_COMMON_BEGIN(h_virt_irq_common)
GEN_COMMON h_virt_irq
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_IRQ
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM h_virt_irq
@@ -2321,7 +2325,7 @@ EXC_COMMON_BEGIN(performance_monitor_common)
GEN_COMMON performance_monitor
addi r3,r1,STACK_FRAME_OVERHEAD
bl performance_monitor_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM performance_monitor
@@ -2360,19 +2364,19 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
#endif
bl load_up_altivec
- b fast_interrupt_return
+ b fast_interrupt_return_srr
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2: /* User process was in a transaction */
addi r3,r1,STACK_FRAME_OVERHEAD
bl altivec_unavailable_tm
- b interrupt_return
+ b interrupt_return_srr
#endif
1:
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
addi r3,r1,STACK_FRAME_OVERHEAD
bl altivec_unavailable_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM altivec_unavailable
@@ -2415,14 +2419,14 @@ BEGIN_FTR_SECTION
2: /* User process was in a transaction */
addi r3,r1,STACK_FRAME_OVERHEAD
bl vsx_unavailable_tm
- b interrupt_return
+ b interrupt_return_srr
#endif
1:
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
addi r3,r1,STACK_FRAME_OVERHEAD
bl vsx_unavailable_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM vsx_unavailable
@@ -2452,7 +2456,7 @@ EXC_COMMON_BEGIN(facility_unavailable_common)
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
REST_NVGPRS(r1) /* instruction emulation may change GPRs */
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM facility_unavailable
@@ -2482,7 +2486,7 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common)
addi r3,r1,STACK_FRAME_OVERHEAD
bl facility_unavailable_exception
REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM h_facility_unavailable
@@ -2513,7 +2517,7 @@ EXC_COMMON_BEGIN(cbe_system_error_common)
GEN_COMMON cbe_system_error
addi r3,r1,STACK_FRAME_OVERHEAD
bl cbe_system_error_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM cbe_system_error
@@ -2546,7 +2550,7 @@ EXC_COMMON_BEGIN(instruction_breakpoint_common)
GEN_COMMON instruction_breakpoint
addi r3,r1,STACK_FRAME_OVERHEAD
bl instruction_breakpoint_exception
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM instruction_breakpoint
@@ -2668,7 +2672,7 @@ EXC_COMMON_BEGIN(denorm_exception_common)
GEN_COMMON denorm_exception
addi r3,r1,STACK_FRAME_OVERHEAD
bl unknown_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM denorm_exception
@@ -2687,7 +2691,7 @@ EXC_COMMON_BEGIN(cbe_maintenance_common)
GEN_COMMON cbe_maintenance
addi r3,r1,STACK_FRAME_OVERHEAD
bl cbe_maintenance_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM cbe_maintenance
@@ -2719,7 +2723,7 @@ EXC_COMMON_BEGIN(altivec_assist_common)
#else
bl unknown_exception
#endif
- b interrupt_return
+ b interrupt_return_srr
GEN_KVM altivec_assist
@@ -2738,7 +2742,7 @@ EXC_COMMON_BEGIN(cbe_thermal_common)
GEN_COMMON cbe_thermal
addi r3,r1,STACK_FRAME_OVERHEAD
bl cbe_thermal_exception
- b interrupt_return
+ b interrupt_return_hsrr
GEN_KVM cbe_thermal
diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S
index f5a52f444e36..54dbefcb4cde 100644
--- a/arch/powerpc/kernel/vector.S
+++ b/arch/powerpc/kernel/vector.S
@@ -131,7 +131,7 @@ _GLOBAL(load_up_vsx)
/* enable use of VSX after return */
oris r12,r12,MSR_VSX@h
std r12,_MSR(r1)
- b fast_interrupt_return
+ b fast_interrupt_return_srr
#endif /* CONFIG_VSX */
--
2.23.0
next prev parent reply other threads:[~2021-06-17 15:53 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-17 15:50 [PATCH v4 00/17] powerpc/64: fast interrupt exits Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 01/17] powerpc/interrupt: Fix CONFIG ifdef typo Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 02/17] powerpc: remove interrupt exit helpers unused argument Nicholas Piggin
2021-06-17 15:51 ` Nicholas Piggin [this message]
2021-06-17 15:51 ` [PATCH v4 04/17] powerpc/64s: avoid reloading (H)SRR registers if they are still valid Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 05/17] powerpc/64: handle MSR EE and RI in interrupt entry wrapper Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 06/17] powerpc/64: move interrupt return asm to interrupt_64.S Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 07/17] powerpc/64s: system call avoid setting MSR[RI] until we set MSR[EE] Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 08/17] powerpc/64s: save one more register in the masked interrupt handler Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 09/17] powerpc/64: allow alternate return locations for soft-masked interrupts Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 10/17] powerpc/64: interrupt soft-enable race fix Nicholas Piggin
2022-05-16 16:36 ` Christophe Leroy
2021-06-17 15:51 ` [PATCH v4 11/17] powerpc/64: treat low kernel text as irqs soft-masked Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 12/17] powerpc/64: use interrupt restart table to speed up return from interrupt Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 13/17] powerpc/interrupt: Rename and lightly change syscall_exit_prepare_main() Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 14/17] powerpc/interrupt: Refactor interrupt_exit_user_prepare() Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 15/17] powerpc/interrupt: Interchange prep_irq_for_{kernel_enabled/user}_exit() Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 16/17] powerpc/interrupt: Refactor prep_irq_for_{user/kernel_enabled}_exit() Nicholas Piggin
2021-06-17 15:51 ` [PATCH v4 17/17] powerpc/interrupt: Remove prep_irq_for_user_exit() Nicholas Piggin
2021-06-25 6:21 ` [PATCH v4 00/17] powerpc/64: fast interrupt exits Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210617155116.2167984-4-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).