From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Nicholas Piggin <npiggin@gmail.com>
Subject: [PATCH v3 1/6] powerpc/64s: move stack SLB pinning out of line from _switch
Date: Tue, 6 Jun 2023 23:24:42 +1000 [thread overview]
Message-ID: <20230606132447.315714-2-npiggin@gmail.com> (raw)
In-Reply-To: <20230606132447.315714-1-npiggin@gmail.com>
The large hunk of SLB pinning in _switch asm code makes it more
difficult to see everything else that's going on. It is a less important
path now, so icache and fetch footprint overhead can be avoided.
Move context switch stack SLB pinning out of line.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/entry_64.S | 113 ++++++++++++++++++---------------
1 file changed, 62 insertions(+), 51 deletions(-)
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 1bf1121e17f1..2e02834c5824 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -105,6 +105,64 @@ flush_branch_caches:
.endr
blr
+
+#ifdef CONFIG_PPC_64S_HASH_MMU
+.balign 32
+/*
+ * New stack pointer in r8, old stack pointer in r1, must not clobber r3
+ */
+pin_stack_slb:
+BEGIN_FTR_SECTION
+ clrrdi r6,r8,28 /* get its ESID */
+ clrrdi r9,r1,28 /* get current sp ESID */
+FTR_SECTION_ELSE
+ clrrdi r6,r8,40 /* get its 1T ESID */
+ clrrdi r9,r1,40 /* get current sp 1T ESID */
+ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
+ clrldi. r0,r6,2 /* is new ESID c00000000? */
+ cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
+ cror eq,4*cr1+eq,eq
+ beq 2f /* if yes, don't slbie it */
+
+ /* Bolt in the new stack SLB entry */
+ ld r7,KSP_VSID(r4) /* Get new stack's VSID */
+ oris r0,r6,(SLB_ESID_V)@h
+ ori r0,r0,(SLB_NUM_BOLTED-1)@l
+BEGIN_FTR_SECTION
+ li r9,MMU_SEGSIZE_1T /* insert B field */
+ oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
+ rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
+
+ /* Update the last bolted SLB. No write barriers are needed
+ * here, provided we only update the current CPU's SLB shadow
+ * buffer.
+ */
+ ld r9,PACA_SLBSHADOWPTR(r13)
+ li r12,0
+ std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
+ li r12,SLBSHADOW_STACKVSID
+ STDX_BE r7,r12,r9 /* Save VSID */
+ li r12,SLBSHADOW_STACKESID
+ STDX_BE r0,r12,r9 /* Save ESID */
+
+ /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
+ * we have 1TB segments, the only CPUs known to have the errata
+ * only support less than 1TB of system memory and we'll never
+ * actually hit this code path.
+ */
+
+ isync
+ slbie r6
+BEGIN_FTR_SECTION
+ slbie r6 /* Workaround POWER5 < DD2.1 issue */
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
+ slbmte r7,r0
+ isync
+2: blr
+ .size pin_stack_slb,.-pin_stack_slb
+#endif /* CONFIG_PPC_64S_HASH_MMU */
+
#else
#define FLUSH_COUNT_CACHE
#endif /* CONFIG_PPC_BOOK3S_64 */
@@ -182,59 +240,12 @@ _GLOBAL(_switch)
#endif
ld r8,KSP(r4) /* new stack pointer */
+
#ifdef CONFIG_PPC_64S_HASH_MMU
BEGIN_MMU_FTR_SECTION
- b 2f
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
-BEGIN_FTR_SECTION
- clrrdi r6,r8,28 /* get its ESID */
- clrrdi r9,r1,28 /* get current sp ESID */
-FTR_SECTION_ELSE
- clrrdi r6,r8,40 /* get its 1T ESID */
- clrrdi r9,r1,40 /* get current sp 1T ESID */
-ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
- clrldi. r0,r6,2 /* is new ESID c00000000? */
- cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
- cror eq,4*cr1+eq,eq
- beq 2f /* if yes, don't slbie it */
-
- /* Bolt in the new stack SLB entry */
- ld r7,KSP_VSID(r4) /* Get new stack's VSID */
- oris r0,r6,(SLB_ESID_V)@h
- ori r0,r0,(SLB_NUM_BOLTED-1)@l
-BEGIN_FTR_SECTION
- li r9,MMU_SEGSIZE_1T /* insert B field */
- oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
- rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
-
- /* Update the last bolted SLB. No write barriers are needed
- * here, provided we only update the current CPU's SLB shadow
- * buffer.
- */
- ld r9,PACA_SLBSHADOWPTR(r13)
- li r12,0
- std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
- li r12,SLBSHADOW_STACKVSID
- STDX_BE r7,r12,r9 /* Save VSID */
- li r12,SLBSHADOW_STACKESID
- STDX_BE r0,r12,r9 /* Save ESID */
-
- /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
- * we have 1TB segments, the only CPUs known to have the errata
- * only support less than 1TB of system memory and we'll never
- * actually hit this code path.
- */
-
- isync
- slbie r6
-BEGIN_FTR_SECTION
- slbie r6 /* Workaround POWER5 < DD2.1 issue */
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
- slbmte r7,r0
- isync
-2:
-#endif /* CONFIG_PPC_64S_HASH_MMU */
+ bl pin_stack_slb
+END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
+#endif
clrrdi r7, r8, THREAD_SHIFT /* base of new stack */
/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
--
2.40.1
next prev parent reply other threads:[~2023-06-06 13:26 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-06-06 13:24 [PATCH v3 0/6] powerpc: merge _switch in 32/64 Nicholas Piggin
2023-06-06 13:24 ` Nicholas Piggin [this message]
2023-06-06 13:24 ` [PATCH v3 2/6] powerpc/64: Rearrange 64-bit _switch to prepare for 32/64 merge Nicholas Piggin
2023-06-06 13:24 ` [PATCH v3 3/6] powerpc/32: Remove sync from _switch Nicholas Piggin
2023-06-06 13:24 ` [PATCH v3 4/6] powerpc/32: Rearrange _switch to prepare for 32/64 merge Nicholas Piggin
2023-06-06 13:24 ` [PATCH v3 5/6] powerpc: merge 32-bit and 64-bit _switch implementation Nicholas Piggin
2023-06-06 13:24 ` [PATCH v3 6/6] powerpc/64: Rename entry_64.S to prom_entry_64.S Nicholas Piggin
2023-07-03 5:26 ` [PATCH v3 0/6] powerpc: merge _switch in 32/64 Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230606132447.315714-2-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).