From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from VA3EHSOBE002.bigfish.com (va3ehsobe002.messaging.microsoft.com [216.32.180.12]) (using TLSv1 with cipher AES128-SHA (128/128 bits)) (Client CN "mail.global.frontbridge.com", Issuer "Cybertrust SureServer Standard Validation CA" (verified OK)) by ozlabs.org (Postfix) with ESMTPS id CB441B6F68 for ; Thu, 19 May 2011 07:20:52 +1000 (EST) Date: Wed, 18 May 2011 16:05:33 -0500 From: Scott Wood To: Subject: [PATCH 4/7] powerpc/mm: 64-bit: Don't load PACA in normal TLB miss exceptions Message-ID: <20110518210533.GC29524@schlenkerla.am.freescale.net> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" In-Reply-To: <20110518210453.GA29500@schlenkerla.am.freescale.net> Cc: linuxppc-dev@lists.ozlabs.org List-Id: Linux on PowerPC Developers Mail List List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Load it only when needed, in recursive/linear/indirect faults, and in the stats code. Signed-off-by: Scott Wood --- arch/powerpc/include/asm/exception-64e.h | 28 +++++++++--------- arch/powerpc/mm/tlb_low_64e.S | 43 +++++++++++++++++------------ 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h index 6921261..9b57a27 100644 --- a/arch/powerpc/include/asm/exception-64e.h +++ b/arch/powerpc/include/asm/exception-64e.h @@ -80,9 +80,9 @@ exc_##label##_book3e: * * This prolog handles re-entrancy (up to 3 levels supported in the PACA * though we currently don't test for overflow). It provides you with a - * re-entrancy safe working space of r10...r16 and CR with r12 being used - * as the exception area pointer in the PACA for that level of re-entrancy - * and r13 containing the PACA pointer. + * re-entrancy safe working space of r10...r16 (except r13) and CR with r12 + * being used as the exception area pointer in the PACA for that level of + * re-entrancy. * * SRR0 and SRR1 are saved, but DEAR and ESR are not, since they don't apply * as-is for instruction exceptions. It's up to the actual exception code @@ -95,8 +95,6 @@ exc_##label##_book3e: mfcr r10; \ std r11,EX_TLB_R11(r12); \ mfspr r11,SPRN_SPRG_TLB_SCRATCH; \ - std r13,EX_TLB_R13(r12); \ - ld r13,EX_TLB_PACA(r12); \ std r14,EX_TLB_R14(r12); \ addi r14,r12,EX_TLB_SIZE; \ std r15,EX_TLB_R15(r12); \ @@ -135,7 +133,6 @@ exc_##label##_book3e: mtspr SPRN_SPRG_TLB_EXFRAME,freg; \ ld r11,EX_TLB_R11(r12); \ mtcr r14; \ - ld r13,EX_TLB_R13(r12); \ ld r14,EX_TLB_R14(r12); \ mtspr SPRN_SRR0,r15; \ ld r15,EX_TLB_R15(r12); \ @@ -148,11 +145,13 @@ exc_##label##_book3e: TLB_MISS_RESTORE(r12) #define TLB_MISS_EPILOG_ERROR \ - addi r12,r13,PACA_EXTLB; \ + ld r10,EX_TLB_PACA(r12); \ + addi r12,r10,PACA_EXTLB; \ TLB_MISS_RESTORE(r12) #define TLB_MISS_EPILOG_ERROR_SPECIAL \ - addi r11,r13,PACA_EXTLB; \ + ld r10,EX_TLB_PACA(r12); \ + addi r11,r10,PACA_EXTLB; \ TLB_MISS_RESTORE(r11) #ifdef CONFIG_BOOK3E_MMU_TLB_STATS @@ -160,25 +159,26 @@ exc_##label##_book3e: mflr r10; \ std r8,EX_TLB_R8(r12); \ std r9,EX_TLB_R9(r12); \ - std r10,EX_TLB_LR(r12); + std r10,EX_TLB_LR(r12); \ + ld r9,EX_TLB_PACA(r12); #define TLB_MISS_RESTORE_STATS \ ld r16,EX_TLB_LR(r12); \ ld r9,EX_TLB_R9(r12); \ ld r8,EX_TLB_R8(r12); \ mtlr r16; #define TLB_MISS_STATS_D(name) \ - addi r9,r13,MMSTAT_DSTATS+name; \ + addi r9,r9,MMSTAT_DSTATS+name; \ bl .tlb_stat_inc; #define TLB_MISS_STATS_I(name) \ - addi r9,r13,MMSTAT_ISTATS+name; \ + addi r9,r9,MMSTAT_ISTATS+name; \ bl .tlb_stat_inc; #define TLB_MISS_STATS_X(name) \ - ld r8,PACA_EXTLB+EX_TLB_ESR(r13); \ + ld r8,PACA_EXTLB+EX_TLB_ESR(r9); \ cmpdi cr2,r8,-1; \ beq cr2,61f; \ - addi r9,r13,MMSTAT_DSTATS+name; \ + addi r9,r9,MMSTAT_DSTATS+name; \ b 62f; \ -61: addi r9,r13,MMSTAT_ISTATS+name; \ +61: addi r9,r9,MMSTAT_ISTATS+name; \ 62: bl .tlb_stat_inc; #define TLB_MISS_STATS_SAVE_INFO \ std r14,EX_TLB_ESR(r12); /* save ESR */ \ diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S index 17726d3..922fece 100644 --- a/arch/powerpc/mm/tlb_low_64e.S +++ b/arch/powerpc/mm/tlb_low_64e.S @@ -160,7 +160,6 @@ * r16 = faulting address * r15 = region ID * r14 = crap (free to use) - * r13 = PACA * r12 = TLB exception frame in PACA * r11 = PTE permission mask * r10 = crap (free to use) @@ -299,8 +298,7 @@ normal_tlb_miss_access_fault: * * r16 = virtual page table faulting address * r15 = region (top 4 bits of address) - * r14 = crap (free to use) - * r13 = PACA + * r14 = crap (we load with PACA) * r12 = TLB exception frame in PACA * r11 = crap (free to use) * r10 = crap (free to use) @@ -318,6 +316,8 @@ normal_tlb_miss_access_fault: * so we could probably optimize things a bit */ virt_page_table_tlb_miss: + ld r14,EX_TLB_PACA(r12) + /* Are we hitting a kernel page table ? */ andi. r10,r15,0x8 @@ -325,7 +325,7 @@ virt_page_table_tlb_miss: * and we happen to have the swapper_pg_dir at offset 8 from the user * pgdir in the PACA :-). */ - add r11,r10,r13 + add r11,r10,r14 /* If kernel, we need to clear MAS1 TID */ beq 1f @@ -417,12 +417,12 @@ virt_page_table_tlb_miss_done: * offset the return address by -4 in order to replay the tlbsrx * instruction there */ - subf r10,r13,r12 + subf r10,r14,r12 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE bne- 1f - ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13) + ld r11,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14) addi r10,r11,-4 - std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r13) + std r10,PACA_EXTLB+EX_TLB_SIZE+EX_TLB_SRR0(r14) 1: END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV) /* Return to caller, normal case */ @@ -449,13 +449,13 @@ virt_page_table_tlb_miss_fault: * level as well. Since we are doing that, we don't need to clear or * restore the TLB reservation neither. */ - subf r10,r13,r12 + subf r10,r14,r12 cmpldi cr0,r10,PACA_EXTLB+EX_TLB_SIZE bne- virt_page_table_tlb_miss_whacko_fault /* We dig the original DEAR and ESR from slot 0 */ - ld r15,EX_TLB_DEAR+PACA_EXTLB(r13) - ld r16,EX_TLB_ESR+PACA_EXTLB(r13) + ld r15,EX_TLB_DEAR+PACA_EXTLB(r14) + ld r16,EX_TLB_ESR+PACA_EXTLB(r14) /* We check for the "special" ESR value for instruction faults */ cmpdi cr0,r16,-1 @@ -489,6 +489,8 @@ virt_page_table_tlb_miss_whacko_fault: START_EXCEPTION(data_tlb_miss_htw) TLB_MISS_PROLOG + ld r15,EX_TLB_PACA(r12) + /* Now we handle the fault proper. We only save DEAR in normal * fault case since that's the only interesting values here. * We could probably also optimize by not saving SRR0/1 in the @@ -503,8 +505,12 @@ virt_page_table_tlb_miss_whacko_fault: /* We do the user/kernel test for the PID here along with the RW test */ + /* The cool thing now is that r11 contains 0 for user and 8 for kernel, + * and we happen to have the swapper_pg_dir at offset 8 from the user + * pgdir in the PACA :-). + */ cmpldi cr0,r11,0 /* Check for user region */ - ld r15,PACAPGD(r13) /* Load user pgdir */ + add r15,r15,r11 beq htw_tlb_miss /* XXX replace the RMW cycles with immediate loads + writes */ @@ -512,7 +518,6 @@ virt_page_table_tlb_miss_whacko_fault: cmpldi cr0,r11,8 /* Check for vmalloc region */ rlwinm r10,r10,0,16,1 /* Clear TID */ mtspr SPRN_MAS1,r10 - ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */ beq+ htw_tlb_miss /* We got a crappy address, just fault with whatever DEAR and ESR @@ -526,6 +531,8 @@ virt_page_table_tlb_miss_whacko_fault: START_EXCEPTION(instruction_tlb_miss_htw) TLB_MISS_PROLOG + ld r15,EX_TLB_PACA(r12) + /* If we take a recursive fault, the second level handler may need * to know whether we are handling a data or instruction fault in * order to get to the right store fault handler. We provide that @@ -548,7 +555,7 @@ virt_page_table_tlb_miss_whacko_fault: /* We do the user/kernel test for the PID here along with the RW test */ cmpldi cr0,r11,0 /* Check for user region */ - ld r15,PACAPGD(r13) /* Load user pgdir */ + add r15,r15,r11 beq htw_tlb_miss /* XXX replace the RMW cycles with immediate loads + writes */ @@ -556,7 +563,6 @@ virt_page_table_tlb_miss_whacko_fault: cmpldi cr0,r11,8 /* Check for vmalloc region */ rlwinm r10,r10,0,16,1 /* Clear TID */ mtspr SPRN_MAS1,r10 - ld r15,PACA_KERNELPGD(r13) /* Load kernel pgdir */ beq+ htw_tlb_miss /* We got a crappy address, just fault */ @@ -570,9 +576,8 @@ virt_page_table_tlb_miss_whacko_fault: * misses. We are entered with: * * r16 = virtual page table faulting address - * r15 = PGD pointer + * r15 = pointer to PGD pointer * r14 = ESR - * r13 = PACA * r12 = TLB exception frame in PACA * r11 = crap (free to use) * r10 = crap (free to use) @@ -581,6 +586,8 @@ virt_page_table_tlb_miss_whacko_fault: * avoid too much complication, it will save/restore things for us */ htw_tlb_miss: + ld r15,0(r15) + /* Search if we already have a TLB entry for that virtual address, and * if we do, bail out. * @@ -692,7 +699,6 @@ htw_tlb_miss_fault: * r16 = faulting address * r15 = crap (free to use) * r14 = ESR (data) or -1 (instruction) - * r13 = PACA * r12 = TLB exception frame in PACA * r11 = crap (free to use) * r10 = crap (free to use) @@ -714,7 +720,8 @@ tlb_load_linear: * we only use 1G pages for now. That might have to be changed in a * final implementation, especially when dealing with hypervisors */ - ld r11,PACATOC(r13) + ld r11,EX_TLB_PACA(r12) + ld r11,PACATOC(r11) ld r11,linear_map_top@got(r11) ld r10,0(r11) cmpld cr0,r10,r16 -- 1.7.4.1