[PATCH v3 0/7] powerpc/8xx: Optimisation of TLB handling for IMMR and RAM

[PATCH v3 0/7] powerpc/8xx: Optimisation of TLB handling for IMMR and RAM

[Christophe Leroy]

The purpose of this set of patches is to continue on TLB handling
optimisation on the 8xx with the handling of IMMR area as a
single 512k area instead of multiple 4k pages.

This set includes a rework of linear RAM mapping in order to not use
page table but direct linear mapping. The result is equivalent to the
use of BATs on other PPCs and it avoids reading the page table.

This set also includes a rework of the handling of PIN_TLB config
option. This option used to be linked to the quantity of RAM available.
It is now independant, users having only 8M memory can also use PIN_TLB.

Christophe Leroy (7):
  powerpc/8xx: Fix vaddr for IMMR early remap
  powerpc/8xx: Map IMMR area with 512k page at a fixed address
  powerpc/8xx: CONFIG_PIN_TLB unneeded for CONFIG_PPC_EARLY_DEBUG_CPM
  powerpc/8xx: unpin all TLBs before flushing
  powerpc/8xx: Don't use page table for linear memory space
  powerpc/8xx: Rework CONFIG_PIN_TLB handling
  powerpc/8xx: add CONFIG_PIN_TLB_IMMR

 arch/powerpc/Kconfig               |   5 ++
 arch/powerpc/Kconfig.debug         |   1 -
 arch/powerpc/include/asm/fixmap.h  |   7 ++
 arch/powerpc/include/asm/mmu-8xx.h |   3 +
 arch/powerpc/kernel/asm-offsets.c  |   8 ++
 arch/powerpc/kernel/head_8xx.S     | 159 +++++++++++++++++++------------------
 arch/powerpc/mm/8xx_mmu.c          | 131 ++++++++++++++++++++----------
 arch/powerpc/mm/mmu_decl.h         |   3 +-
 arch/powerpc/sysdev/cpm_common.c   |  22 +++--
 9 files changed, 212 insertions(+), 127 deletions(-)

-- 
v2: Fixed a test in the 6/7
v3: Fixed issue with 82xx

2.1.0

[PATCH v3 1/7] powerpc/8xx: Fix vaddr for IMMR early remap

[Christophe Leroy]

Memory: 124428K/131072K available (3748K kernel code, 188K rwdata,
648K rodata, 508K init, 290K bss, 6644K reserved)
Kernel virtual memory layout:
  * 0xfffdf000..0xfffff000  : fixmap
  * 0xfde00000..0xfe000000  : consistent mem
  * 0xfddf6000..0xfde00000  : early ioremap
  * 0xc9000000..0xfddf6000  : vmalloc & ioremap
SLUB: HWalign=16, Order=0-3, MinObjects=0, CPUs=1, Nodes=1

Today, IMMR is mapped 1:1 at startup

Mapping IMMR 1:1 is just wrong because it may overlap with another
area. On most mpc8xx boards it is OK as IMMR is set to 0xff000000
but for instance on EP88xC board, IMMR is at 0xfa200000 which
overlaps with VM ioremap area

This patch fixes the virtual address for remapping IMMR with the fixmap
regardless of the value of IMMR.

The size of IMMR area is 256kbytes (CPM at offset 0, security engine
at offset 128k) so a 512k page is enough

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: Fixed issue with EARLY_DEBUG on 82xx

 arch/powerpc/include/asm/fixmap.h  |  7 +++++++
 arch/powerpc/include/asm/mmu-8xx.h |  3 +++
 arch/powerpc/kernel/asm-offsets.c  |  8 ++++++++
 arch/powerpc/kernel/head_8xx.S     | 11 ++++++-----
 arch/powerpc/sysdev/cpm_common.c   | 22 +++++++++++++++++-----
 5 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h
index 90f604b..4508b32 100644
--- a/arch/powerpc/include/asm/fixmap.h
+++ b/arch/powerpc/include/asm/fixmap.h
@@ -51,6 +51,13 @@ enum fixed_addresses {
 	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */
 	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
 #endif
+#ifdef CONFIG_PPC_8xx
+	/* For IMMR we need an aligned 512K area */
+#define FIX_IMMR_SIZE	(512 * 1024 / PAGE_SIZE)
+	FIX_IMMR_START,
+	FIX_IMMR_BASE = __ALIGN_MASK(FIX_IMMR_START, FIX_IMMR_SIZE - 1) - 1 +
+		       FIX_IMMR_SIZE,
+#endif
 	/* FIX_PCIE_MCFG, */
 	__end_of_fixed_addresses
 };
diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h
index 0a566f1..3e0e492 100644
--- a/arch/powerpc/include/asm/mmu-8xx.h
+++ b/arch/powerpc/include/asm/mmu-8xx.h
@@ -169,6 +169,9 @@ typedef struct {
 	unsigned int active;
 	unsigned long vdso_base;
 } mm_context_t;
+
+#define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000)
+#define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE))
 #endif /* !__ASSEMBLY__ */
 
 #if defined(CONFIG_PPC_4K_PAGES)
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 9ea0955..2652233 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -68,6 +68,10 @@
 #include "../mm/mmu_decl.h"
 #endif
 
+#ifdef CONFIG_PPC_8xx
+#include <asm/fixmap.h>
+#endif
+
 int main(void)
 {
 	DEFINE(THREAD, offsetof(struct task_struct, thread));
@@ -783,5 +787,9 @@ int main(void)
 
 	DEFINE(PPC_DBELL_SERVER, PPC_DBELL_SERVER);
 
+#ifdef CONFIG_PPC_8xx
+	DEFINE(VIRT_IMMR_BASE, __fix_to_virt(FIX_IMMR_BASE));
+#endif
+
 	return 0;
 }
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 80c6947..378a185 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -30,6 +30,7 @@
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/ptrace.h>
+#include <asm/fixmap.h>
 
 /* Macro to make the code more readable. */
 #ifdef CONFIG_8xx_CPU6
@@ -763,7 +764,7 @@ start_here:
  * virtual to physical.  Also, set the cache mode since that is defined
  * by TLB entries and perform any additional mapping (like of the IMMR).
  * If configured to pin some TLBs, we pin the first 8 Mbytes of kernel,
- * 24 Mbytes of data, and the 8M IMMR space.  Anything not covered by
+ * 24 Mbytes of data, and the 512k IMMR space.  Anything not covered by
  * these mappings is mapped by page tables.
  */
 initial_mmu:
@@ -812,7 +813,7 @@ initial_mmu:
 	ori	r8, r8, MD_APG_INIT@l
 	mtspr	SPRN_MD_AP, r8
 
-	/* Map another 8 MByte at the IMMR to get the processor
+	/* Map a 512k page for the IMMR to get the processor
 	 * internal registers (among other things).
 	 */
 #ifdef CONFIG_PIN_TLB
@@ -820,12 +821,12 @@ initial_mmu:
 	mtspr	SPRN_MD_CTR, r10
 #endif
 	mfspr	r9, 638			/* Get current IMMR */
-	andis.	r9, r9, 0xff80		/* Get 8Mbyte boundary */
+	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
 
-	mr	r8, r9			/* Create vaddr for TLB */
+	lis	r8, VIRT_IMMR_BASE@h	/* Create vaddr for TLB */
 	ori	r8, r8, MD_EVALID	/* Mark it valid */
 	mtspr	SPRN_MD_EPN, r8
-	li	r8, MD_PS8MEG		/* Set 8M byte page */
+	li	r8, MD_PS512K | MD_GUARDED	/* Set 512k byte page */
 	ori	r8, r8, MD_SVALID	/* Make it valid */
 	mtspr	SPRN_MD_TWC, r8
 	mr	r8, r9			/* Create paddr for TLB */
diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c
index 0ac12e5..911456d 100644
--- a/arch/powerpc/sysdev/cpm_common.c
+++ b/arch/powerpc/sysdev/cpm_common.c
@@ -28,6 +28,7 @@
 #include <asm/udbg.h>
 #include <asm/io.h>
 #include <asm/cpm.h>
+#include <asm/fixmap.h>
 #include <soc/fsl/qe/qe.h>
 
 #include <mm/mmu_decl.h>
@@ -37,25 +38,36 @@
 #endif
 
 #ifdef CONFIG_PPC_EARLY_DEBUG_CPM
-static u32 __iomem *cpm_udbg_txdesc =
-	(u32 __iomem __force *)CONFIG_PPC_EARLY_DEBUG_CPM_ADDR;
+static u32 __iomem *cpm_udbg_txdesc;
+static u8 __iomem *cpm_udbg_txbuf;
 
 static void udbg_putc_cpm(char c)
 {
-	u8 __iomem *txbuf = (u8 __iomem __force *)in_be32(&cpm_udbg_txdesc[1]);
-
 	if (c == '\n')
 		udbg_putc_cpm('\r');
 
 	while (in_be32(&cpm_udbg_txdesc[0]) & 0x80000000)
 		;
 
-	out_8(txbuf, c);
+	out_8(cpm_udbg_txbuf, c);
 	out_be32(&cpm_udbg_txdesc[0], 0xa0000001);
 }
 
 void __init udbg_init_cpm(void)
 {
+#ifdef CONFIG_PPC_8xx
+	cpm_udbg_txdesc = (u32 __iomem __force *)
+			  (CONFIG_PPC_EARLY_DEBUG_CPM_ADDR - PHYS_IMMR_BASE +
+			   VIRT_IMMR_BASE);
+	cpm_udbg_txbuf = (u8 __iomem __force *)
+			 (in_be32(&cpm_udbg_txdesc[1]) - PHYS_IMMR_BASE +
+			  VIRT_IMMR_BASE);
+#else
+	cpm_udbg_txdesc = (u32 __iomem __force *)
+			  CONFIG_PPC_EARLY_DEBUG_CPM_ADDR;
+	cpm_udbg_txbuf = (u8 __iomem __force *)in_be32(&cpm_udbg_txdesc[1]);
+#endif
+
 	if (cpm_udbg_txdesc) {
 #ifdef CONFIG_CPM2
 		setbat(1, 0xf0000000, 0xf0000000, 1024*1024, PAGE_KERNEL_NCG);
-- 
2.1.0

[PATCH v3 2/7] powerpc/8xx: Map IMMR area with 512k page at a fixed address

[Christophe Leroy]

Once the linear memory space has been mapped with 8Mb pages, as
seen in the related commit, we get 11 millions DTLB missed during
the reference 600s period. 77% of the misses are on user addresses
and 23% are on kernel addresses (1 fourth for linear address space
and 3 fourth for virtual address space)

Traditionaly, each driver manages one computer board which has its
own components with its own memory maps.
But on embedded chips like the MPC8xx, the SOC has all registers
located in the same IO area.

When looking at ioremaps done during startup, we see that
many drivers are re-mapping small parts of the IMMR for their own use
and all those small pieces gets their own 4k page, amplifying the
number of TLB misses: in our system we get 0xff000000 mapped 31 times
and 0xff003000 mapped 9 times.

Even if each part of IMMR was mapped only once with 4k pages, it would
still be several small mappings towards linear area.

This patch maps the IMMR with a single 512k page.

With this patch applied, the number of DTLB misses during the 10 min
period is reduced to 11.8 millions for a duration of 5.8s, which
represents 2% of the non-idle time hence yet another 10% reduction.

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: No change

 arch/powerpc/kernel/head_8xx.S | 29 ++++++++++++++++++++++
 arch/powerpc/mm/8xx_mmu.c      | 56 +++++++++++++++++++++++++++++++++++++++++-
 arch/powerpc/mm/mmu_decl.h     |  3 ++-
 3 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 378a185..44f4edb 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -384,6 +384,27 @@ InstructionTLBMiss:
 	EXCEPTION_EPILOG_0
 	rfi
 
+/*
+ * Bottom part of DataStoreTLBMiss handler for IMMR area
+ * not enough space in the DataStoreTLBMiss area
+ */
+DTLBMissIMMR:
+	mtcr	r3
+	/* Set 512k byte guarded page and mark it valid */
+	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
+	MTSPR_CPU6(SPRN_MD_TWC, r10, r3)
+	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
+	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
+	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
+			  _PAGE_PRESENT | _PAGE_NO_CACHE
+	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
+
+	li	r11, RPN_PATTERN
+	mfspr	r3, SPRN_SPRG_SCRATCH2
+	mtspr	SPRN_DAR, r11	/* Tag DAR */
+	EXCEPTION_EPILOG_0
+	rfi
+
 	. = 0x1200
 DataStoreTLBMiss:
 	mtspr	SPRN_SPRG_SCRATCH2, r3
@@ -397,6 +418,14 @@ DataStoreTLBMiss:
 	IS_KERNEL(r11, r10)
 	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
 	BRANCH_UNLESS_KERNEL(3f)
+
+	rlwinm	r11, r10, 16, 0xfff8
+#ifndef CONFIG_PIN_TLB
+	cmpli	cr0, r11, VIRT_IMMR_BASE@h
+_ENTRY(DTLBMiss_jmp)
+	beq-	DTLBMissIMMR
+#endif
+
 	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
 3:
 
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 9491005..2207725 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -13,10 +13,43 @@
  */
 
 #include <linux/memblock.h>
+#include <asm/fixmap.h>
+#include <asm/code-patching.h>
 
 #include "mmu_decl.h"
 
+#define IMMR_SIZE (FIX_IMMR_SIZE << PAGE_SHIFT)
+
 extern int __map_without_ltlbs;
+
+/*
+ * Return PA for this VA if it is in IMMR area, or 0
+ */
+phys_addr_t v_block_mapped(unsigned long va)
+{
+	unsigned long p = PHYS_IMMR_BASE;
+
+	if (__map_without_ltlbs)
+		return 0;
+	if (va >= VIRT_IMMR_BASE && va < VIRT_IMMR_BASE + IMMR_SIZE)
+		return p + va - VIRT_IMMR_BASE;
+	return 0;
+}
+
+/*
+ * Return VA for a given PA or 0 if not mapped
+ */
+unsigned long p_block_mapped(phys_addr_t pa)
+{
+	unsigned long p = PHYS_IMMR_BASE;
+
+	if (__map_without_ltlbs)
+		return 0;
+	if (pa >= p && pa < p + IMMR_SIZE)
+		return VIRT_IMMR_BASE + pa - p;
+	return 0;
+}
+
 /*
  * MMU_init_hw does the chip-specific initialization of the MMU hardware.
  */
@@ -29,6 +62,22 @@ void __init MMU_init_hw(void)
 #define LARGE_PAGE_SIZE_8M	(1<<23)
 #define LARGE_PAGE_SIZE_64M	(1<<26)
 
+static void mmu_mapin_immr(void)
+{
+	unsigned long p = PHYS_IMMR_BASE;
+	unsigned long v = VIRT_IMMR_BASE;
+	unsigned long f = pgprot_val(PAGE_KERNEL_NCG);
+	int offset;
+
+	for (offset = 0; offset < IMMR_SIZE; offset += PAGE_SIZE)
+		map_page(v + offset, p + offset, f);
+}
+
+/* Address of instructions to patch */
+#ifndef CONFIG_PIN_TLB
+extern unsigned int DTLBMiss_jmp;
+#endif
+
 unsigned long __init mmu_mapin_ram(unsigned long top)
 {
 	unsigned long v, s, mapped;
@@ -38,8 +87,13 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
 	p = 0;
 	s = top;
 
-	if (__map_without_ltlbs)
+	if (__map_without_ltlbs) {
+		mmu_mapin_immr();
+#ifndef CONFIG_PIN_TLB
+		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
+#endif
 		return 0;
+	}
 
 #ifdef CONFIG_PPC_4K_PAGES
 	while (s >= LARGE_PAGE_SIZE_8M) {
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index 6af6532..f988db6 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -154,9 +154,10 @@ struct tlbcam {
 };
 #endif
 
-#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE)
+#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE) || defined(CONFIG_PPC_8xx)
 /* 6xx have BATS */
 /* FSL_BOOKE have TLBCAM */
+/* 8xx have LTLB */
 phys_addr_t v_block_mapped(unsigned long va);
 unsigned long p_block_mapped(phys_addr_t pa);
 #else
-- 
2.1.0

[PATCH v3 3/7] powerpc/8xx: CONFIG_PIN_TLB unneeded for CONFIG_PPC_EARLY_DEBUG_CPM

[Christophe Leroy]

IMMR is now mapped by a fixed 512k page managed by the TLB miss
handler so it is not anymore necessary to PIN TLBs

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: No change

 arch/powerpc/Kconfig.debug | 1 -
 1 file changed, 1 deletion(-)

diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug
index d3fcf7e..5eccbb5 100644
--- a/arch/powerpc/Kconfig.debug
+++ b/arch/powerpc/Kconfig.debug
@@ -212,7 +212,6 @@ config PPC_EARLY_DEBUG_40x
 config PPC_EARLY_DEBUG_CPM
 	bool "Early serial debugging for Freescale CPM-based serial ports"
 	depends on SERIAL_CPM
-	select PIN_TLB if PPC_8xx
 	help
 	  Select this to enable early debugging for Freescale chips
 	  using a CPM-based serial port.  This assumes that the bootwrapper
-- 
2.1.0

[PATCH v3 4/7] powerpc/8xx: unpin all TLBs before flushing

[Christophe Leroy]

Bootloader may have pinned some TLB entries so the kernel must
unpin them before flushing TLBs with tlbia otherwise pinned TLB
entries won't get flushed

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: No change

 arch/powerpc/kernel/head_8xx.S | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 44f4edb..d9a1656 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -797,6 +797,14 @@ start_here:
  * these mappings is mapped by page tables.
  */
 initial_mmu:
+	li	r8, 0
+	mtspr	SPRN_MI_CTR, r8		/* remove PINNED ITLB entries */
+	lis	r10, MD_RESETVAL@h
+#ifndef CONFIG_8xx_COPYBACK
+	oris	r10, r10, MD_WTDEF@h
+#endif
+	mtspr	SPRN_MD_CTR, r10	/* remove PINNED DTLB entries */
+
 	tlbia			/* Invalidate all TLB entries */
 /* Always pin the first 8 MB ITLB to prevent ITLB
    misses while mucking around with SRR0/SRR1 in asm
@@ -807,16 +815,10 @@ initial_mmu:
 	mtspr	SPRN_MI_CTR, r8	/* Set instruction MMU control */
 
 #ifdef CONFIG_PIN_TLB
-	lis	r10, (MD_RSV4I | MD_RESETVAL)@h
+	oris	r10, r10, MD_RSV4I@h
 	ori	r10, r10, 0x1c00
-	mr	r8, r10
-#else
-	lis	r10, MD_RESETVAL@h
-#endif
-#ifndef CONFIG_8xx_COPYBACK
-	oris	r10, r10, MD_WTDEF@h
-#endif
 	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
+#endif
 
 	/* Now map the lower 8 Meg into the TLBs.  For this quick hack,
 	 * we can load the instruction and data TLB registers with the
-- 
2.1.0

[PATCH v3 5/7] powerpc/8xx: Don't use page table for linear memory space

[Christophe Leroy]

Instead of using the first level page table to define mappings for
the linear memory space, we can use direct mapping from the TLB
handling routines. This has several advantages:
* No need to read the tables at each TLB miss
* No issue in 16k pages mode where the 1st level table maps 64 Mbytes

The size of the available linear space is known at system startup.
In order to avoid data access at each TLB miss to know the memory
size, the TLB routine is patched at startup with the proper size

This patch provides a 10%-15% improvment of TLB miss handling for
kernel addresses

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: No change

 arch/powerpc/kernel/head_8xx.S | 71 ++++++++++++++++++++++--------------------
 arch/powerpc/mm/8xx_mmu.c      | 56 +++++++++++----------------------
 2 files changed, 55 insertions(+), 72 deletions(-)

diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index d9a1656..3de7d02 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -389,52 +389,52 @@ InstructionTLBMiss:
  * not enough space in the DataStoreTLBMiss area
  */
 DTLBMissIMMR:
-	mtcr	r3
+	mtcr	r10
 	/* Set 512k byte guarded page and mark it valid */
 	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
-	MTSPR_CPU6(SPRN_MD_TWC, r10, r3)
+	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)
 	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
 	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
 	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
 			  _PAGE_PRESENT | _PAGE_NO_CACHE
-	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
+	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */
 
 	li	r11, RPN_PATTERN
-	mfspr	r3, SPRN_SPRG_SCRATCH2
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
 	EXCEPTION_EPILOG_0
 	rfi
 
 	. = 0x1200
 DataStoreTLBMiss:
-	mtspr	SPRN_SPRG_SCRATCH2, r3
 	EXCEPTION_PROLOG_0
-	mfcr	r3
+	mfcr	r10
 
 	/* If we are faulting a kernel address, we have to use the
 	 * kernel page tables.
 	 */
-	mfspr	r10, SPRN_MD_EPN
-	IS_KERNEL(r11, r10)
-	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
-	BRANCH_UNLESS_KERNEL(3f)
-
-	rlwinm	r11, r10, 16, 0xfff8
+	mfspr	r11, SPRN_MD_EPN
+	rlwinm	r11, r11, 16, 0xfff8
 #ifndef CONFIG_PIN_TLB
 	cmpli	cr0, r11, VIRT_IMMR_BASE@h
+#endif
+	cmpli	cr7, r11, PAGE_OFFSET@h
+#ifndef CONFIG_PIN_TLB
 _ENTRY(DTLBMiss_jmp)
 	beq-	DTLBMissIMMR
 #endif
+	bge-	cr7, 4f
 
-	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
+	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
 3:
+	mtcr	r10
+#ifdef CONFIG_8xx_CPU6
+	mtspr	SPRN_SPRG_SCRATCH2, r3
+#endif
+	mfspr	r10, SPRN_MD_EPN
 
 	/* Insert level 1 index */
 	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
 	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
-	mtcr	r11
-	bt-	28,DTLBMiss8M		/* bit 28 = Large page (8M) */
-	mtcr	r3
 
 	/* We have a pte table, so load fetch the pte from the table.
 	 */
@@ -482,29 +482,30 @@ _ENTRY(DTLBMiss_jmp)
 	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
 
 	/* Restore registers */
+#ifdef CONFIG_8xx_CPU6
 	mfspr	r3, SPRN_SPRG_SCRATCH2
+#endif
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
 	EXCEPTION_EPILOG_0
 	rfi
 
-DTLBMiss8M:
-	mtcr	r3
-	ori	r11, r11, MD_SVALID
-	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
-#ifdef CONFIG_PPC_16K_PAGES
-	/*
-	 * In 16k pages mode, each PGD entry defines a 64M block.
-	 * Here we select the 8M page within the block.
-	 */
-	rlwimi	r11, r10, 0, 0x03800000
-#endif
-	rlwinm	r10, r11, 0, 0xff800000
+4:
+_ENTRY(DTLBMiss_cmp)
+	cmpli	cr0, r11, PAGE_OFFSET@h
+	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
+	bge-	3b
+
+	mtcr	r10
+	/* Set 8M byte page and mark it valid */
+	li	r10, MD_PS8MEG | MD_SVALID
+	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)
+	mfspr	r10, SPRN_MD_EPN
+	rlwinm	r10, r10, 0, 0x0f800000		/* 8xx supports max 256Mb RAM */
 	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
 			  _PAGE_PRESENT
-	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
+	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */
 
 	li	r11, RPN_PATTERN
-	mfspr	r3, SPRN_SPRG_SCRATCH2
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
 	EXCEPTION_EPILOG_0
 	rfi
@@ -583,12 +584,14 @@ FixupDAR:/* Entry point for dcbx workaround. */
 	IS_KERNEL(r11, r10)
 	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
 	BRANCH_UNLESS_KERNEL(3f)
+	rlwinm	r11, r10, 16, 0xfff8
+_ENTRY(FixupDAR_cmp)
+	cmpli	cr7, r11, PAGE_OFFSET@h
+	blt-	cr7, 200f
 	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
 	/* Insert level 1 index */
 3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
 	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
-	mtcr	r11
-	bt	28,200f		/* bit 28 = Large page (8M) */
 	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */
 	/* Insert level 2 index */
 	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
@@ -614,8 +617,8 @@ FixupDAR:/* Entry point for dcbx workaround. */
 141:	mfspr	r10,SPRN_SPRG_SCRATCH2
 	b	DARFixed	/* Nope, go back to normal TLB processing */
 
-	/* concat physical page address(r11) and page offset(r10) */
-200:	rlwimi	r11, r10, 0, 32 - (PAGE_SHIFT << 1), 31
+	/* create physical page address from effective address */
+200:	tophys(r11, r10)
 	b	201b
 
 144:	mfspr	r10, SPRN_DSISR
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 2207725..996dfaa 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -58,9 +58,7 @@ void __init MMU_init_hw(void)
 	/* Nothing to do for the time being but keep it similar to other PPC */
 }
 
-#define LARGE_PAGE_SIZE_4M	(1<<22)
 #define LARGE_PAGE_SIZE_8M	(1<<23)
-#define LARGE_PAGE_SIZE_64M	(1<<26)
 
 static void mmu_mapin_immr(void)
 {
@@ -77,52 +75,33 @@ static void mmu_mapin_immr(void)
 #ifndef CONFIG_PIN_TLB
 extern unsigned int DTLBMiss_jmp;
 #endif
+extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
 
-unsigned long __init mmu_mapin_ram(unsigned long top)
+void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)
 {
-	unsigned long v, s, mapped;
-	phys_addr_t p;
+	unsigned int instr = *addr;
 
-	v = KERNELBASE;
-	p = 0;
-	s = top;
+	instr &= 0xffff0000;
+	instr |= (unsigned long)__va(mapped) >> 16;
+	patch_instruction(addr, instr);
+}
+
+unsigned long __init mmu_mapin_ram(unsigned long top)
+{
+	unsigned long mapped;
 
 	if (__map_without_ltlbs) {
+		mapped = 0;
 		mmu_mapin_immr();
 #ifndef CONFIG_PIN_TLB
 		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
 #endif
-		return 0;
-	}
-
-#ifdef CONFIG_PPC_4K_PAGES
-	while (s >= LARGE_PAGE_SIZE_8M) {
-		pmd_t *pmdp;
-		unsigned long val = p | MD_PS8MEG;
-
-		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);
-		*pmdp++ = __pmd(val);
-		*pmdp++ = __pmd(val + LARGE_PAGE_SIZE_4M);
-
-		v += LARGE_PAGE_SIZE_8M;
-		p += LARGE_PAGE_SIZE_8M;
-		s -= LARGE_PAGE_SIZE_8M;
+	} else {
+		mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);
 	}
-#else /* CONFIG_PPC_16K_PAGES */
-	while (s >= LARGE_PAGE_SIZE_64M) {
-		pmd_t *pmdp;
-		unsigned long val = p | MD_PS8MEG;
-
-		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);
-		*pmdp++ = __pmd(val);
-
-		v += LARGE_PAGE_SIZE_64M;
-		p += LARGE_PAGE_SIZE_64M;
-		s -= LARGE_PAGE_SIZE_64M;
-	}
-#endif
 
-	mapped = top - s;
+	mmu_patch_cmp_limit(&DTLBMiss_cmp, mapped);
+	mmu_patch_cmp_limit(&FixupDAR_cmp, mapped);
 
 	/* If the size of RAM is not an exact power of two, we may not
 	 * have covered RAM in its entirety with 8 MiB
@@ -131,7 +110,8 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
 	 * coverage with normal-sized pages (or other reasons) do not
 	 * attempt to allocate outside the allowed range.
 	 */
-	memblock_set_current_limit(mapped);
+	if (mapped)
+		memblock_set_current_limit(mapped);
 
 	return mapped;
 }
-- 
2.1.0

[PATCH v3 6/7] powerpc/8xx: Rework CONFIG_PIN_TLB handling

[Christophe Leroy]

On recent kernels, with some debug options like for instance
CONFIG_LOCKDEP, the BSS requires more than 8M memory, allthough
the kernel code fits in the first 8M.
Today, it is necessary to activate CONFIG_PIN_TLB to get more than 8M
at startup, allthough pinning TLB is not necessary for that.

We could have inconditionaly mapped 16 or 24M bytes at startup
but some old hardware only have 8M and mapping non-existing RAM
would be an issue due to speculative accesses.

With the preceding patch however, the TLB entries are populated on
demand. By setting up the TLB miss handler to handle up to 24M until
the handler is patched for the entire memory space, it is possible
to allow access up to more memory without mapping non-existing RAM.

It is therefore not needed anymore to map memory data at all
at startup. It will be handled by the TLB miss handler.

One might still want to PIN the IMMR and the first 24M of RAM.
It is now possible to do it in the C memory initialisation
functions. In addition, we now know how much memory we have
when we do it, so we are able to adapt the pining to the
real amount of memory available. So boards with less than 24M
can now also benefit from PIN_TLB.

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: Fixed a test
v3: No change

 arch/powerpc/kernel/head_8xx.S | 44 ++++--------------------------------------
 arch/powerpc/mm/8xx_mmu.c      | 27 ++++++++++++++++++--------
 2 files changed, 23 insertions(+), 48 deletions(-)

diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 3de7d02..00cc9df 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -491,7 +491,7 @@ _ENTRY(DTLBMiss_jmp)
 
 4:
 _ENTRY(DTLBMiss_cmp)
-	cmpli	cr0, r11, PAGE_OFFSET@h
+	cmpli	cr0, r11, (PAGE_OFFSET + 0x1800000)@h
 	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
 	bge-	3b
 
@@ -586,7 +586,7 @@ FixupDAR:/* Entry point for dcbx workaround. */
 	BRANCH_UNLESS_KERNEL(3f)
 	rlwinm	r11, r10, 16, 0xfff8
 _ENTRY(FixupDAR_cmp)
-	cmpli	cr7, r11, PAGE_OFFSET@h
+	cmpli	cr7, r11, (PAGE_OFFSET + 0x1800000)@h
 	blt-	cr7, 200f
 	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
 	/* Insert level 1 index */
@@ -823,23 +823,16 @@ initial_mmu:
 	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
 #endif
 
-	/* Now map the lower 8 Meg into the TLBs.  For this quick hack,
-	 * we can load the instruction and data TLB registers with the
-	 * same values.
-	 */
+	/* Now map the lower 8 Meg into the ITLB. */
 	lis	r8, KERNELBASE@h	/* Create vaddr for TLB */
 	ori	r8, r8, MI_EVALID	/* Mark it valid */
 	mtspr	SPRN_MI_EPN, r8
-	mtspr	SPRN_MD_EPN, r8
 	li	r8, MI_PS8MEG | (2 << 5)	/* Set 8M byte page, APG 2 */
 	ori	r8, r8, MI_SVALID	/* Make it valid */
 	mtspr	SPRN_MI_TWC, r8
-	li	r8, MI_PS8MEG		/* Set 8M byte page, APG 0 */
-	ori	r8, r8, MI_SVALID	/* Make it valid */
-	mtspr	SPRN_MD_TWC, r8
 	li	r8, MI_BOOTINIT		/* Create RPN for address 0 */
 	mtspr	SPRN_MI_RPN, r8		/* Store TLB entry */
-	mtspr	SPRN_MD_RPN, r8
+
 	lis	r8, MI_APG_INIT@h	/* Set protection modes */
 	ori	r8, r8, MI_APG_INIT@l
 	mtspr	SPRN_MI_AP, r8
@@ -851,9 +844,6 @@ initial_mmu:
 	 * internal registers (among other things).
 	 */
 #ifdef CONFIG_PIN_TLB
-	addi	r10, r10, 0x0100
-	mtspr	SPRN_MD_CTR, r10
-#endif
 	mfspr	r9, 638			/* Get current IMMR */
 	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
 
@@ -866,32 +856,6 @@ initial_mmu:
 	mr	r8, r9			/* Create paddr for TLB */
 	ori	r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
 	mtspr	SPRN_MD_RPN, r8
-
-#ifdef CONFIG_PIN_TLB
-	/* Map two more 8M kernel data pages.
-	*/
-	addi	r10, r10, 0x0100
-	mtspr	SPRN_MD_CTR, r10
-
-	lis	r8, KERNELBASE@h	/* Create vaddr for TLB */
-	addis	r8, r8, 0x0080		/* Add 8M */
-	ori	r8, r8, MI_EVALID	/* Mark it valid */
-	mtspr	SPRN_MD_EPN, r8
-	li	r9, MI_PS8MEG		/* Set 8M byte page */
-	ori	r9, r9, MI_SVALID	/* Make it valid */
-	mtspr	SPRN_MD_TWC, r9
-	li	r11, MI_BOOTINIT	/* Create RPN for address 0 */
-	addis	r11, r11, 0x0080	/* Add 8M */
-	mtspr	SPRN_MD_RPN, r11
-
-	addi	r10, r10, 0x0100
-	mtspr	SPRN_MD_CTR, r10
-
-	addis	r8, r8, 0x0080		/* Add 8M */
-	mtspr	SPRN_MD_EPN, r8
-	mtspr	SPRN_MD_TWC, r9
-	addis	r11, r11, 0x0080	/* Add 8M */
-	mtspr	SPRN_MD_RPN, r11
 #endif
 
 	/* Since the cache is enabled according to the information we
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 996dfaa..0f0a83e 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -50,16 +50,32 @@ unsigned long p_block_mapped(phys_addr_t pa)
 	return 0;
 }
 
+#define LARGE_PAGE_SIZE_8M	(1<<23)
+
 /*
  * MMU_init_hw does the chip-specific initialization of the MMU hardware.
  */
 void __init MMU_init_hw(void)
 {
-	/* Nothing to do for the time being but keep it similar to other PPC */
+	/* PIN up to the 3 first 8Mb after IMMR in DTLB table */
+#ifdef CONFIG_PIN_TLB
+	unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
+	unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
+	int i;
+	unsigned long addr = 0;
+	unsigned long mem = total_lowmem;
+
+	for (i = 29; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) {
+		mtspr(SPRN_MD_CTR, ctr | (i << 8));
+		mtspr(SPRN_MD_EPN, (unsigned long)__va(addr) | MD_EVALID);
+		mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID);
+		mtspr(SPRN_MD_RPN, addr | flags | _PAGE_PRESENT);
+		addr += LARGE_PAGE_SIZE_8M;
+		mem -= LARGE_PAGE_SIZE_8M;
+	}
+#endif
 }
 
-#define LARGE_PAGE_SIZE_8M	(1<<23)
-
 static void mmu_mapin_immr(void)
 {
 	unsigned long p = PHYS_IMMR_BASE;
@@ -124,13 +140,8 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base,
 	 */
 	BUG_ON(first_memblock_base != 0);
 
-#ifdef CONFIG_PIN_TLB
 	/* 8xx can only access 24MB at the moment */
 	memblock_set_current_limit(min_t(u64, first_memblock_size, 0x01800000));
-#else
-	/* 8xx can only access 8MB at the moment */
-	memblock_set_current_limit(min_t(u64, first_memblock_size, 0x00800000));
-#endif
 }
 
 /*
-- 
2.1.0

[PATCH v3 7/7] powerpc/8xx: add CONFIG_PIN_TLB_IMMR

[Christophe Leroy]

CONFIG_PIN_TLB maps IMMR area and the first 24 Mbytes of memory.
In some circunstances it might be more interesting to not map
IMMR but map 32 Mbytes of memory instead.

Therefore we add config option CONFIG_PIN_TLB_IMMR to select if
IMMR shall be pinned or not, hence whether we pin 24 or 32 Mbytes of RAM

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2: No change
v3: No change

 arch/powerpc/Kconfig           |  5 +++++
 arch/powerpc/kernel/head_8xx.S | 10 ++++++----
 arch/powerpc/mm/8xx_mmu.c      | 12 ++++++++----
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index ffca388..5b3a1ee 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -1055,6 +1055,11 @@ config CONSISTENT_SIZE
 config PIN_TLB
 	bool "Pinned Kernel TLBs (860 ONLY)"
 	depends on ADVANCED_OPTIONS && 8xx
+
+config PIN_TLB_IMMR
+	bool "Pinned TLB for IMMR"
+	depends on PIN_TLB
+	default y
 endmenu
 
 if PPC64
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 00cc9df..43ddaae 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -414,11 +414,11 @@ DataStoreTLBMiss:
 	 */
 	mfspr	r11, SPRN_MD_EPN
 	rlwinm	r11, r11, 16, 0xfff8
-#ifndef CONFIG_PIN_TLB
+#ifndef CONFIG_PIN_TLB_IMMR
 	cmpli	cr0, r11, VIRT_IMMR_BASE@h
 #endif
 	cmpli	cr7, r11, PAGE_OFFSET@h
-#ifndef CONFIG_PIN_TLB
+#ifndef CONFIG_PIN_TLB_IMMR
 _ENTRY(DTLBMiss_jmp)
 	beq-	DTLBMissIMMR
 #endif
@@ -819,7 +819,6 @@ initial_mmu:
 
 #ifdef CONFIG_PIN_TLB
 	oris	r10, r10, MD_RSV4I@h
-	ori	r10, r10, 0x1c00
 	mtspr	SPRN_MD_CTR, r10	/* Set data TLB control */
 #endif
 
@@ -843,7 +842,10 @@ initial_mmu:
 	/* Map a 512k page for the IMMR to get the processor
 	 * internal registers (among other things).
 	 */
-#ifdef CONFIG_PIN_TLB
+#ifdef CONFIG_PIN_TLB_IMMR
+	ori	r10, r10, 0x1c00
+	mtspr	SPRN_MD_CTR, r10
+
 	mfspr	r9, 638			/* Get current IMMR */
 	andis.	r9, r9, 0xfff8		/* Get 512 kbytes boundary */
 
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 0f0a83e..6c5025e 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -61,11 +61,15 @@ void __init MMU_init_hw(void)
 #ifdef CONFIG_PIN_TLB
 	unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
 	unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
-	int i;
+#ifdef CONFIG_PIN_TLB_IMMR
+	int i = 29;
+#else
+	int i = 28;
+#endif
 	unsigned long addr = 0;
 	unsigned long mem = total_lowmem;
 
-	for (i = 29; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) {
+	for (; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) {
 		mtspr(SPRN_MD_CTR, ctr | (i << 8));
 		mtspr(SPRN_MD_EPN, (unsigned long)__va(addr) | MD_EVALID);
 		mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID);
@@ -88,7 +92,7 @@ static void mmu_mapin_immr(void)
 }
 
 /* Address of instructions to patch */
-#ifndef CONFIG_PIN_TLB
+#ifndef CONFIG_PIN_TLB_IMMR
 extern unsigned int DTLBMiss_jmp;
 #endif
 extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
@@ -109,7 +113,7 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
 	if (__map_without_ltlbs) {
 		mapped = 0;
 		mmu_mapin_immr();
-#ifndef CONFIG_PIN_TLB
+#ifndef CONFIG_PIN_TLB_IMMR
 		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
 #endif
 	} else {
-- 
2.1.0