[PATCH 0/3] Allow processors with scratch registers to use them for TLB refill.

[PATCH 0/3] Allow processors with scratch registers to use them for TLB refill.

[David Daney]

The MIPS32r2 and MIPS64r2 specifications allow processors to have
scratch registers in coprocessor 0.  If these are present, we can use
one of them to carry the current PGD and save three instructions in
the TLB handlers.

There are three patches:

1 - Probe for presence of scratch registers an print number found in
    /proc/cpuinfo.

2 - Add DINSM to uasm for use by patch 3.

3 - Convert the TLB handlers.  This also involves dynamically
    generating tlbmiss_handler_setup_pgd, which used to be statically
    defined.


David Daney (3):
  MIPS: Probe for presence of KScratch registers.
  MIPS: Add DINSM to uasm.
  MIPS: Use C0_KScratch (if present) to hold PGD pointer.

 arch/mips/include/asm/cpu-info.h    |    1 +
 arch/mips/include/asm/mmu_context.h |    8 +--
 arch/mips/include/asm/uasm.h        |    1 +
 arch/mips/kernel/cpu-probe.c        |    2 +
 arch/mips/kernel/proc.c             |    2 +
 arch/mips/kernel/traps.c            |    2 +-
 arch/mips/mm/tlbex.c                |  110 +++++++++++++++++++++++++++++++---
 arch/mips/mm/uasm.c                 |   11 +++-
 8 files changed, 118 insertions(+), 19 deletions(-)

-- 
1.7.2.3

[PATCH 1/3] MIPS: Probe for presence of KScratch registers.

[David Daney]

Probe c0_config4 for KScratch registers and report them in
/proc/cpuinfo.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/include/asm/cpu-info.h |    1 +
 arch/mips/kernel/cpu-probe.c     |    2 ++
 arch/mips/kernel/proc.c          |    2 ++
 3 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h
index b39def3..c454550 100644
--- a/arch/mips/include/asm/cpu-info.h
+++ b/arch/mips/include/asm/cpu-info.h
@@ -78,6 +78,7 @@ struct cpuinfo_mips {
 	unsigned int		watch_reg_use_cnt; /* Usable by ptrace */
 #define NUM_WATCH_REGS 4
 	u16			watch_reg_masks[NUM_WATCH_REGS];
+	unsigned int		kscratch_mask; /* Usable KScratch mask. */
 } __attribute__((aligned(SMP_CACHE_BYTES)));
 
 extern struct cpuinfo_mips cpu_data[];
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
index 68dae7b..f65d4c8 100644
--- a/arch/mips/kernel/cpu-probe.c
+++ b/arch/mips/kernel/cpu-probe.c
@@ -739,6 +739,8 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c)
 	    && cpu_has_tlb)
 		c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40;
 
+	c->kscratch_mask = (config4 >> 16) & 0xff;
+
 	return config4 & MIPS_CONF_M;
 }
 
diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c
index 26109c4..f40bd6b 100644
--- a/arch/mips/kernel/proc.c
+++ b/arch/mips/kernel/proc.c
@@ -69,6 +69,8 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 		);
 	seq_printf(m, "shadow register sets\t: %d\n",
 		       cpu_data[n].srsets);
+	seq_printf(m, "kscratch registers\t: %d\n",
+		   hweight8(cpu_data[n].kscratch_mask));
 	seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core);
 
 	sprintf(fmt, "VCE%%c exceptions\t\t: %s\n",
-- 
1.7.2.3

[PATCH 2/3] MIPS: Add DINSM to uasm.

[David Daney]

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/include/asm/uasm.h |    1 +
 arch/mips/mm/uasm.c          |   11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletions(-)

diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h
index 892062d..99dae68 100644
--- a/arch/mips/include/asm/uasm.h
+++ b/arch/mips/include/asm/uasm.h
@@ -115,6 +115,7 @@ Ip_0(_tlbwr);
 Ip_u3u1u2(_xor);
 Ip_u2u1u3(_xori);
 Ip_u2u1msbu3(_dins);
+Ip_u2u1msbu3(_dinsm);
 Ip_u1(_syscall);
 
 /* Handle labels. */
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 23afdeb..99f0347 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -68,7 +68,7 @@ enum opcode {
 	insn_pref, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
 	insn_sra, insn_srl, insn_rotr, insn_subu, insn_sw, insn_tlbp,
 	insn_tlbr, insn_tlbwi, insn_tlbwr, insn_xor, insn_xori,
-	insn_dins, insn_syscall, insn_bbit0, insn_bbit1
+	insn_dins, insn_dinsm, insn_syscall, insn_bbit0, insn_bbit1
 };
 
 struct insn {
@@ -142,6 +142,7 @@ static struct insn insn_table[] __uasminitdata = {
 	{ insn_xor,  M(spec_op, 0, 0, 0, 0, xor_op),  RS | RT | RD },
 	{ insn_xori,  M(xori_op, 0, 0, 0, 0, 0),  RS | RT | UIMM },
 	{ insn_dins, M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE },
+	{ insn_dinsm, M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE },
 	{ insn_syscall, M(spec_op, 0, 0, 0, 0, syscall_op), SCIMM},
 	{ insn_bbit0, M(lwc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
 	{ insn_bbit1, M(swc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
@@ -340,6 +341,13 @@ Ip_u2u1msbu3(op)					\
 }							\
 UASM_EXPORT_SYMBOL(uasm_i##op);
 
+#define I_u2u1msb32u3(op)				\
+Ip_u2u1msbu3(op)					\
+{							\
+	build_insn(buf, insn##op, b, a, c+d-33, c);	\
+}							\
+UASM_EXPORT_SYMBOL(uasm_i##op);
+
 #define I_u1u2(op)					\
 Ip_u1u2(op)						\
 {							\
@@ -422,6 +430,7 @@ I_0(_tlbwr)
 I_u3u1u2(_xor)
 I_u2u1u3(_xori)
 I_u2u1msbu3(_dins);
+I_u2u1msb32u3(_dinsm);
 I_u1(_syscall);
 I_u1u2s3(_bbit0);
 I_u1u2s3(_bbit1);
-- 
1.7.2.3

[PATCH 3/3] MIPS: Use C0_KScratch (if present) to hold PGD pointer.

[David Daney]

Decide at runtime to use either Context or KScratch to hold the PGD
pointer.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
---
 arch/mips/include/asm/mmu_context.h |    8 +--
 arch/mips/kernel/traps.c            |    2 +-
 arch/mips/mm/tlbex.c                |  110 +++++++++++++++++++++++++++++++---
 3 files changed, 102 insertions(+), 18 deletions(-)

diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h
index d959273..73c0d45 100644
--- a/arch/mips/include/asm/mmu_context.h
+++ b/arch/mips/include/asm/mmu_context.h
@@ -29,13 +29,7 @@
 #define TLBMISS_HANDLER_SETUP_PGD(pgd)				\
 	tlbmiss_handler_setup_pgd((unsigned long)(pgd))
 
-static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
-{
-	/* Check for swapper_pg_dir and convert to physical address. */
-	if ((pgd & CKSEG3) == CKSEG0)
-		pgd = CPHYSADDR(pgd);
-	write_c0_context(pgd << 11);
-}
+extern void tlbmiss_handler_setup_pgd(unsigned long pgd);
 
 #define TLBMISS_HANDLER_SETUP()						\
 	do {								\
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index d42e267..0e64aec 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -1631,7 +1631,6 @@ void __cpuinit per_cpu_trap_init(void)
 #endif /* CONFIG_MIPS_MT_SMTC */
 
 	cpu_data[cpu].asid_cache = ASID_FIRST_VERSION;
-	TLBMISS_HANDLER_SETUP();
 
 	atomic_inc(&init_mm.mm_count);
 	current->active_mm = &init_mm;
@@ -1653,6 +1652,7 @@ void __cpuinit per_cpu_trap_init(void)
 		write_c0_wired(0);
 	}
 #endif /* CONFIG_MIPS_MT_SMTC */
+	TLBMISS_HANDLER_SETUP();
 }
 
 /* Install CPU exception handler */
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 6184f0a..2e15aa6 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -26,8 +26,10 @@
 #include <linux/smp.h>
 #include <linux/string.h>
 #include <linux/init.h>
+#include <linux/cache.h>
 
-#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
 #include <asm/war.h>
 #include <asm/uasm.h>
 
@@ -173,7 +175,30 @@ static struct uasm_reloc relocs[128] __cpuinitdata;
 static int check_for_high_segbits __cpuinitdata;
 #endif
 
-#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+
+static unsigned int kscratch_used_mask __cpuinitdata;
+
+static int __cpuinit allocate_kscratch(void)
+{
+	int r;
+	unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
+
+	r = ffs(a);
+
+	if (r == 0)
+		return -1;
+
+	r--; /* make it zero based */
+
+	kscratch_used_mask |= (1 << r);
+
+	return r;
+}
+
+static int pgd_reg __cpuinitdata;
+
+#else /* !CONFIG_MIPS_PGD_C0_CONTEXT*/
 /*
  * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
  * we cannot do r3000 under these circumstances.
@@ -573,13 +598,22 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 	/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 
 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
-	/*
-	 * &pgd << 11 stored in CONTEXT [23..63].
-	 */
-	UASM_i_MFC0(p, ptr, C0_CONTEXT);
-	uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
-	uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0  1 0 1  << 6  xkphys cached */
-	uasm_i_drotr(p, ptr, ptr, 11);
+	if (pgd_reg != -1) {
+		/* pgd is in pgd_reg */
+		UASM_i_MFC0(p, ptr, 31, pgd_reg);
+	} else {
+		/*
+		 * &pgd << 11 stored in CONTEXT [23..63].
+		 */
+		UASM_i_MFC0(p, ptr, C0_CONTEXT);
+
+		/* Clear lower 23 bits of context. */
+		uasm_i_dins(p, ptr, 0, 0, 23);
+
+		/* 1 0  1 0 1  << 6  xkphys cached */
+		uasm_i_ori(p, ptr, ptr, 0x540);
+		uasm_i_drotr(p, ptr, ptr, 11);
+	}
 #elif defined(CONFIG_SMP)
 # ifdef  CONFIG_MIPS_MT_SMTC
 	/*
@@ -1017,6 +1051,55 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
+
+static void __cpuinit build_r4000_setup_pgd(void)
+{
+	const int a0 = 4;
+	const int a1 = 5;
+	u32 *p = tlbmiss_handler_setup_pgd;
+	struct uasm_label *l = labels;
+	struct uasm_reloc *r = relocs;
+
+	memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
+	memset(labels, 0, sizeof(labels));
+	memset(relocs, 0, sizeof(relocs));
+
+	pgd_reg = allocate_kscratch();
+
+	if (pgd_reg == -1) {
+		/* PGD << 11 in c0_Context */
+		/*
+		 * If it is a ckseg0 address, convert to a physical
+		 * address.  Shifting right by 29 and adding 4 will
+		 * result in zero for these addresses.
+		 *
+		 */
+		UASM_i_SRA(&p, a1, a0, 29);
+		UASM_i_ADDIU(&p, a1, a1, 4);
+		uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
+		uasm_i_nop(&p);
+		uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
+		uasm_l_tlbl_goaround1(&l, p);
+		UASM_i_SLL(&p, a0, a0, 11);
+		uasm_i_jr(&p, 31);
+		UASM_i_MTC0(&p, a0, C0_CONTEXT);
+	} else {
+		/* PGD in c0_KScratch */
+		uasm_i_jr(&p, 31);
+		UASM_i_MTC0(&p, a0, 31, pgd_reg);
+	}
+	if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
+		panic("tlbmiss_handler_setup_pgd space exceeded");
+	uasm_resolve_relocs(relocs, labels);
+	pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
+		 (unsigned int)(p - tlbmiss_handler_setup_pgd));
+
+	dump_handler(tlbmiss_handler_setup_pgd,
+		     ARRAY_SIZE(tlbmiss_handler_setup_pgd));
+}
+#endif
 
 static void __cpuinit
 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
@@ -1629,13 +1712,16 @@ void __cpuinit build_tlb_refill_handler(void)
 		break;
 
 	default:
-		build_r4000_tlb_refill_handler();
 		if (!run_once) {
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+			build_r4000_setup_pgd();
+#endif
 			build_r4000_tlb_load_handler();
 			build_r4000_tlb_store_handler();
 			build_r4000_tlb_modify_handler();
 			run_once++;
 		}
+		build_r4000_tlb_refill_handler();
 	}
 }
 
@@ -1647,4 +1733,8 @@ void __cpuinit flush_tlb_handlers(void)
 			   (unsigned long)handle_tlbs + sizeof(handle_tlbs));
 	local_flush_icache_range((unsigned long)handle_tlbm,
 			   (unsigned long)handle_tlbm + sizeof(handle_tlbm));
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+	local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
+			   (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
+#endif
 }
-- 
1.7.2.3

Re: [PATCH 3/3] MIPS: Use C0_KScratch (if present) to hold PGD pointer.

[David Daney]

On 12/20/2010 01:17 PM, David Daney wrote:
> Decide at runtime to use either Context or KScratch to hold the PGD
> pointer.
>
> Signed-off-by: David Daney<ddaney@caviumnetworks.com>
> ---
>   arch/mips/include/asm/mmu_context.h |    8 +--
>   arch/mips/kernel/traps.c            |    2 +-
>   arch/mips/mm/tlbex.c                |  110 +++++++++++++++++++++++++++++++---
>   3 files changed, 102 insertions(+), 18 deletions(-)
>
[...]
>   /* Install CPU exception handler */
> diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
> index 6184f0a..2e15aa6 100644
> --- a/arch/mips/mm/tlbex.c
> +++ b/arch/mips/mm/tlbex.c
> @@ -26,8 +26,10 @@
>   #include<linux/smp.h>
>   #include<linux/string.h>
>   #include<linux/init.h>
> +#include<linux/cache.h>
>
> -#include<asm/mmu_context.h>

Whoops, that include should stay, it is needed for ip32.

> +#include<asm/cacheflush.h>
> +#include<asm/pgtable.h>
>   #include<asm/war.h>
>   #include<asm/uasm.h>
>[...]

I will send a revised patch.