* [PATCH 06/10] powerpc/mm/radix: implement LPID based TLB flushes to be used by KVM
[not found] <20180506073731.13097-1-npiggin@gmail.com>
@ 2018-05-06 7:37 ` Nicholas Piggin
2018-05-07 5:15 ` Paul Mackerras
0 siblings, 1 reply; 3+ messages in thread
From: Nicholas Piggin @ 2018-05-06 7:37 UTC (permalink / raw)
To: kvm-ppc; +Cc: Nicholas Piggin, linuxppc-dev
Implement a local TLB flush for invalidating an LPID with variants for
process or partition scope. And a global TLB flush for invalidating
a partition scoped page of an LPID.
These will be used by KVM in subsequent patches.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
.../include/asm/book3s/64/tlbflush-radix.h | 7 +
arch/powerpc/mm/tlb-radix.c | 207 ++++++++++++++++++
2 files changed, 214 insertions(+)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 19b45ba6caf9..ef5c3f2994c9 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -51,4 +51,11 @@ extern void radix__flush_tlb_all(void);
extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
unsigned long address);
+extern void radix__flush_tlb_lpid_page(unsigned int lpid,
+ unsigned long addr,
+ unsigned long page_size);
+extern void radix__flush_pwc_lpid(unsigned int lpid);
+extern void radix__local_flush_tlb_lpid(unsigned int lpid);
+extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
+
#endif
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index a5d7309c2d05..5ac3206c51cc 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -118,6 +118,53 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
+static inline void __tlbiel_lpid(unsigned long lpid, int set,
+ unsigned long ric)
+{
+ unsigned long rb,rs,prs,r;
+
+ rb = PPC_BIT(52); /* IS = 2 */
+ rb |= set << PPC_BITLSHIFT(51);
+ rs = 0; /* LPID comes from LPIDR */
+ prs = 0; /* partition scoped */
+ r = 1; /* radix format */
+
+ asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+ trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
+}
+
+static inline void __tlbie_lpid(unsigned long lpid, unsigned long ric)
+{
+ unsigned long rb,rs,prs,r;
+
+ rb = PPC_BIT(52); /* IS = 2 */
+ rs = lpid;
+ prs = 0; /* partition scoped */
+ r = 1; /* radix format */
+
+ asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+ trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
+}
+
+static inline void __tlbiel_lpid_guest(unsigned long lpid, int set,
+ unsigned long ric)
+{
+ unsigned long rb,rs,prs,r;
+
+ rb = PPC_BIT(52); /* IS = 2 */
+ rb |= set << PPC_BITLSHIFT(51);
+ rs = 0; /* LPID comes from LPIDR */
+ prs = 1; /* process scoped */
+ r = 1; /* radix format */
+
+ asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+ trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
+}
+
+
static inline void __tlbiel_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric)
{
@@ -150,6 +197,22 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
+static inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid,
+ unsigned long ap, unsigned long ric)
+{
+ unsigned long rb,rs,prs,r;
+
+ rb = va & ~(PPC_BITMASK(52, 63));
+ rb |= ap << PPC_BITLSHIFT(58);
+ rs = lpid;
+ prs = 0; /* partition scoped */
+ r = 1; /* radix format */
+
+ asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+ trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
+}
+
static inline void fixup_tlbie(void)
{
unsigned long pid = 0;
@@ -161,6 +224,16 @@ static inline void fixup_tlbie(void)
}
}
+static inline void fixup_tlbie_lpid(unsigned long lpid)
+{
+ unsigned long va = ((1UL << 52) - 1);
+
+ if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+ asm volatile("ptesync": : :"memory");
+ __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
+ }
+}
+
/*
* We use 128 set in radix mode and 256 set in hpt mode.
*/
@@ -214,6 +287,86 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
+static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric)
+{
+ int set;
+
+ VM_BUG_ON(mfspr(SPRN_LPID) != lpid);
+
+ asm volatile("ptesync": : :"memory");
+
+ /*
+ * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
+ * also flush the entire Page Walk Cache.
+ */
+ __tlbiel_lpid(lpid, 0, ric);
+
+ /* For PWC, only one flush is needed */
+ if (ric == RIC_FLUSH_PWC) {
+ asm volatile("ptesync": : :"memory");
+ return;
+ }
+
+ /* For the remaining sets, just flush the TLB */
+ for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
+ __tlbiel_lpid(lpid, set, RIC_FLUSH_TLB);
+
+ asm volatile("ptesync": : :"memory");
+ asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
+}
+
+static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
+{
+ asm volatile("ptesync": : :"memory");
+
+ /*
+ * Workaround the fact that the "ric" argument to __tlbie_pid
+ * must be a compile-time contraint to match the "i" constraint
+ * in the asm statement.
+ */
+ switch (ric) {
+ case RIC_FLUSH_TLB:
+ __tlbie_lpid(lpid, RIC_FLUSH_TLB);
+ break;
+ case RIC_FLUSH_PWC:
+ __tlbie_lpid(lpid, RIC_FLUSH_PWC);
+ break;
+ case RIC_FLUSH_ALL:
+ default:
+ __tlbie_lpid(lpid, RIC_FLUSH_ALL);
+ }
+ fixup_tlbie_lpid(lpid);
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
+
+static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric)
+{
+ int set;
+
+ VM_BUG_ON(mfspr(SPRN_LPID) != lpid);
+
+ asm volatile("ptesync": : :"memory");
+
+ /*
+ * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
+ * also flush the entire Page Walk Cache.
+ */
+ __tlbiel_lpid_guest(lpid, 0, ric);
+
+ /* For PWC, only one flush is needed */
+ if (ric == RIC_FLUSH_PWC) {
+ asm volatile("ptesync": : :"memory");
+ return;
+ }
+
+ /* For the remaining sets, just flush the TLB */
+ for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
+ __tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB);
+
+ asm volatile("ptesync": : :"memory");
+}
+
+
static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize)
@@ -268,6 +421,17 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
+static inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid,
+ unsigned long psize, unsigned long ric)
+{
+ unsigned long ap = mmu_get_ap(psize);
+
+ asm volatile("ptesync": : :"memory");
+ __tlbie_lpid_va(va, lpid, ap, ric);
+ fixup_tlbie_lpid(lpid);
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
+
static inline void _tlbie_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize, bool also_pwc)
@@ -534,6 +698,49 @@ static int radix_get_mmu_psize(int page_size)
return psize;
}
+/*
+ * Flush partition scoped LPID address translation for all CPUs.
+ */
+void radix__flush_tlb_lpid_page(unsigned int lpid,
+ unsigned long addr,
+ unsigned long page_size)
+{
+ int psize = radix_get_mmu_psize(page_size);
+
+ _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB);
+}
+EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page);
+
+/*
+ * Flush partition scoped PWC from LPID for all CPUs.
+ */
+void radix__flush_pwc_lpid(unsigned int lpid)
+{
+ _tlbie_lpid(lpid, RIC_FLUSH_PWC);
+}
+EXPORT_SYMBOL_GPL(radix__flush_pwc_lpid);
+
+/*
+ * Flush partition scoped translations from LPID (=LPIDR)
+ */
+void radix__local_flush_tlb_lpid(unsigned int lpid)
+{
+ _tlbiel_lpid(lpid, RIC_FLUSH_ALL);
+}
+EXPORT_SYMBOL_GPL(radix__local_flush_tlb_lpid);
+
+/*
+ * Flush process scoped translations from LPID (=LPIDR).
+ * Important difference, the guest normally manages its own translations,
+ * but some cases e.g., vCPU CPU migration require KVM to flush.
+ */
+void radix__local_flush_tlb_lpid_guest(unsigned int lpid)
+{
+ _tlbiel_lpid_guest(lpid, RIC_FLUSH_ALL);
+}
+EXPORT_SYMBOL_GPL(radix__local_flush_tlb_lpid_guest);
+
+
static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long end, int psize);
--
2.17.0
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH 06/10] powerpc/mm/radix: implement LPID based TLB flushes to be used by KVM
2018-05-06 7:37 ` [PATCH 06/10] powerpc/mm/radix: implement LPID based TLB flushes to be used by KVM Nicholas Piggin
@ 2018-05-07 5:15 ` Paul Mackerras
2018-05-07 9:38 ` Nicholas Piggin
0 siblings, 1 reply; 3+ messages in thread
From: Paul Mackerras @ 2018-05-07 5:15 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: kvm-ppc, linuxppc-dev
On Sun, May 06, 2018 at 05:37:27PM +1000, Nicholas Piggin wrote:
> Implement a local TLB flush for invalidating an LPID with variants for
> process or partition scope. And a global TLB flush for invalidating
> a partition scoped page of an LPID.
>
> These will be used by KVM in subsequent patches.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
> .../include/asm/book3s/64/tlbflush-radix.h | 7 +
> arch/powerpc/mm/tlb-radix.c | 207 ++++++++++++++++++
> 2 files changed, 214 insertions(+)
>
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index 19b45ba6caf9..ef5c3f2994c9 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -51,4 +51,11 @@ extern void radix__flush_tlb_all(void);
> extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
> unsigned long address);
>
> +extern void radix__flush_tlb_lpid_page(unsigned int lpid,
> + unsigned long addr,
> + unsigned long page_size);
> +extern void radix__flush_pwc_lpid(unsigned int lpid);
> +extern void radix__local_flush_tlb_lpid(unsigned int lpid);
> +extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
> +
> #endif
> diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> index a5d7309c2d05..5ac3206c51cc 100644
> --- a/arch/powerpc/mm/tlb-radix.c
> +++ b/arch/powerpc/mm/tlb-radix.c
> @@ -118,6 +118,53 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
> trace_tlbie(0, 0, rb, rs, ric, prs, r);
> }
>
> +static inline void __tlbiel_lpid(unsigned long lpid, int set,
> + unsigned long ric)
> +{
> + unsigned long rb,rs,prs,r;
> +
> + rb = PPC_BIT(52); /* IS = 2 */
> + rb |= set << PPC_BITLSHIFT(51);
> + rs = 0; /* LPID comes from LPIDR */
> + prs = 0; /* partition scoped */
> + r = 1; /* radix format */
> +
> + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
> + : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
> + trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
Do we really want 128 trace entries every time
radix__local_flush_tlb_lpid() or radix__local_flush_tlb_lpid_guest()
is called? That seems like overkill to me. Could we move the
trace_tlbie calls here and in __tlbiel_lpid_guest() into the callers
instead?
Paul.
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [PATCH 06/10] powerpc/mm/radix: implement LPID based TLB flushes to be used by KVM
2018-05-07 5:15 ` Paul Mackerras
@ 2018-05-07 9:38 ` Nicholas Piggin
0 siblings, 0 replies; 3+ messages in thread
From: Nicholas Piggin @ 2018-05-07 9:38 UTC (permalink / raw)
To: Paul Mackerras; +Cc: kvm-ppc, linuxppc-dev, Balbir Singh
On Mon, 7 May 2018 15:15:26 +1000
Paul Mackerras <paulus@ozlabs.org> wrote:
> On Sun, May 06, 2018 at 05:37:27PM +1000, Nicholas Piggin wrote:
> > Implement a local TLB flush for invalidating an LPID with variants for
> > process or partition scope. And a global TLB flush for invalidating
> > a partition scoped page of an LPID.
> >
> > These will be used by KVM in subsequent patches.
> >
> > Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> > ---
> > .../include/asm/book3s/64/tlbflush-radix.h | 7 +
> > arch/powerpc/mm/tlb-radix.c | 207 ++++++++++++++++++
> > 2 files changed, 214 insertions(+)
> >
> > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > index 19b45ba6caf9..ef5c3f2994c9 100644
> > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > @@ -51,4 +51,11 @@ extern void radix__flush_tlb_all(void);
> > extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
> > unsigned long address);
> >
> > +extern void radix__flush_tlb_lpid_page(unsigned int lpid,
> > + unsigned long addr,
> > + unsigned long page_size);
> > +extern void radix__flush_pwc_lpid(unsigned int lpid);
> > +extern void radix__local_flush_tlb_lpid(unsigned int lpid);
> > +extern void radix__local_flush_tlb_lpid_guest(unsigned int lpid);
> > +
> > #endif
> > diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> > index a5d7309c2d05..5ac3206c51cc 100644
> > --- a/arch/powerpc/mm/tlb-radix.c
> > +++ b/arch/powerpc/mm/tlb-radix.c
> > @@ -118,6 +118,53 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
> > trace_tlbie(0, 0, rb, rs, ric, prs, r);
> > }
> >
> > +static inline void __tlbiel_lpid(unsigned long lpid, int set,
> > + unsigned long ric)
> > +{
> > + unsigned long rb,rs,prs,r;
> > +
> > + rb = PPC_BIT(52); /* IS = 2 */
> > + rb |= set << PPC_BITLSHIFT(51);
> > + rs = 0; /* LPID comes from LPIDR */
> > + prs = 0; /* partition scoped */
> > + r = 1; /* radix format */
> > +
> > + asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
> > + : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
> > + trace_tlbie(lpid, 1, rb, rs, ric, prs, r);
>
> Do we really want 128 trace entries every time
> radix__local_flush_tlb_lpid() or radix__local_flush_tlb_lpid_guest()
> is called? That seems like overkill to me. Could we move the
> trace_tlbie calls here and in __tlbiel_lpid_guest() into the callers
> instead?
Yeah well that's how the powerpc:tlbie tracepoints are defined. I can
see possible benefit to feeding the trace directly to a model or
simulator, but it does make other things harder.
I have a patch around that changes all those tracepoints around to
match the Linux semantics rather than the instructions... If people
would prefer that I can send it (cc'ed Balbir).
But for this patch we should just match the existing tracepoints.
Thanks,
Nick
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2018-05-07 9:39 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20180506073731.13097-1-npiggin@gmail.com>
2018-05-06 7:37 ` [PATCH 06/10] powerpc/mm/radix: implement LPID based TLB flushes to be used by KVM Nicholas Piggin
2018-05-07 5:15 ` Paul Mackerras
2018-05-07 9:38 ` Nicholas Piggin
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).