From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Nicholas Piggin <npiggin@gmail.com>
Subject: [PATCH 3/7] powerpc/64s/radix: Check for no TLB flush required
Date: Thu, 17 Dec 2020 23:47:27 +1000 [thread overview]
Message-ID: <20201217134731.488135-4-npiggin@gmail.com> (raw)
In-Reply-To: <20201217134731.488135-1-npiggin@gmail.com>
If there are no CPUs in mm_cpumask, no TLB flush is required at all.
This patch adds a check for this case.
Currently it's not tested for, in fact mm_is_thread_local() returns
false if the current CPU is not in mm_cpumask, so it's treated as a
global flush.
This can come up in some cases like exec failure before the new mm has
ever been switched to. This patch reduces TLBIE instructions required
to build a kernel from about 120,000 to 45,000. Another situation it
could help is page reclaim, KSM, THP, etc., (i.e., asynch operations
external to the process) where the process is sleeping and has all TLBs
flushed out of all CPUs.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/book3s64/radix_tlb.c | 38 ++++++++++++++++++----------
1 file changed, 25 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 12481c864ab6..5b62e2e7371c 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -694,13 +694,19 @@ static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { }
#endif /* CONFIG_SMP */
enum tlb_flush_type {
+ FLUSH_TYPE_NONE,
FLUSH_TYPE_LOCAL,
FLUSH_TYPE_GLOBAL,
};
static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
{
- if (mm_is_thread_local(mm))
+ int active_cpus = atomic_read(&mm->context.active_cpus);
+ int cpu = smp_processor_id();
+
+ if (active_cpus == 0)
+ return FLUSH_TYPE_NONE;
+ if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm)))
return FLUSH_TYPE_LOCAL;
/* Coprocessors require TLBIE to invalidate nMMU. */
@@ -754,7 +760,9 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
*/
smp_mb();
type = flush_type_needed(mm, false);
- if (type == FLUSH_TYPE_GLOBAL) {
+ if (type == FLUSH_TYPE_LOCAL) {
+ _tlbiel_pid(pid, RIC_FLUSH_TLB);
+ } else if (type == FLUSH_TYPE_GLOBAL) {
if (!mmu_has_feature(MMU_FTR_GTSE)) {
unsigned long tgt = H_RPTI_TARGET_CMMU;
@@ -770,8 +778,6 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
} else {
_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
}
- } else {
- _tlbiel_pid(pid, RIC_FLUSH_TLB);
}
preempt_enable();
}
@@ -789,7 +795,9 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
preempt_disable();
smp_mb(); /* see radix__flush_tlb_mm */
type = flush_type_needed(mm, fullmm);
- if (type == FLUSH_TYPE_GLOBAL) {
+ if (type == FLUSH_TYPE_LOCAL) {
+ _tlbiel_pid(pid, RIC_FLUSH_ALL);
+ } else if (type == FLUSH_TYPE_GLOBAL) {
if (!mmu_has_feature(MMU_FTR_GTSE)) {
unsigned long tgt = H_RPTI_TARGET_CMMU;
unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
@@ -803,8 +811,6 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm)
_tlbie_pid(pid, RIC_FLUSH_ALL);
else
_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
- } else {
- _tlbiel_pid(pid, RIC_FLUSH_ALL);
}
preempt_enable();
}
@@ -828,7 +834,9 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
preempt_disable();
smp_mb(); /* see radix__flush_tlb_mm */
type = flush_type_needed(mm, false);
- if (type == FLUSH_TYPE_GLOBAL) {
+ if (type == FLUSH_TYPE_LOCAL) {
+ _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
+ } else if (type == FLUSH_TYPE_GLOBAL) {
if (!mmu_has_feature(MMU_FTR_GTSE)) {
unsigned long tgt, pg_sizes, size;
@@ -845,8 +853,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
_tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
else
_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
- } else {
- _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
}
preempt_enable();
}
@@ -935,6 +941,8 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
preempt_disable();
smp_mb(); /* see radix__flush_tlb_mm */
type = flush_type_needed(mm, fullmm);
+ if (type == FLUSH_TYPE_NONE)
+ goto out;
if (fullmm)
flush_pid = true;
@@ -999,6 +1007,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm,
hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false);
}
}
+out:
preempt_enable();
}
@@ -1123,6 +1132,8 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
preempt_disable();
smp_mb(); /* see radix__flush_tlb_mm */
type = flush_type_needed(mm, fullmm);
+ if (type == FLUSH_TYPE_NONE)
+ goto out;
if (fullmm)
flush_pid = true;
@@ -1166,6 +1177,7 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
_tlbiel_va_range_multicast(mm,
start, end, pid, page_size, psize, also_pwc);
}
+out:
preempt_enable();
}
@@ -1203,7 +1215,9 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
preempt_disable();
smp_mb(); /* see radix__flush_tlb_mm */
type = flush_type_needed(mm, false);
- if (type == FLUSH_TYPE_GLOBAL) {
+ if (type == FLUSH_TYPE_LOCAL) {
+ _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
+ } else if (type == FLUSH_TYPE_GLOBAL) {
if (!mmu_has_feature(MMU_FTR_GTSE)) {
unsigned long tgt, type, pg_sizes;
@@ -1221,8 +1235,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
else
_tlbiel_va_range_multicast(mm,
addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
- } else {
- _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
}
preempt_enable();
--
2.23.0
next prev parent reply other threads:[~2020-12-17 13:54 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-12-17 13:47 [PATCH 0/7] powerpc/64s: TLB flushing improvements Nicholas Piggin
2020-12-17 13:47 ` [PATCH 1/7] powerpc/64s/radix: add warning and comments in mm_cpumask trim Nicholas Piggin
2020-12-17 13:47 ` [PATCH 2/7] powerpc/64s/radix: refactor TLB flush type selection Nicholas Piggin
2020-12-17 13:47 ` Nicholas Piggin [this message]
2020-12-17 13:47 ` [PATCH 4/7] powerpc/64s/radix: Allow mm_cpumask trimming from external sources Nicholas Piggin
2020-12-17 13:47 ` [PATCH 5/7] powerpc/64s/radix: occasionally attempt to trim mm_cpumask Nicholas Piggin
2020-12-17 13:47 ` [PATCH 6/7] powerpc/64s/radix: serialize_against_pte_lookup IPIs " Nicholas Piggin
2020-12-17 18:23 ` kernel test robot
2020-12-17 18:23 ` kernel test robot
2020-12-17 13:47 ` [PATCH 7/7] powerpc/64s: Implement ptep_clear_flush_young that does not flush TLBs Nicholas Piggin
2021-02-10 12:57 ` [PATCH 0/7] powerpc/64s: TLB flushing improvements Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201217134731.488135-4-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.