linux-sunxi.lists.linux.dev archive mirror
 help / color / mirror / Atom feed
From: guoren@kernel.org
To: guoren@kernel.org, anup.patel@wdc.com, palmerdabbelt@google.com,
	arnd@arndb.de, wens@csie.org, maxime@cerno.tech,
	drew@beagleboard.org, liush@allwinnertech.com,
	lazyparser@gmail.com, wefu@redhat.com
Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org,
	linux-arch@vger.kernel.org, linux-sunxi@lists.linux.dev,
	Guo Ren <guoren@linux.alibaba.com>,
	Christoph Hellwig <hch@lst.de>
Subject: [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods
Date: Sun,  6 Jun 2021 09:03:58 +0000	[thread overview]
Message-ID: <1622970249-50770-4-git-send-email-guoren@kernel.org> (raw)
In-Reply-To: <1622970249-50770-1-git-send-email-guoren@kernel.org>

From: Guo Ren <guoren@linux.alibaba.com>

Implement optimized version of the tlb flushing routines for systems
using ASIDs. These are behind the use_asid_allocator static branch to
not affect existing systems not using ASIDs.

Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
Reviewed-by: Anup Patel <anup.patel@wdc.com>
Cc: Palmer Dabbelt <palmerdabbelt@google.com>
Cc: Christoph Hellwig <hch@lst.de>
---
 arch/riscv/include/asm/mmu_context.h |  2 ++
 arch/riscv/include/asm/tlbflush.h    | 22 +++++++++++++++++
 arch/riscv/mm/context.c              |  2 +-
 arch/riscv/mm/tlbflush.c             | 46 +++++++++++++++++++++++++++++++++---
 4 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h
index b065941..7030837 100644
--- a/arch/riscv/include/asm/mmu_context.h
+++ b/arch/riscv/include/asm/mmu_context.h
@@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk,
 	return 0;
 }
 
+DECLARE_STATIC_KEY_FALSE(use_asid_allocator);
+
 #include <asm-generic/mmu_context.h>
 
 #endif /* _ASM_RISCV_MMU_CONTEXT_H */
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index c84218a..894cf75 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -22,9 +22,31 @@ static inline void local_flush_tlb_page(unsigned long addr)
 {
 	ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory"));
 }
+
+static inline void local_flush_tlb_all_asid(unsigned long asid)
+{
+	__asm__ __volatile__ ("sfence.vma x0, %0"
+			:
+			: "r" (asid)
+			: "memory");
+}
+
+static inline void local_flush_tlb_range_asid(unsigned long start,
+				unsigned long size, unsigned long asid)
+{
+	unsigned long tmp, end = ALIGN(start + size, PAGE_SIZE);
+
+	for (tmp = start & PAGE_MASK; tmp < end; tmp += PAGE_SIZE) {
+		__asm__ __volatile__ ("sfence.vma %0, %1"
+				:
+				: "r" (tmp), "r" (asid)
+				: "memory");
+	}
+}
 #else /* CONFIG_MMU */
 #define local_flush_tlb_all()			do { } while (0)
 #define local_flush_tlb_page(addr)		do { } while (0)
+#define local_flush_tlb_range_asid(addr)	do { } while (0)
 #endif /* CONFIG_MMU */
 
 #if defined(CONFIG_SMP) && defined(CONFIG_MMU)
diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c
index 68aa312..45c1b04 100644
--- a/arch/riscv/mm/context.c
+++ b/arch/riscv/mm/context.c
@@ -18,7 +18,7 @@
 
 #ifdef CONFIG_MMU
 
-static DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
+DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
 
 static unsigned long asid_bits;
 static unsigned long num_asids;
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 720b443..87b4e52 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -4,6 +4,7 @@
 #include <linux/smp.h>
 #include <linux/sched.h>
 #include <asm/sbi.h>
+#include <asm/mmu_context.h>
 
 void flush_tlb_all(void)
 {
@@ -39,18 +40,57 @@ static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start,
 	put_cpu();
 }
 
+static void __sbi_tlb_flush_range_asid(struct cpumask *cmask,
+				       unsigned long start,
+				       unsigned long size,
+				       unsigned long asid)
+{
+	struct cpumask hmask;
+	unsigned int cpuid;
+
+	if (cpumask_empty(cmask))
+		return;
+
+	cpuid = get_cpu();
+
+	if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
+		if (size == -1)
+			local_flush_tlb_all_asid(asid);
+		else
+			local_flush_tlb_range_asid(start, size, asid);
+	} else {
+		riscv_cpuid_to_hartid_mask(cmask, &hmask);
+		sbi_remote_sfence_vma_asid(cpumask_bits(&hmask),
+					   start, size, asid);
+	}
+
+	put_cpu();
+}
+
 void flush_tlb_mm(struct mm_struct *mm)
 {
-	__sbi_tlb_flush_range(mm_cpumask(mm), 0, -1);
+	if (static_branch_unlikely(&use_asid_allocator))
+		__sbi_tlb_flush_range_asid(mm_cpumask(mm), 0, -1,
+					   atomic_long_read(&mm->context.id));
+	else
+		__sbi_tlb_flush_range(mm_cpumask(mm), 0, -1);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 {
-	__sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE);
+	if (static_branch_unlikely(&use_asid_allocator))
+		__sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE,
+					   atomic_long_read(&vma->vm_mm->context.id));
+	else
+		__sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE);
 }
 
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 		     unsigned long end)
 {
-	__sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start);
+	if (static_branch_unlikely(&use_asid_allocator))
+		__sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), start, end - start,
+					   atomic_long_read(&vma->vm_mm->context.id));
+	else
+		__sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start);
 }
-- 
2.7.4


  parent reply	other threads:[~2021-06-06  9:05 UTC|newest]

Thread overview: 50+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-06-06  9:03 [RFC PATCH v2 00/11] riscv: Add DMA_COHERENT support for Allwinner D1 guoren
2021-06-06  9:03 ` [RFC PATCH v2 01/11] riscv: asid: Use global mappings for kernel pages guoren
2021-06-06  9:03 ` [PATCH V5 1/3] riscv: " guoren
2021-06-06  9:03 ` guoren [this message]
2021-06-06 14:38   ` [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods Christoph Hellwig
2021-06-06  9:03 ` [RFC PATCH v2 02/11] riscv: asid: " guoren
2021-06-06  9:04 ` [RFC PATCH v2 03/11] riscv: asid: Optimize tlbflush coding convention guoren
2021-06-06  9:04 ` [PATCH V5 3/3] riscv: tlbflush: Optimize " guoren
2021-06-06  9:04 ` [RFC PATCH v2 04/11] riscv: pgtable: Fixup _PAGE_CHG_MASK usage guoren
2021-06-06  9:04 ` [RFC PATCH v2 05/11] riscv: pgtable: Add custom protection_map init guoren
2021-06-06  9:04 ` [RFC PATCH v2 06/11] riscv: pgtable: Add DMA_COHERENT with custom PTE attributes guoren
2021-06-06 14:39   ` Christoph Hellwig
2021-06-06 15:08     ` Guo Ren
2021-06-06 17:22   ` Nick Kossifidis
2021-06-07  6:19     ` Christoph Hellwig
2021-06-06  9:04 ` [RFC PATCH v2 07/11] riscv: cmo: Add dma-noncoherency support guoren
2021-06-06  9:04 ` [RFC PATCH v2 08/11] riscv: cmo: Add vendor custom icache sync guoren
2021-06-06  9:04 ` [RFC PATCH v2 09/11] riscv: soc: Initial DTS for Allwinner D1 NeZha board guoren
2021-06-06 16:26   ` Jernej Škrabec
2021-06-06 17:05     ` Guo Ren
2021-06-07  3:44     ` Guo Ren
2021-06-07  7:27       ` Maxime Ripard
2021-06-07  7:53         ` Guo Ren
2021-06-07  7:24   ` Maxime Ripard
2021-06-07  8:07     ` Guo Ren
2021-06-14 15:33       ` Maxime Ripard
2021-06-14 16:28         ` Guo Ren
2021-06-14 16:31           ` Jernej Škrabec
2021-06-06  9:04 ` [RFC PATCH v2 10/11] riscv: soc: Add Allwinner SoC kconfig option guoren
2021-06-07  7:19   ` Maxime Ripard
2021-06-07  7:27     ` Arnd Bergmann
2021-06-07  7:45       ` Guo Ren
2021-06-07  7:43     ` Guo Ren
2021-06-07 12:12       ` Maxime Ripard
2021-06-07 12:39         ` Guo Ren
2021-06-06  9:04 ` [RFC PATCH v2 11/11] riscv: soc: Allwinner D1 GMAC driver only for temp use guoren
2021-06-06 10:50   ` Andre Przywara
2021-06-06 15:32     ` Guo Ren
2021-06-06 15:39       ` Jernej Škrabec
2021-06-06 15:41         ` Guo Ren
2021-06-06 16:16   ` Arnd Bergmann
2021-06-06 16:32     ` Jernej Škrabec
2021-06-06 16:53       ` Guo Ren
2021-06-06 16:53     ` Guo Ren
2021-06-06 16:29 ` [RFC PATCH v2 00/11] riscv: Add DMA_COHERENT support for Allwinner D1 Jernej Škrabec
2021-06-06 16:54   ` Guo Ren
2021-06-06 17:14     ` Jernej Škrabec
2021-06-06 23:42       ` Guo Ren
2021-06-07  3:44 ` Anup Patel
2021-06-07  4:36   ` Guo Ren

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1622970249-50770-4-git-send-email-guoren@kernel.org \
    --to=guoren@kernel.org \
    --cc=anup.patel@wdc.com \
    --cc=arnd@arndb.de \
    --cc=drew@beagleboard.org \
    --cc=guoren@linux.alibaba.com \
    --cc=hch@lst.de \
    --cc=lazyparser@gmail.com \
    --cc=linux-arch@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-riscv@lists.infradead.org \
    --cc=linux-sunxi@lists.linux.dev \
    --cc=liush@allwinnertech.com \
    --cc=maxime@cerno.tech \
    --cc=palmerdabbelt@google.com \
    --cc=wefu@redhat.com \
    --cc=wens@csie.org \
    --subject='Re: [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
on how to clone and mirror all data and code used for this inbox