[RFC PATCH v1 0/3] arm64: tlb: add support for TTL field

[RFC PATCH v1 0/3] arm64: tlb: add support for TTL field

[Zhenyu Ye]

ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
the level of translation table walk holding the leaf entry for the
address that is being invalidated. Hardware can use this information
to determine if there was a risk of splintering.

This set of patches adds TTL field to __TLBI_ADDR, and uses
Architecture-specific MM context to pass the TTL value to tlb interface.

The default value of TTL is 0, which will not have any impact on the
TLB maintenance instructions. The last patch trys to use TTL field in
some obviously tlb-flush interface.

Zhenyu Ye (3):
  arm64: tlb: add TTL field to __TLBI_ADDR
  arm64: tlb: use mm_struct.context.flags to indicate TTL
  arm64: tlb: add support for TTL in some functions

 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/mmu.h      | 11 ++++++++++
 arch/arm64/include/asm/sysreg.h   |  4 ++++
 arch/arm64/include/asm/tlb.h      |  3 +++
 arch/arm64/include/asm/tlbflush.h | 35 +++++++++++++++++++++++--------
 arch/arm64/kernel/cpufeature.c    | 10 +++++++++
 arch/arm64/kernel/sys_compat.c    |  2 +-
 arch/arm64/mm/hugetlbpage.c       |  2 ++
 8 files changed, 59 insertions(+), 11 deletions(-)

-- 
2.19.1

[RFC PATCH v1 1/3] arm64: tlb: add TTL field to __TLBI_ADDR

[Zhenyu Ye]

ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
the level of translation table walk holding the leaf entry for the
address that is being invalidated.

This patch add support for TTL feature and set the default value 0
where __TLBI_ADDR is called.

Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
---
 arch/arm64/include/asm/cpucaps.h  |  3 ++-
 arch/arm64/include/asm/sysreg.h   |  4 ++++
 arch/arm64/include/asm/tlbflush.h | 33 ++++++++++++++++++++++---------
 arch/arm64/kernel/cpufeature.c    | 10 ++++++++++
 arch/arm64/kernel/sys_compat.c    |  2 +-
 5 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 865e0253fc1e..60f3b090ea9a 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -58,7 +58,8 @@
 #define ARM64_WORKAROUND_SPECULATIVE_AT_NVHE	48
 #define ARM64_HAS_E0PD				49
 #define ARM64_HAS_RNG				50
+#define ARM64_HAS_TLBI_TTL			51
 
-#define ARM64_NCAPS				51
+#define ARM64_NCAPS				52
 
 #endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index b91570ff9db1..7caa3c9facfe 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -685,6 +685,7 @@
 
 /* id_aa64mmfr2 */
 #define ID_AA64MMFR2_E0PD_SHIFT		60
+#define ID_AA64MMFR2_TTL_SHIFT		48
 #define ID_AA64MMFR2_FWB_SHIFT		40
 #define ID_AA64MMFR2_AT_SHIFT		32
 #define ID_AA64MMFR2_LVA_SHIFT		16
@@ -693,6 +694,9 @@
 #define ID_AA64MMFR2_UAO_SHIFT		4
 #define ID_AA64MMFR2_CNP_SHIFT		0
 
+#define ID_AA64MMFR2_TTL_NI		0x0
+#define ID_AA64MMFR2_TTL_SUPPORTED	0x1
+
 /* id_aa64dfr0 */
 #define ID_AA64DFR0_PMSVER_SHIFT	32
 #define ID_AA64DFR0_CTX_CMPS_SHIFT	28
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index bc3949064725..10b12710b7cc 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -50,15 +50,30 @@
 		__tlbi(op, (arg) | USER_ASID_FLAG);				\
 } while (0)
 
-/* This macro creates a properly formatted VA operand for the TLBI */
-#define __TLBI_VADDR(addr, asid)				\
+/* This macro creates a properly formatted VA operand for the TLBI.
+ * The TTL field is introduced since ARMv8.4. ARMv8.4-TTL provides the TTL
+ * field to indicate the level of translation table walk holding the leaf entry
+ * for the address that is being invalidated.
+ */
+#define __TLBI_VADDR(addr, asid, ttl)				\
 	({							\
 		unsigned long __ta = (addr) >> 12;		\
 		__ta &= GENMASK_ULL(43, 0);			\
+		if (cpus_have_const_cap(ARM64_HAS_TLBI_TTL))	\
+			__ta |= (unsigned long)(ttl) << 44;	\
 		__ta |= (unsigned long)(asid) << 48;		\
 		__ta;						\
 	})
 
+/*
+ * __TLB_TG defines translation granule of the system, which is defined by
+ * PAGE_SHIFT.  Used by TTL.
+ *  - 4KB	: 1
+ *  - 16KB	: 2
+ *  - 64KB	: 3
+ */
+#define __TLB_TG	((PAGE_SHIFT - 12) / 2 + 1)
+
 /*
  *	TLB Invalidation
  *	================
@@ -146,7 +161,7 @@ static inline void flush_tlb_all(void)
 
 static inline void flush_tlb_mm(struct mm_struct *mm)
 {
-	unsigned long asid = __TLBI_VADDR(0, ASID(mm));
+	unsigned long asid = __TLBI_VADDR(0, ASID(mm), 0);
 
 	dsb(ishst);
 	__tlbi(aside1is, asid);
@@ -157,7 +172,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
 					 unsigned long uaddr)
 {
-	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
+	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm), 0);
 
 	dsb(ishst);
 	__tlbi(vale1is, addr);
@@ -195,8 +210,8 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 	/* Convert the stride into units of 4k */
 	stride >>= 12;
 
-	start = __TLBI_VADDR(start, asid);
-	end = __TLBI_VADDR(end, asid);
+	start = __TLBI_VADDR(start, asid, 0);
+	end = __TLBI_VADDR(end, asid, 0);
 
 	dsb(ishst);
 	for (addr = start; addr < end; addr += stride) {
@@ -230,8 +245,8 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
 		return;
 	}
 
-	start = __TLBI_VADDR(start, 0);
-	end = __TLBI_VADDR(end, 0);
+	start = __TLBI_VADDR(start, 0, 0);
+	end = __TLBI_VADDR(end, 0, 0);
 
 	dsb(ishst);
 	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
@@ -246,7 +261,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
  */
 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
 {
-	unsigned long addr = __TLBI_VADDR(kaddr, 0);
+	unsigned long addr = __TLBI_VADDR(kaddr, 0, 0);
 
 	dsb(ishst);
 	__tlbi(vaae1is, addr);
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 0b6715625cf6..9a84c59e1482 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1672,6 +1672,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.min_field_value = 1,
 	},
 #endif
+	{
+		.desc = "TTL field in TLBI operation",
+		.capability = ARM64_HAS_TLBI_TTL,
+		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
+		.matches = has_cpuid_feature,
+		.sys_reg = SYS_ID_AA64MMFR2_EL1,
+		.field_pos = ID_AA64MMFR2_TTL_SHIFT,
+		.sign = FTR_UNSIGNED,
+		.min_field_value = ID_AA64MMFR2_TTL_SUPPORTED,
+	},
 	{},
 };
 
diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
index 3c18c2454089..5d1570354d29 100644
--- a/arch/arm64/kernel/sys_compat.c
+++ b/arch/arm64/kernel/sys_compat.c
@@ -37,7 +37,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end)
 			 * The workaround requires an inner-shareable tlbi.
 			 * We pick the reserved-ASID to minimise the impact.
 			 */
-			__tlbi(aside1is, __TLBI_VADDR(0, 0));
+			__tlbi(aside1is, __TLBI_VADDR(0, 0, 0));
 			dsb(ish);
 		}
 
-- 
2.19.1

[RFC PATCH v1 2/3] arm64: tlb: use mm_struct.context.flags to indicate TTL

[Zhenyu Ye]

Use Architecture-specific MM context to indicate the level of page
table walk. This avoids lots of changes to common-interface.

Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
---
 arch/arm64/include/asm/mmu.h      | 11 +++++++++++
 arch/arm64/include/asm/tlbflush.h |  8 +++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index e4d862420bb4..7410d2997c2a 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -8,6 +8,10 @@
 #include <asm/cputype.h>
 
 #define MMCF_AARCH32	0x1	/* mm context flag for AArch32 executables */
+#define TLBI_LEVEL_1	0x10	/* mm context flag for the level of ptw */
+#define TLBI_LEVEL_2	0x20
+#define TLBI_LEVEL_3	0x30
+
 #define USER_ASID_BIT	48
 #define USER_ASID_FLAG	(UL(1) << USER_ASID_BIT)
 #define TTBR_ASID_MASK	(UL(0xffff) << 48)
@@ -19,6 +23,10 @@
 typedef struct {
 	atomic64_t	id;
 	void		*vdso;
+	/*
+	 * flags[3:0]: AArch32 executables
+	 * flags[7:4]: the level of page table walk
+	 */
 	unsigned long	flags;
 } mm_context_t;
 
@@ -29,6 +37,9 @@ typedef struct {
  */
 #define ASID(mm)	((mm)->context.id.counter & 0xffff)
 
+/* This macro is only used by TLBI TTL */
+#define TLBI_LEVEL(mm)	((mm)->context.flags >> 4 & 0xf)
+
 extern bool arm64_use_ng_mappings;
 
 static inline bool arm64_kernel_unmapped_at_el0(void)
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 10b12710b7cc..9f02a5383ac3 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -172,7 +172,8 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
 					 unsigned long uaddr)
 {
-	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm), 0);
+	unsigned long ttl = (__TLB_TG << 2) + TLBI_LEVEL(vma->vm_mm);
+	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm), ttl);
 
 	dsb(ishst);
 	__tlbi(vale1is, addr);
@@ -197,6 +198,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 				     unsigned long stride, bool last_level)
 {
 	unsigned long asid = ASID(vma->vm_mm);
+	unsigned long ttl = (__TLB_TG << 2) + TLBI_LEVEL(vma->vm_mm);
 	unsigned long addr;
 
 	start = round_down(start, stride);
@@ -210,8 +212,8 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 	/* Convert the stride into units of 4k */
 	stride >>= 12;
 
-	start = __TLBI_VADDR(start, asid, 0);
-	end = __TLBI_VADDR(end, asid, 0);
+	start = __TLBI_VADDR(start, asid, ttl);
+	end = __TLBI_VADDR(end, asid, ttl);
 
 	dsb(ishst);
 	for (addr = start; addr < end; addr += stride) {
-- 
2.19.1

[RFC PATCH v1 3/3] arm64: tlb: add support for TTL in some functions

[Zhenyu Ye]

Add support for TTL in some ARM64-Architecture functions. The
relevant functions are:

	__pte_free_tlb
	__pmd_free_tlb
	__pud_free_tlb
	clear_flush
	get_clear_flush

Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
---
 arch/arm64/include/asm/tlb.h | 3 +++
 arch/arm64/mm/hugetlbpage.c  | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index b76df828e6b7..36428ce53185 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -44,6 +44,7 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 				  unsigned long addr)
 {
+	tlb->mm->context.flags = TLBI_LEVEL_3;
 	pgtable_pte_page_dtor(pte);
 	tlb_remove_table(tlb, pte);
 }
@@ -53,6 +54,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 				  unsigned long addr)
 {
 	struct page *page = virt_to_page(pmdp);
+	tlb->mm->context.flags = TLBI_LEVEL_2;
 
 	pgtable_pmd_page_dtor(page);
 	tlb_remove_table(tlb, page);
@@ -63,6 +65,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
 				  unsigned long addr)
 {
+	tlb->mm->context.flags = TLBI_LEVEL_1;
 	tlb_remove_table(tlb, virt_to_page(pudp));
 }
 #endif
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index bbeb6a5a6ba6..4c2f1b802cb8 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -141,6 +141,7 @@ static pte_t get_clear_flush(struct mm_struct *mm,
 
 	if (valid) {
 		struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
+		mm->context.flags = TLBI_LEVEL_3;
 		flush_tlb_range(&vma, saddr, addr);
 	}
 	return orig_pte;
@@ -163,6 +164,7 @@ static void clear_flush(struct mm_struct *mm,
 {
 	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
 	unsigned long i, saddr = addr;
+	mm->context.flags = TLBI_LEVEL_3;
 
 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
 		pte_clear(mm, addr, ptep);
-- 
2.19.1

Re: [RFC PATCH v1 0/3] arm64: tlb: add support for TTL field

[Marc Zyngier]

Zhenyu,

On 2020-03-11 02:53, Zhenyu Ye wrote:
> ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
> the level of translation table walk holding the leaf entry for the
> address that is being invalidated. Hardware can use this information
> to determine if there was a risk of splintering.
> 
> This set of patches adds TTL field to __TLBI_ADDR, and uses
> Architecture-specific MM context to pass the TTL value to tlb 
> interface.
> 
> The default value of TTL is 0, which will not have any impact on the
> TLB maintenance instructions. The last patch trys to use TTL field in
> some obviously tlb-flush interface.

I have already posted some support for ARMv8.4-TTL as part of my NV 
series [1],
patches 62, 67, 68 and 69. This only deals with Stage-2 translation so 
far.
If you intend to add Stage-1, please build on top of what I have already 
posted
(I can extract the patches on a separate branch if you want).

Thanks,

         M.

[1] 
https://lore.kernel.org/linux-arm-kernel/20200211174938.27809-1-maz@kernel.org/
-- 
Jazz is not dead. It just smells funny...

Re: [RFC PATCH v1 0/3] arm64: tlb: add support for TTL field

[yezhenyu (A)]

Hi Marc,

On 2020/3/11 17:12, Marc Zyngier wrote:
> Zhenyu,
> 
> On 2020-03-11 02:53, Zhenyu Ye wrote:
>> ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
>> the level of translation table walk holding the leaf entry for the
>> address that is being invalidated. Hardware can use this information
>> to determine if there was a risk of splintering.
>>
>> This set of patches adds TTL field to __TLBI_ADDR, and uses
>> Architecture-specific MM context to pass the TTL value to tlb interface.
>>
>> The default value of TTL is 0, which will not have any impact on the
>> TLB maintenance instructions. The last patch trys to use TTL field in
>> some obviously tlb-flush interface.
> 
> I have already posted some support for ARMv8.4-TTL as part of my NV series [1],
> patches 62, 67, 68 and 69. This only deals with Stage-2 translation so far.
> If you intend to add Stage-1, please build on top of what I have already posted
> (I can extract the patches on a separate branch if you want).
> 
> Thanks,
> 
>         M.
> 
> [1] https://lore.kernel.org/linux-arm-kernel/20200211174938.27809-1-maz@kernel.org/

I just readed your code changes to TTL. You pass the TTL value by changing the
function interface, which only involves the ARM and ARM64 architectures in Stage-2
translation.

However, in Stage-1, many common interfaces(such as flush_tlb_range) need to be
modified, which involves very much architectures. So I try to use MM context in
mm_struct to pass the TTL value.

I will send patch v2 based on top of your kvm-arm64/nv-5.6-rc1 branch soon.

https://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git/log/?h=kvm-arm64/nv-5.6-rc1


Thanks,
Zhenyu

Re: [RFC PATCH v1 0/3] arm64: tlb: add support for TTL field

[yezhenyu (A)]

Hi Marc,

On 2020/3/11 17:12, Marc Zyngier wrote:
> Zhenyu,
> 
> On 2020-03-11 02:53, Zhenyu Ye wrote:
>> ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
>> the level of translation table walk holding the leaf entry for the
>> address that is being invalidated. Hardware can use this information
>> to determine if there was a risk of splintering.
>>
>> This set of patches adds TTL field to __TLBI_ADDR, and uses
>> Architecture-specific MM context to pass the TTL value to tlb interface.
>>
>> The default value of TTL is 0, which will not have any impact on the
>> TLB maintenance instructions. The last patch trys to use TTL field in
>> some obviously tlb-flush interface.
> 
> I have already posted some support for ARMv8.4-TTL as part of my NV series [1],
> patches 62, 67, 68 and 69. This only deals with Stage-2 translation so far.
> If you intend to add Stage-1, please build on top of what I have already posted
> (I can extract the patches on a separate branch if you want).
> 
> Thanks,
> 
>         M.
> 
> [1] https://lore.kernel.org/linux-arm-kernel/20200211174938.27809-1-maz@kernel.org/

I have sent the PATCH v2 on 2020-03-12 [1]. Do you have any suggestion
for my code?  Is it appropriate to post my code based on your branch?

Looking forward to your suggestions.


Thanks,

	zhenyu

[1] https://lkml.org/lkml/2020/3/12/2