* [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
@ 2016-03-29 21:11 ` Nitin Gupta
0 siblings, 0 replies; 8+ messages in thread
From: Nitin Gupta @ 2016-03-29 21:11 UTC (permalink / raw)
To: David S. Miller
Cc: Nitin Gupta, David S. Miller, Andrew Morton, Kirill A. Shutemov,
Khalid Aziz, Minchan Kim, Aneesh Kumar K.V, Zhang Zhen,
Stephen Rothwell, Ingo Molnar, Chris Hyser, David Hildenbrand,
Toshi Kani, Tony Luck, sparclinux, linux-kernel
During hugepage map/unmap, TSB and TLB flushes are currently
issued at every PAGE_SIZE'd boundary which is unnecessary.
We now issue the flush at REAL_HPAGE_SIZE boundaries only.
Without this patch workloads which unmap a large hugepage
backed VMA region get CPU lockups due to excessive TLB
flush calls.
Orabug: 22365539, 22643230, 22995196
Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
---
Changelog v3 vs v2:
- Changed patch title to reflect that both map/unmap cases
are affected.
- Don't do TLB flush if original PTE wasn't valid (DaveM)
- Use tlb_batch_add() instead of directly calling TLB flush
function. This routine also flushes dcache (needed by older
sparcs) (DaveM)
Changelog v1 vs v2:
- Access PTEs in order (David Miller)
- Issue TLB and TSB flush after clearing PTEs (David Miller)
---
arch/sparc/include/asm/pgtable_64.h | 38 +++++++++++++++++++++++++-----------
arch/sparc/include/asm/tlbflush_64.h | 3 ++-
arch/sparc/mm/hugetlbpage.c | 33 ++++++++++++++++++++++++++-----
arch/sparc/mm/init_64.c | 12 ------------
arch/sparc/mm/tlb.c | 18 +++++++++++++----
arch/sparc/mm/tsb.c | 32 ++++++++++++++++--------------
6 files changed, 88 insertions(+), 48 deletions(-)
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 7a38d6a..0e706b8 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -375,7 +375,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot)
#define pgprot_noncached pgprot_noncached
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline pte_t pte_mkhuge(pte_t pte)
+static inline unsigned long __pte_huge_mask(void)
{
unsigned long mask;
@@ -390,8 +390,19 @@ static inline pte_t pte_mkhuge(pte_t pte)
: "=r" (mask)
: "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
- return __pte(pte_val(pte) | mask);
+ return mask;
+}
+
+static inline pte_t pte_mkhuge(pte_t pte)
+{
+ return __pte(pte_val(pte) | __pte_huge_mask());
+}
+
+static inline bool is_hugetlb_pte(pte_t pte)
+{
+ return !!(pte_val(pte) & __pte_huge_mask());
}
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
@@ -858,6 +869,19 @@ static inline unsigned long pud_pfn(pud_t pud)
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm);
+static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
+ pte_t *ptep, pte_t orig, int fullmm)
+{
+ /* It is more efficient to let flush_tlb_kernel_range()
+ * handle init_mm tlb flushes.
+ *
+ * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
+ * and SUN4V pte layout, so this inline test is fine.
+ */
+ if (likely(mm != &init_mm) && pte_accessible(mm, orig))
+ tlb_batch_add(mm, vaddr, ptep, orig, fullmm);
+}
+
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr,
@@ -874,15 +898,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t orig = *ptep;
*ptep = pte;
-
- /* It is more efficient to let flush_tlb_kernel_range()
- * handle init_mm tlb flushes.
- *
- * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
- * and SUN4V pte layout, so this inline test is fine.
- */
- if (likely(mm != &init_mm) && pte_accessible(mm, orig))
- tlb_batch_add(mm, addr, ptep, orig, fullmm);
+ maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm);
}
#define set_pte_at(mm,addr,ptep,pte) \
diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h
index dea1cfa..a8e192e 100644
--- a/arch/sparc/include/asm/tlbflush_64.h
+++ b/arch/sparc/include/asm/tlbflush_64.h
@@ -8,6 +8,7 @@
#define TLB_BATCH_NR 192
struct tlb_batch {
+ bool huge;
struct mm_struct *mm;
unsigned long tlb_nr;
unsigned long active;
@@ -16,7 +17,7 @@ struct tlb_batch {
void flush_tsb_kernel_range(unsigned long start, unsigned long end);
void flush_tsb_user(struct tlb_batch *tb);
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr);
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge);
/* TLB flush operations. */
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 4977800..ba52e64 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -176,17 +176,31 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
int i;
+ pte_t orig[2];
+ unsigned long nptes;
if (!pte_present(*ptep) && pte_present(entry))
mm->context.huge_pte_count++;
addr &= HPAGE_MASK;
- for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
- set_pte_at(mm, addr, ptep, entry);
+
+ nptes = 1 << HUGETLB_PAGE_ORDER;
+ orig[0] = *ptep;
+ orig[1] = *(ptep + nptes / 2);
+ for (i = 0; i < nptes; i++) {
+ *ptep = entry;
ptep++;
addr += PAGE_SIZE;
pte_val(entry) += PAGE_SIZE;
}
+
+ /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, orig[1], 0);
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, orig[0], 0);
}
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
@@ -194,19 +208,28 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
{
pte_t entry;
int i;
+ unsigned long nptes;
entry = *ptep;
if (pte_present(entry))
mm->context.huge_pte_count--;
addr &= HPAGE_MASK;
-
- for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
- pte_clear(mm, addr, ptep);
+ nptes = 1 << HUGETLB_PAGE_ORDER;
+ for (i = 0; i < nptes; i++) {
+ *ptep = __pte(0UL);
addr += PAGE_SIZE;
ptep++;
}
+ /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
+
return entry;
}
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 1cfe6aa..bf181a9 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -324,18 +324,6 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde
tsb_insert(tsb, tag, tte);
}
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline bool is_hugetlb_pte(pte_t pte)
-{
- if ((tlb_type == hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
- (tlb_type != hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))
- return true;
- return false;
-}
-#endif
-
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
struct mm_struct *mm;
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 9df2190..0bee48a 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void)
}
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
- bool exec)
+ bool exec, bool huge)
{
struct tlb_batch *tb = &get_cpu_var(tlb_batch);
unsigned long nr;
@@ -84,13 +84,21 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
}
if (!tb->active) {
- flush_tsb_user_page(mm, vaddr);
+ flush_tsb_user_page(mm, vaddr, huge);
global_flush_tlb_page(mm, vaddr);
goto out;
}
- if (nr == 0)
+ if (nr == 0) {
tb->mm = mm;
+ tb->huge = huge;
+ }
+
+ if (tb->huge != huge) {
+ flush_tlb_pending();
+ tb->huge = huge;
+ nr = 0;
+ }
tb->vaddrs[nr] = vaddr;
tb->tlb_nr = ++nr;
@@ -104,6 +112,8 @@ out:
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm)
{
+ bool huge = is_hugetlb_pte(orig);
+
if (tlb_type != hypervisor &&
pte_dirty(orig)) {
unsigned long paddr, pfn = pte_pfn(orig);
@@ -129,7 +139,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
no_cache_flush:
if (!fullmm)
- tlb_batch_add_one(mm, vaddr, pte_exec(orig));
+ tlb_batch_add_one(mm, vaddr, pte_exec(orig), huge);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index a065766..a0604a4 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -76,14 +76,15 @@ void flush_tsb_user(struct tlb_batch *tb)
spin_lock_irqsave(&mm->context.lock, flags);
- base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
- if (tlb_type == cheetah_plus || tlb_type == hypervisor)
- base = __pa(base);
- __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
-
+ if (!tb->huge) {
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+ if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+ base = __pa(base);
+ __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
+ }
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ if (tb->huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
@@ -94,20 +95,21 @@ void flush_tsb_user(struct tlb_batch *tb)
spin_unlock_irqrestore(&mm->context.lock, flags);
}
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge)
{
unsigned long nentries, base, flags;
spin_lock_irqsave(&mm->context.lock, flags);
- base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
- if (tlb_type == cheetah_plus || tlb_type == hypervisor)
- base = __pa(base);
- __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
-
+ if (!huge) {
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+ if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+ base = __pa(base);
+ __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
+ }
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ if (huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
--
2.6.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
@ 2016-03-29 21:11 ` Nitin Gupta
0 siblings, 0 replies; 8+ messages in thread
From: Nitin Gupta @ 2016-03-29 21:11 UTC (permalink / raw)
To: David S. Miller
Cc: Nitin Gupta, Andrew Morton, Kirill A. Shutemov, Khalid Aziz,
Minchan Kim, Aneesh Kumar K.V, Zhang Zhen, Stephen Rothwell,
Ingo Molnar, Chris Hyser, David Hildenbrand, Toshi Kani,
Tony Luck, sparclinux, linux-kernel
During hugepage map/unmap, TSB and TLB flushes are currently
issued at every PAGE_SIZE'd boundary which is unnecessary.
We now issue the flush at REAL_HPAGE_SIZE boundaries only.
Without this patch workloads which unmap a large hugepage
backed VMA region get CPU lockups due to excessive TLB
flush calls.
Orabug: 22365539, 22643230, 22995196
Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
---
Changelog v3 vs v2:
- Changed patch title to reflect that both map/unmap cases
are affected.
- Don't do TLB flush if original PTE wasn't valid (DaveM)
- Use tlb_batch_add() instead of directly calling TLB flush
function. This routine also flushes dcache (needed by older
sparcs) (DaveM)
Changelog v1 vs v2:
- Access PTEs in order (David Miller)
- Issue TLB and TSB flush after clearing PTEs (David Miller)
---
arch/sparc/include/asm/pgtable_64.h | 38 +++++++++++++++++++++++++-----------
arch/sparc/include/asm/tlbflush_64.h | 3 ++-
arch/sparc/mm/hugetlbpage.c | 33 ++++++++++++++++++++++++++-----
arch/sparc/mm/init_64.c | 12 ------------
arch/sparc/mm/tlb.c | 18 +++++++++++++----
arch/sparc/mm/tsb.c | 32 ++++++++++++++++--------------
6 files changed, 88 insertions(+), 48 deletions(-)
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 7a38d6a..0e706b8 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -375,7 +375,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot)
#define pgprot_noncached pgprot_noncached
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline pte_t pte_mkhuge(pte_t pte)
+static inline unsigned long __pte_huge_mask(void)
{
unsigned long mask;
@@ -390,8 +390,19 @@ static inline pte_t pte_mkhuge(pte_t pte)
: "=r" (mask)
: "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
- return __pte(pte_val(pte) | mask);
+ return mask;
+}
+
+static inline pte_t pte_mkhuge(pte_t pte)
+{
+ return __pte(pte_val(pte) | __pte_huge_mask());
+}
+
+static inline bool is_hugetlb_pte(pte_t pte)
+{
+ return !!(pte_val(pte) & __pte_huge_mask());
}
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
@@ -858,6 +869,19 @@ static inline unsigned long pud_pfn(pud_t pud)
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm);
+static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
+ pte_t *ptep, pte_t orig, int fullmm)
+{
+ /* It is more efficient to let flush_tlb_kernel_range()
+ * handle init_mm tlb flushes.
+ *
+ * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
+ * and SUN4V pte layout, so this inline test is fine.
+ */
+ if (likely(mm != &init_mm) && pte_accessible(mm, orig))
+ tlb_batch_add(mm, vaddr, ptep, orig, fullmm);
+}
+
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr,
@@ -874,15 +898,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t orig = *ptep;
*ptep = pte;
-
- /* It is more efficient to let flush_tlb_kernel_range()
- * handle init_mm tlb flushes.
- *
- * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
- * and SUN4V pte layout, so this inline test is fine.
- */
- if (likely(mm != &init_mm) && pte_accessible(mm, orig))
- tlb_batch_add(mm, addr, ptep, orig, fullmm);
+ maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm);
}
#define set_pte_at(mm,addr,ptep,pte) \
diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h
index dea1cfa..a8e192e 100644
--- a/arch/sparc/include/asm/tlbflush_64.h
+++ b/arch/sparc/include/asm/tlbflush_64.h
@@ -8,6 +8,7 @@
#define TLB_BATCH_NR 192
struct tlb_batch {
+ bool huge;
struct mm_struct *mm;
unsigned long tlb_nr;
unsigned long active;
@@ -16,7 +17,7 @@ struct tlb_batch {
void flush_tsb_kernel_range(unsigned long start, unsigned long end);
void flush_tsb_user(struct tlb_batch *tb);
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr);
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge);
/* TLB flush operations. */
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 4977800..ba52e64 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -176,17 +176,31 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
int i;
+ pte_t orig[2];
+ unsigned long nptes;
if (!pte_present(*ptep) && pte_present(entry))
mm->context.huge_pte_count++;
addr &= HPAGE_MASK;
- for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
- set_pte_at(mm, addr, ptep, entry);
+
+ nptes = 1 << HUGETLB_PAGE_ORDER;
+ orig[0] = *ptep;
+ orig[1] = *(ptep + nptes / 2);
+ for (i = 0; i < nptes; i++) {
+ *ptep = entry;
ptep++;
addr += PAGE_SIZE;
pte_val(entry) += PAGE_SIZE;
}
+
+ /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, orig[1], 0);
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, orig[0], 0);
}
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
@@ -194,19 +208,28 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
{
pte_t entry;
int i;
+ unsigned long nptes;
entry = *ptep;
if (pte_present(entry))
mm->context.huge_pte_count--;
addr &= HPAGE_MASK;
-
- for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) {
- pte_clear(mm, addr, ptep);
+ nptes = 1 << HUGETLB_PAGE_ORDER;
+ for (i = 0; i < nptes; i++) {
+ *ptep = __pte(0UL);
addr += PAGE_SIZE;
ptep++;
}
+ /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
+ addr -= REAL_HPAGE_SIZE;
+ ptep -= nptes / 2;
+ maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
+
return entry;
}
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 1cfe6aa..bf181a9 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -324,18 +324,6 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde
tsb_insert(tsb, tag, tte);
}
-#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline bool is_hugetlb_pte(pte_t pte)
-{
- if ((tlb_type = hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4V) = _PAGE_SZHUGE_4V) ||
- (tlb_type != hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4U) = _PAGE_SZHUGE_4U))
- return true;
- return false;
-}
-#endif
-
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
struct mm_struct *mm;
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 9df2190..0bee48a 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void)
}
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
- bool exec)
+ bool exec, bool huge)
{
struct tlb_batch *tb = &get_cpu_var(tlb_batch);
unsigned long nr;
@@ -84,13 +84,21 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
}
if (!tb->active) {
- flush_tsb_user_page(mm, vaddr);
+ flush_tsb_user_page(mm, vaddr, huge);
global_flush_tlb_page(mm, vaddr);
goto out;
}
- if (nr = 0)
+ if (nr = 0) {
tb->mm = mm;
+ tb->huge = huge;
+ }
+
+ if (tb->huge != huge) {
+ flush_tlb_pending();
+ tb->huge = huge;
+ nr = 0;
+ }
tb->vaddrs[nr] = vaddr;
tb->tlb_nr = ++nr;
@@ -104,6 +112,8 @@ out:
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm)
{
+ bool huge = is_hugetlb_pte(orig);
+
if (tlb_type != hypervisor &&
pte_dirty(orig)) {
unsigned long paddr, pfn = pte_pfn(orig);
@@ -129,7 +139,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
no_cache_flush:
if (!fullmm)
- tlb_batch_add_one(mm, vaddr, pte_exec(orig));
+ tlb_batch_add_one(mm, vaddr, pte_exec(orig), huge);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index a065766..a0604a4 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -76,14 +76,15 @@ void flush_tsb_user(struct tlb_batch *tb)
spin_lock_irqsave(&mm->context.lock, flags);
- base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
- if (tlb_type = cheetah_plus || tlb_type = hypervisor)
- base = __pa(base);
- __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
-
+ if (!tb->huge) {
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+ if (tlb_type = cheetah_plus || tlb_type = hypervisor)
+ base = __pa(base);
+ __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
+ }
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ if (tb->huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type = cheetah_plus || tlb_type = hypervisor)
@@ -94,20 +95,21 @@ void flush_tsb_user(struct tlb_batch *tb)
spin_unlock_irqrestore(&mm->context.lock, flags);
}
-void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge)
{
unsigned long nentries, base, flags;
spin_lock_irqsave(&mm->context.lock, flags);
- base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
- nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
- if (tlb_type = cheetah_plus || tlb_type = hypervisor)
- base = __pa(base);
- __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
-
+ if (!huge) {
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+ if (tlb_type = cheetah_plus || tlb_type = hypervisor)
+ base = __pa(base);
+ __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
+ }
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ if (huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) {
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type = cheetah_plus || tlb_type = hypervisor)
--
2.6.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
2016-03-29 21:11 ` Nitin Gupta
@ 2016-03-29 21:25 ` kbuild test robot
-1 siblings, 0 replies; 8+ messages in thread
From: kbuild test robot @ 2016-03-29 21:25 UTC (permalink / raw)
To: Nitin Gupta
Cc: kbuild-all, David S. Miller, Nitin Gupta, Andrew Morton,
Kirill A. Shutemov, Khalid Aziz, Minchan Kim, Aneesh Kumar K.V,
Zhang Zhen, Stephen Rothwell, Ingo Molnar, Chris Hyser,
David Hildenbrand, Toshi Kani, Tony Luck, sparclinux,
linux-kernel
[-- Attachment #1: Type: text/plain, Size: 2843 bytes --]
Hi Nitin,
[auto build test ERROR on sparc/master]
[also build test ERROR on v4.6-rc1 next-20160329]
[if your patch is applied to the wrong git tree, please drop us a note to help improving the system]
url: https://github.com/0day-ci/linux/commits/Nitin-Gupta/sparc64-Reduce-TLB-flushes-during-hugepte-changes/20160330-051327
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git master
config: sparc64-allyesconfig (attached as .config)
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sparc64
All errors (new ones prefixed by >>):
arch/sparc/mm/tlb.c: In function 'tlb_batch_pmd_scan':
>> arch/sparc/mm/tlb.c:158:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, vaddr, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
arch/sparc/mm/tlb.c: In function 'set_pmd_at':
arch/sparc/mm/tlb.c:198:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, addr, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
arch/sparc/mm/tlb.c:199:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
vim +/tlb_batch_add_one +158 arch/sparc/mm/tlb.c
9e695d2e David Miller 2012-10-08 152 pte = pte_offset_map(&pmd, vaddr);
9e695d2e David Miller 2012-10-08 153 end = vaddr + HPAGE_SIZE;
9e695d2e David Miller 2012-10-08 154 while (vaddr < end) {
5b1e94fa David S. Miller 2014-04-20 155 if (pte_val(*pte) & _PAGE_VALID) {
5b1e94fa David S. Miller 2014-04-20 156 bool exec = pte_exec(*pte);
5b1e94fa David S. Miller 2014-04-20 157
9e695d2e David Miller 2012-10-08 @158 tlb_batch_add_one(mm, vaddr, exec);
5b1e94fa David S. Miller 2014-04-20 159 }
9e695d2e David Miller 2012-10-08 160 pte++;
9e695d2e David Miller 2012-10-08 161 vaddr += PAGE_SIZE;
:::::: The code at line 158 was first introduced by commit
:::::: 9e695d2ecc8451cc2c1603d60b5c8e7f5581923a sparc64: Support transparent huge pages.
:::::: TO: David Miller <davem@davemloft.net>
:::::: CC: Linus Torvalds <torvalds@linux-foundation.org>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 45817 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
@ 2016-03-29 21:25 ` kbuild test robot
0 siblings, 0 replies; 8+ messages in thread
From: kbuild test robot @ 2016-03-29 21:25 UTC (permalink / raw)
To: sparclinux
[-- Attachment #1: Type: text/plain, Size: 2843 bytes --]
Hi Nitin,
[auto build test ERROR on sparc/master]
[also build test ERROR on v4.6-rc1 next-20160329]
[if your patch is applied to the wrong git tree, please drop us a note to help improving the system]
url: https://github.com/0day-ci/linux/commits/Nitin-Gupta/sparc64-Reduce-TLB-flushes-during-hugepte-changes/20160330-051327
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git master
config: sparc64-allyesconfig (attached as .config)
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sparc64
All errors (new ones prefixed by >>):
arch/sparc/mm/tlb.c: In function 'tlb_batch_pmd_scan':
>> arch/sparc/mm/tlb.c:158:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, vaddr, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
arch/sparc/mm/tlb.c: In function 'set_pmd_at':
arch/sparc/mm/tlb.c:198:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, addr, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
arch/sparc/mm/tlb.c:199:4: error: too few arguments to function 'tlb_batch_add_one'
tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
^
arch/sparc/mm/tlb.c:69:13: note: declared here
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
^
vim +/tlb_batch_add_one +158 arch/sparc/mm/tlb.c
9e695d2e David Miller 2012-10-08 152 pte = pte_offset_map(&pmd, vaddr);
9e695d2e David Miller 2012-10-08 153 end = vaddr + HPAGE_SIZE;
9e695d2e David Miller 2012-10-08 154 while (vaddr < end) {
5b1e94fa David S. Miller 2014-04-20 155 if (pte_val(*pte) & _PAGE_VALID) {
5b1e94fa David S. Miller 2014-04-20 156 bool exec = pte_exec(*pte);
5b1e94fa David S. Miller 2014-04-20 157
9e695d2e David Miller 2012-10-08 @158 tlb_batch_add_one(mm, vaddr, exec);
5b1e94fa David S. Miller 2014-04-20 159 }
9e695d2e David Miller 2012-10-08 160 pte++;
9e695d2e David Miller 2012-10-08 161 vaddr += PAGE_SIZE;
:::::: The code at line 158 was first introduced by commit
:::::: 9e695d2ecc8451cc2c1603d60b5c8e7f5581923a sparc64: Support transparent huge pages.
:::::: TO: David Miller <davem@davemloft.net>
:::::: CC: Linus Torvalds <torvalds@linux-foundation.org>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 45817 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
2016-03-29 21:11 ` Nitin Gupta
@ 2016-03-29 22:38 ` kbuild test robot
-1 siblings, 0 replies; 8+ messages in thread
From: kbuild test robot @ 2016-03-29 22:38 UTC (permalink / raw)
To: Nitin Gupta
Cc: kbuild-all, David S. Miller, Nitin Gupta, Andrew Morton,
Kirill A. Shutemov, Khalid Aziz, Minchan Kim, Aneesh Kumar K.V,
Zhang Zhen, Stephen Rothwell, Ingo Molnar, Chris Hyser,
David Hildenbrand, Toshi Kani, Tony Luck, sparclinux,
linux-kernel
[-- Attachment #1: Type: text/plain, Size: 1526 bytes --]
Hi Nitin,
[auto build test ERROR on sparc/master]
[also build test ERROR on v4.6-rc1 next-20160329]
[if your patch is applied to the wrong git tree, please drop us a note to help improving the system]
url: https://github.com/0day-ci/linux/commits/Nitin-Gupta/sparc64-Reduce-TLB-flushes-during-hugepte-changes/20160330-051327
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git master
config: sparc64-allnoconfig (attached as .config)
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sparc64
All errors (new ones prefixed by >>):
arch/sparc/mm/tlb.c: In function 'tlb_batch_add':
>> arch/sparc/mm/tlb.c:115:2: error: implicit declaration of function 'is_hugetlb_pte' [-Werror=implicit-function-declaration]
bool huge = is_hugetlb_pte(orig);
^
cc1: all warnings being treated as errors
vim +/is_hugetlb_pte +115 arch/sparc/mm/tlb.c
109 put_cpu_var(tlb_batch);
110 }
111
112 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
113 pte_t *ptep, pte_t orig, int fullmm)
114 {
> 115 bool huge = is_hugetlb_pte(orig);
116
117 if (tlb_type != hypervisor &&
118 pte_dirty(orig)) {
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 4864 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
@ 2016-03-29 22:38 ` kbuild test robot
0 siblings, 0 replies; 8+ messages in thread
From: kbuild test robot @ 2016-03-29 22:38 UTC (permalink / raw)
To: sparclinux
[-- Attachment #1: Type: text/plain, Size: 1526 bytes --]
Hi Nitin,
[auto build test ERROR on sparc/master]
[also build test ERROR on v4.6-rc1 next-20160329]
[if your patch is applied to the wrong git tree, please drop us a note to help improving the system]
url: https://github.com/0day-ci/linux/commits/Nitin-Gupta/sparc64-Reduce-TLB-flushes-during-hugepte-changes/20160330-051327
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git master
config: sparc64-allnoconfig (attached as .config)
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=sparc64
All errors (new ones prefixed by >>):
arch/sparc/mm/tlb.c: In function 'tlb_batch_add':
>> arch/sparc/mm/tlb.c:115:2: error: implicit declaration of function 'is_hugetlb_pte' [-Werror=implicit-function-declaration]
bool huge = is_hugetlb_pte(orig);
^
cc1: all warnings being treated as errors
vim +/is_hugetlb_pte +115 arch/sparc/mm/tlb.c
109 put_cpu_var(tlb_batch);
110 }
111
112 void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
113 pte_t *ptep, pte_t orig, int fullmm)
114 {
> 115 bool huge = is_hugetlb_pte(orig);
116
117 if (tlb_type != hypervisor &&
118 pte_dirty(orig)) {
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 4864 bytes --]
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
2016-03-29 21:11 ` Nitin Gupta
@ 2016-03-29 23:41 ` David Miller
-1 siblings, 0 replies; 8+ messages in thread
From: David Miller @ 2016-03-29 23:41 UTC (permalink / raw)
To: nitin.m.gupta
Cc: akpm, kirill.shutemov, khalid.aziz, minchan, aneesh.kumar,
zhenzhang.zhang, sfr, mingo, chris.hyser, dahi, toshi.kani,
tony.luck, sparclinux, linux-kernel
From: Nitin Gupta <nitin.m.gupta@oracle.com>
Date: Tue, 29 Mar 2016 14:11:14 -0700
> During hugepage map/unmap, TSB and TLB flushes are currently
> issued at every PAGE_SIZE'd boundary which is unnecessary.
> We now issue the flush at REAL_HPAGE_SIZE boundaries only.
>
> Without this patch workloads which unmap a large hugepage
> backed VMA region get CPU lockups due to excessive TLB
> flush calls.
>
> Orabug: 22365539, 22643230, 22995196
>
> Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
You really need to put some more work into this, there are lots of
config variants you didn't even test.
See the kbuild robot replies...
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes
@ 2016-03-29 23:41 ` David Miller
0 siblings, 0 replies; 8+ messages in thread
From: David Miller @ 2016-03-29 23:41 UTC (permalink / raw)
To: nitin.m.gupta
Cc: akpm, kirill.shutemov, khalid.aziz, minchan, aneesh.kumar,
zhenzhang.zhang, sfr, mingo, chris.hyser, dahi, toshi.kani,
tony.luck, sparclinux, linux-kernel
From: Nitin Gupta <nitin.m.gupta@oracle.com>
Date: Tue, 29 Mar 2016 14:11:14 -0700
> During hugepage map/unmap, TSB and TLB flushes are currently
> issued at every PAGE_SIZE'd boundary which is unnecessary.
> We now issue the flush at REAL_HPAGE_SIZE boundaries only.
>
> Without this patch workloads which unmap a large hugepage
> backed VMA region get CPU lockups due to excessive TLB
> flush calls.
>
> Orabug: 22365539, 22643230, 22995196
>
> Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
You really need to put some more work into this, there are lots of
config variants you didn't even test.
See the kbuild robot replies...
^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2016-03-29 23:41 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2016-03-29 21:11 [PATCH v3] sparc64: Reduce TLB flushes during hugepte changes Nitin Gupta
2016-03-29 21:11 ` Nitin Gupta
2016-03-29 21:25 ` kbuild test robot
2016-03-29 21:25 ` kbuild test robot
2016-03-29 22:38 ` kbuild test robot
2016-03-29 22:38 ` kbuild test robot
2016-03-29 23:41 ` David Miller
2016-03-29 23:41 ` David Miller
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.