[RFC PATCH 0/2] mm: tlb: unmap scalability

[RFC PATCH 0/2] mm: tlb: unmap scalability

[Aaro Koskinen]

Hello,

Here's a patch proposal to make unmap to scale linearly on architectures 
that implement tlb_start_vma() and tlb_end_vma(), by adding range 
parameters. See <http://marc.info/?l=linux-kernel&m=123610437815468&w=2> 
for the current problem.

The first patch only adds the new parameters. The second one changes the 
ARM architecture to use those parameters. A similar change should be of 
course made also for other architectures implementing those routines.

The patch was made for 2.6.29-rc7.

A.

[RFC PATCH 0/2] mm: tlb: unmap scalability

[Aaro Koskinen]

Hello,

Here's a patch proposal to make unmap to scale linearly on architectures 
that implement tlb_start_vma() and tlb_end_vma(), by adding range 
parameters. See <http://marc.info/?l=linux-kernel&m=123610437815468&w=2> 
for the current problem.

The first patch only adds the new parameters. The second one changes the 
ARM architecture to use those parameters. A similar change should be of 
course made also for other architectures implementing those routines.

The patch was made for 2.6.29-rc7.

A.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Pass the range to be teared down with tlb_start_vma() and
tlb_end_vma(). This allows architectures doing per-VMA handling to flush
only the needed range instead of the full VMA region.

This patch changes the interface only, no changes in functionality.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
 arch/alpha/include/asm/tlb.h    |    4 ++--
 arch/arm/include/asm/tlb.h      |    6 ++++--
 arch/avr32/include/asm/tlb.h    |    4 ++--
 arch/blackfin/include/asm/tlb.h |    4 ++--
 arch/cris/include/asm/tlb.h     |    4 ++--
 arch/ia64/include/asm/tlb.h     |    8 ++++----
 arch/m68k/include/asm/tlb.h     |    4 ++--
 arch/mips/include/asm/tlb.h     |    4 ++--
 arch/parisc/include/asm/tlb.h   |    4 ++--
 arch/powerpc/include/asm/tlb.h  |    4 ++--
 arch/s390/include/asm/tlb.h     |    4 ++--
 arch/sh/include/asm/tlb.h       |    4 ++--
 arch/sparc/include/asm/tlb_32.h |    4 ++--
 arch/sparc/include/asm/tlb_64.h |    4 ++--
 arch/um/include/asm/tlb.h       |    4 ++--
 arch/x86/include/asm/tlb.h      |    4 ++--
 arch/xtensa/include/asm/tlb.h   |    8 ++++----
 include/asm-frv/tlb.h           |    4 ++--
 include/asm-m32r/tlb.h          |    4 ++--
 include/asm-mn10300/tlb.h       |    4 ++--
 mm/memory.c                     |   10 +++++++---
 21 files changed, 53 insertions(+), 47 deletions(-)

diff --git a/arch/alpha/include/asm/tlb.h b/arch/alpha/include/asm/tlb.h
index c136365..26991bc 100644
--- a/arch/alpha/include/asm/tlb.h
+++ b/arch/alpha/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _ALPHA_TLB_H
 #define _ALPHA_TLB_H
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
 
 #define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..d10c9c3 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -71,14 +71,16 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
  * the vmas are adjusted to only cover the region to be torn down.
  */
 static inline void
-tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
 }
 
 static inline void
-tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
diff --git a/arch/avr32/include/asm/tlb.h b/arch/avr32/include/asm/tlb.h
index 5c55f9c..41381e9 100644
--- a/arch/avr32/include/asm/tlb.h
+++ b/arch/avr32/include/asm/tlb.h
@@ -8,10 +8,10 @@
 #ifndef __ASM_AVR32_TLB_H
 #define __ASM_AVR32_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while(0)
diff --git a/arch/blackfin/include/asm/tlb.h b/arch/blackfin/include/asm/tlb.h
index 89a12ee..cf7eb67 100644
--- a/arch/blackfin/include/asm/tlb.h
+++ b/arch/blackfin/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _BLACKFIN_TLB_H
 #define _BLACKFIN_TLB_H
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/arch/cris/include/asm/tlb.h b/arch/cris/include/asm/tlb.h
index 77384ea..87e9879 100644
--- a/arch/cris/include/asm/tlb.h
+++ b/arch/cris/include/asm/tlb.h
@@ -9,8 +9,8 @@
  * cris doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h
index 20d8a39..b1c7bbf 100644
--- a/arch/ia64/include/asm/tlb.h
+++ b/arch/ia64/include/asm/tlb.h
@@ -25,14 +25,14 @@
  *	tlb <- tlb_gather_mmu(mm, full_mm_flush);	// start unmap for address space MM
  *	{
  *	  for each vma that needs a shootdown do {
- *	    tlb_start_vma(tlb, vma);
+ *	    tlb_start_vma(tlb, vma, range_start, range_end);
  *	      for each page-table-entry PTE that needs to be removed do {
  *		tlb_remove_tlb_entry(tlb, pte, address);
  *		if (pte refers to a normal page) {
  *		  tlb_remove_page(tlb, page);
  *		}
  *	      }
- *	    tlb_end_vma(tlb, vma);
+ *	    tlb_end_vma(tlb, vma, range_start, range_end);
  *	  }
  *	}
  *	tlb_finish_mmu(tlb, start, end);	// finish unmap for address space MM
@@ -227,8 +227,8 @@ __tlb_remove_tlb_entry (struct mmu_gather *tlb, pte_t *ptep, unsigned long addre
 
 #define tlb_migrate_finish(mm)	platform_tlb_migrate_finish(mm)
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #define tlb_remove_tlb_entry(tlb, ptep, addr)		\
 do {							\
diff --git a/arch/m68k/include/asm/tlb.h b/arch/m68k/include/asm/tlb.h
index 1785cff..0363248 100644
--- a/arch/m68k/include/asm/tlb.h
+++ b/arch/m68k/include/asm/tlb.h
@@ -5,8 +5,8 @@
  * m68k doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/arch/mips/include/asm/tlb.h b/arch/mips/include/asm/tlb.h
index 80d9dfc..8491179 100644
--- a/arch/mips/include/asm/tlb.h
+++ b/arch/mips/include/asm/tlb.h
@@ -5,12 +5,12 @@
  * MIPS doesn't need any special per-pte or per-vma handling, except
  * we need to flush cache for area to be unmapped.
  */
-#define tlb_start_vma(tlb, vma) 				\
+#define tlb_start_vma(tlb, vma, range_start, range_end) 	\
 	do {							\
 		if (!tlb->fullmm)				\
 			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 	}  while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 /*
diff --git a/arch/parisc/include/asm/tlb.h b/arch/parisc/include/asm/tlb.h
index 383b1db..37c40e5 100644
--- a/arch/parisc/include/asm/tlb.h
+++ b/arch/parisc/include/asm/tlb.h
@@ -6,12 +6,12 @@ do {	if ((tlb)->fullmm)		\
 		flush_tlb_mm((tlb)->mm);\
 } while (0)
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {	if (!(tlb)->fullmm)	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 do {	if (!(tlb)->fullmm)	\
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index e20ff75..d7ab142 100644
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h
@@ -27,8 +27,8 @@
 
 struct mmu_gather;
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #if !defined(CONFIG_PPC_STD_MMU)
 
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 3d8a96d..718d16f 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -148,8 +148,8 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 #endif
 }
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)	do { } while (0)
 #define tlb_migrate_finish(mm)			do { } while (0)
 
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 88ff1ae..84ad1f9 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -7,10 +7,10 @@
 
 #ifndef __ASSEMBLY__
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address)	do { } while (0)
diff --git a/arch/sparc/include/asm/tlb_32.h b/arch/sparc/include/asm/tlb_32.h
index 6d02d1c..8161627 100644
--- a/arch/sparc/include/asm/tlb_32.h
+++ b/arch/sparc/include/asm/tlb_32.h
@@ -1,12 +1,12 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)
diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h
index ec81cde..e5d121e 100644
--- a/arch/sparc/include/asm/tlb_64.h
+++ b/arch/sparc/include/asm/tlb_64.h
@@ -105,7 +105,7 @@ static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page)
 #define pud_free_tlb(tlb,pudp) __pud_free_tlb(tlb,pudp)
 
 #define tlb_migrate_finish(mm)	do { } while (0)
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #endif /* _SPARC64_TLB_H */
diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h
index 5240fa1..a2eafcc 100644
--- a/arch/um/include/asm/tlb.h
+++ b/arch/um/include/asm/tlb.h
@@ -7,8 +7,8 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 /* struct mmu_gather is an opaque type used by the mm code for passing around
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..7421c06 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _ASM_X86_TLB_H
 #define _ASM_X86_TLB_H
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h
index 31c220f..8f99a8e 100644
--- a/arch/xtensa/include/asm/tlb.h
+++ b/arch/xtensa/include/asm/tlb.h
@@ -18,18 +18,18 @@
 
 /* Note, read http://lkml.org/lkml/2004/1/15/6 */
 
-# define tlb_start_vma(tlb,vma)			do { } while (0)
-# define tlb_end_vma(tlb,vma)			do { } while (0)
+# define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+# define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 
 #else
 
-# define tlb_start_vma(tlb, vma)					      \
+# define tlb_start_vma(tlb, vma, range_start, range_end)		      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_cache_range(vma, vma->vm_start, vma->vm_end);   \
 	} while(0)
 
-# define tlb_end_vma(tlb, vma)						      \
+# define tlb_end_vma(tlb, vma, range_start, range_end)			      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_tlb_range(vma, vma->vm_start, vma->vm_end);     \
diff --git a/include/asm-frv/tlb.h b/include/asm-frv/tlb.h
index cd458eb..8553784 100644
--- a/include/asm-frv/tlb.h
+++ b/include/asm-frv/tlb.h
@@ -12,8 +12,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/include/asm-m32r/tlb.h b/include/asm-m32r/tlb.h
index c7ebd8d..3f4c08d 100644
--- a/include/asm-m32r/tlb.h
+++ b/include/asm-m32r/tlb.h
@@ -5,8 +5,8 @@
  * x86 doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
 
 /*
diff --git a/include/asm-mn10300/tlb.h b/include/asm-mn10300/tlb.h
index 65d232b..89acf74 100644
--- a/include/asm-mn10300/tlb.h
+++ b/include/asm-mn10300/tlb.h
@@ -19,8 +19,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/mm/memory.c b/mm/memory.c
index baa999e..44996b6 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
 
 static unsigned long unmap_page_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma,
-				unsigned long addr, unsigned long end,
+				unsigned long range_start, unsigned long end,
 				long *zap_work, struct zap_details *details)
 {
 	pgd_t *pgd;
 	unsigned long next;
+	unsigned long addr = range_start;
+	unsigned long range_end;
 
 	if (details && !details->check_mapping && !details->nonlinear_vma)
 		details = NULL;
 
 	BUG_ON(addr >= end);
-	tlb_start_vma(tlb, vma);
+	BUG_ON(*zap_work <= 0);
+	range_end = addr + min(end - addr, (unsigned long)*zap_work);
+	tlb_start_vma(tlb, vma, range_start, range_end);
 	pgd = pgd_offset(vma->vm_mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);
@@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
 		next = zap_pud_range(tlb, vma, pgd, addr, next,
 						zap_work, details);
 	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
-	tlb_end_vma(tlb, vma);
+	tlb_end_vma(tlb, vma, range_start, range_end);
 
 	return addr;
 }
-- 
1.5.4.3

[RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Pass the range to be teared down with tlb_start_vma() and
tlb_end_vma(). This allows architectures doing per-VMA handling to flush
only the needed range instead of the full VMA region.

This patch changes the interface only, no changes in functionality.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
 arch/alpha/include/asm/tlb.h    |    4 ++--
 arch/arm/include/asm/tlb.h      |    6 ++++--
 arch/avr32/include/asm/tlb.h    |    4 ++--
 arch/blackfin/include/asm/tlb.h |    4 ++--
 arch/cris/include/asm/tlb.h     |    4 ++--
 arch/ia64/include/asm/tlb.h     |    8 ++++----
 arch/m68k/include/asm/tlb.h     |    4 ++--
 arch/mips/include/asm/tlb.h     |    4 ++--
 arch/parisc/include/asm/tlb.h   |    4 ++--
 arch/powerpc/include/asm/tlb.h  |    4 ++--
 arch/s390/include/asm/tlb.h     |    4 ++--
 arch/sh/include/asm/tlb.h       |    4 ++--
 arch/sparc/include/asm/tlb_32.h |    4 ++--
 arch/sparc/include/asm/tlb_64.h |    4 ++--
 arch/um/include/asm/tlb.h       |    4 ++--
 arch/x86/include/asm/tlb.h      |    4 ++--
 arch/xtensa/include/asm/tlb.h   |    8 ++++----
 include/asm-frv/tlb.h           |    4 ++--
 include/asm-m32r/tlb.h          |    4 ++--
 include/asm-mn10300/tlb.h       |    4 ++--
 mm/memory.c                     |   10 +++++++---
 21 files changed, 53 insertions(+), 47 deletions(-)

diff --git a/arch/alpha/include/asm/tlb.h b/arch/alpha/include/asm/tlb.h
index c136365..26991bc 100644
--- a/arch/alpha/include/asm/tlb.h
+++ b/arch/alpha/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _ALPHA_TLB_H
 #define _ALPHA_TLB_H
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
 
 #define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..d10c9c3 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -71,14 +71,16 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
  * the vmas are adjusted to only cover the region to be torn down.
  */
 static inline void
-tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
 }
 
 static inline void
-tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
+tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
+	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
diff --git a/arch/avr32/include/asm/tlb.h b/arch/avr32/include/asm/tlb.h
index 5c55f9c..41381e9 100644
--- a/arch/avr32/include/asm/tlb.h
+++ b/arch/avr32/include/asm/tlb.h
@@ -8,10 +8,10 @@
 #ifndef __ASM_AVR32_TLB_H
 #define __ASM_AVR32_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while(0)
diff --git a/arch/blackfin/include/asm/tlb.h b/arch/blackfin/include/asm/tlb.h
index 89a12ee..cf7eb67 100644
--- a/arch/blackfin/include/asm/tlb.h
+++ b/arch/blackfin/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _BLACKFIN_TLB_H
 #define _BLACKFIN_TLB_H
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/arch/cris/include/asm/tlb.h b/arch/cris/include/asm/tlb.h
index 77384ea..87e9879 100644
--- a/arch/cris/include/asm/tlb.h
+++ b/arch/cris/include/asm/tlb.h
@@ -9,8 +9,8 @@
  * cris doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h
index 20d8a39..b1c7bbf 100644
--- a/arch/ia64/include/asm/tlb.h
+++ b/arch/ia64/include/asm/tlb.h
@@ -25,14 +25,14 @@
  *	tlb <- tlb_gather_mmu(mm, full_mm_flush);	// start unmap for address space MM
  *	{
  *	  for each vma that needs a shootdown do {
- *	    tlb_start_vma(tlb, vma);
+ *	    tlb_start_vma(tlb, vma, range_start, range_end);
  *	      for each page-table-entry PTE that needs to be removed do {
  *		tlb_remove_tlb_entry(tlb, pte, address);
  *		if (pte refers to a normal page) {
  *		  tlb_remove_page(tlb, page);
  *		}
  *	      }
- *	    tlb_end_vma(tlb, vma);
+ *	    tlb_end_vma(tlb, vma, range_start, range_end);
  *	  }
  *	}
  *	tlb_finish_mmu(tlb, start, end);	// finish unmap for address space MM
@@ -227,8 +227,8 @@ __tlb_remove_tlb_entry (struct mmu_gather *tlb, pte_t *ptep, unsigned long addre
 
 #define tlb_migrate_finish(mm)	platform_tlb_migrate_finish(mm)
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #define tlb_remove_tlb_entry(tlb, ptep, addr)		\
 do {							\
diff --git a/arch/m68k/include/asm/tlb.h b/arch/m68k/include/asm/tlb.h
index 1785cff..0363248 100644
--- a/arch/m68k/include/asm/tlb.h
+++ b/arch/m68k/include/asm/tlb.h
@@ -5,8 +5,8 @@
  * m68k doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/arch/mips/include/asm/tlb.h b/arch/mips/include/asm/tlb.h
index 80d9dfc..8491179 100644
--- a/arch/mips/include/asm/tlb.h
+++ b/arch/mips/include/asm/tlb.h
@@ -5,12 +5,12 @@
  * MIPS doesn't need any special per-pte or per-vma handling, except
  * we need to flush cache for area to be unmapped.
  */
-#define tlb_start_vma(tlb, vma) 				\
+#define tlb_start_vma(tlb, vma, range_start, range_end) 	\
 	do {							\
 		if (!tlb->fullmm)				\
 			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 	}  while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 
 /*
diff --git a/arch/parisc/include/asm/tlb.h b/arch/parisc/include/asm/tlb.h
index 383b1db..37c40e5 100644
--- a/arch/parisc/include/asm/tlb.h
+++ b/arch/parisc/include/asm/tlb.h
@@ -6,12 +6,12 @@ do {	if ((tlb)->fullmm)		\
 		flush_tlb_mm((tlb)->mm);\
 } while (0)
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {	if (!(tlb)->fullmm)	\
 		flush_cache_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 do {	if (!(tlb)->fullmm)	\
 		flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
 } while (0)
diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index e20ff75..d7ab142 100644
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h
@@ -27,8 +27,8 @@
 
 struct mmu_gather;
 
-#define tlb_start_vma(tlb, vma)	do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #if !defined(CONFIG_PPC_STD_MMU)
 
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 3d8a96d..718d16f 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -148,8 +148,8 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 #endif
 }
 
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)	do { } while (0)
 #define tlb_migrate_finish(mm)			do { } while (0)
 
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 88ff1ae..84ad1f9 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -7,10 +7,10 @@
 
 #ifndef __ASSEMBLY__
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 	flush_cache_range(vma, vma->vm_start, vma->vm_end)
 
-#define tlb_end_vma(tlb, vma)	\
+#define tlb_end_vma(tlb, vma, range_start, range_end)	\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end)
 
 #define __tlb_remove_tlb_entry(tlb, pte, address)	do { } while (0)
diff --git a/arch/sparc/include/asm/tlb_32.h b/arch/sparc/include/asm/tlb_32.h
index 6d02d1c..8161627 100644
--- a/arch/sparc/include/asm/tlb_32.h
+++ b/arch/sparc/include/asm/tlb_32.h
@@ -1,12 +1,12 @@
 #ifndef _SPARC_TLB_H
 #define _SPARC_TLB_H
 
-#define tlb_start_vma(tlb, vma) \
+#define tlb_start_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_cache_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)
 
-#define tlb_end_vma(tlb, vma) \
+#define tlb_end_vma(tlb, vma, range_start, range_end) \
 do {								\
 	flush_tlb_range(vma, vma->vm_start, vma->vm_end);	\
 } while (0)
diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h
index ec81cde..e5d121e 100644
--- a/arch/sparc/include/asm/tlb_64.h
+++ b/arch/sparc/include/asm/tlb_64.h
@@ -105,7 +105,7 @@ static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page)
 #define pud_free_tlb(tlb,pudp) __pud_free_tlb(tlb,pudp)
 
 #define tlb_migrate_finish(mm)	do { } while (0)
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 
 #endif /* _SPARC64_TLB_H */
diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h
index 5240fa1..a2eafcc 100644
--- a/arch/um/include/asm/tlb.h
+++ b/arch/um/include/asm/tlb.h
@@ -7,8 +7,8 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
 /* struct mmu_gather is an opaque type used by the mm code for passing around
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..7421c06 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -1,8 +1,8 @@
 #ifndef _ASM_X86_TLB_H
 #define _ASM_X86_TLB_H
 
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 #define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 
diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h
index 31c220f..8f99a8e 100644
--- a/arch/xtensa/include/asm/tlb.h
+++ b/arch/xtensa/include/asm/tlb.h
@@ -18,18 +18,18 @@
 
 /* Note, read http://lkml.org/lkml/2004/1/15/6 */
 
-# define tlb_start_vma(tlb,vma)			do { } while (0)
-# define tlb_end_vma(tlb,vma)			do { } while (0)
+# define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+# define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 
 #else
 
-# define tlb_start_vma(tlb, vma)					      \
+# define tlb_start_vma(tlb, vma, range_start, range_end)		      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_cache_range(vma, vma->vm_start, vma->vm_end);   \
 	} while(0)
 
-# define tlb_end_vma(tlb, vma)						      \
+# define tlb_end_vma(tlb, vma, range_start, range_end)			      \
 	do {								      \
 		if (!tlb->fullmm)					      \
 			flush_tlb_range(vma, vma->vm_start, vma->vm_end);     \
diff --git a/include/asm-frv/tlb.h b/include/asm-frv/tlb.h
index cd458eb..8553784 100644
--- a/include/asm-frv/tlb.h
+++ b/include/asm-frv/tlb.h
@@ -12,8 +12,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/include/asm-m32r/tlb.h b/include/asm-m32r/tlb.h
index c7ebd8d..3f4c08d 100644
--- a/include/asm-m32r/tlb.h
+++ b/include/asm-m32r/tlb.h
@@ -5,8 +5,8 @@
  * x86 doesn't need any special per-pte or
  * per-vma handling..
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end) do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end) do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
 
 /*
diff --git a/include/asm-mn10300/tlb.h b/include/asm-mn10300/tlb.h
index 65d232b..89acf74 100644
--- a/include/asm-mn10300/tlb.h
+++ b/include/asm-mn10300/tlb.h
@@ -19,8 +19,8 @@ extern void check_pgt_cache(void);
 /*
  * we don't need any special per-pte or per-vma handling...
  */
-#define tlb_start_vma(tlb, vma)				do { } while (0)
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+#define tlb_start_vma(tlb, vma, range_start, range_end)	do { } while (0)
+#define tlb_end_vma(tlb, vma, range_start, range_end)	do { } while (0)
 #define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 
 /*
diff --git a/mm/memory.c b/mm/memory.c
index baa999e..44996b6 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
 
 static unsigned long unmap_page_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma,
-				unsigned long addr, unsigned long end,
+				unsigned long range_start, unsigned long end,
 				long *zap_work, struct zap_details *details)
 {
 	pgd_t *pgd;
 	unsigned long next;
+	unsigned long addr = range_start;
+	unsigned long range_end;
 
 	if (details && !details->check_mapping && !details->nonlinear_vma)
 		details = NULL;
 
 	BUG_ON(addr >= end);
-	tlb_start_vma(tlb, vma);
+	BUG_ON(*zap_work <= 0);
+	range_end = addr + min(end - addr, (unsigned long)*zap_work);
+	tlb_start_vma(tlb, vma, range_start, range_end);
 	pgd = pgd_offset(vma->vm_mm, addr);
 	do {
 		next = pgd_addr_end(addr, end);
@@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
 		next = zap_pud_range(tlb, vma, pgd, addr, next,
 						zap_work, details);
 	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
-	tlb_end_vma(tlb, vma);
+	tlb_end_vma(tlb, vma, range_start, range_end);
 
 	return addr;
 }
-- 
1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[RFC PATCH 2/2] ARM: tlb: Use range in tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Flush only the pages that were unmapped.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
 arch/arm/include/asm/tlb.h |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index d10c9c3..a034b6d 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -68,14 +68,14 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 /*
  * In the case of tlb vma handling, we can optimise these away in the
  * case where we're doing a full MM flush.  When we're doing a munmap,
- * the vmas are adjusted to only cover the region to be torn down.
+ * the range is adjusted to only cover the region to be torn down.
  */
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		flush_cache_range(vma, range_start, range_end);
 }
 
 static inline void
@@ -83,7 +83,7 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+		flush_tlb_range(vma, range_start, range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

[RFC PATCH 2/2] ARM: tlb: Use range in tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Flush only the pages that were unmapped.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
 arch/arm/include/asm/tlb.h |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index d10c9c3..a034b6d 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -68,14 +68,14 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 /*
  * In the case of tlb vma handling, we can optimise these away in the
  * case where we're doing a full MM flush.  When we're doing a munmap,
- * the vmas are adjusted to only cover the region to be torn down.
+ * the range is adjusted to only cover the region to be torn down.
  */
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	      unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		flush_cache_range(vma, range_start, range_end);
 }
 
 static inline void
@@ -83,7 +83,7 @@ tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	    unsigned long range_start, unsigned long range_end)
 {
 	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+		flush_tlb_range(vma, range_start, range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Hugh Dickins]

On Mon, 9 Mar 2009, Aaro Koskinen wrote:

> Pass the range to be teared down with tlb_start_vma() and
> tlb_end_vma(). This allows architectures doing per-VMA handling to flush
> only the needed range instead of the full VMA region.
> 
> This patch changes the interface only, no changes in functionality.
> 
> Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> ---
>  arch/alpha/include/asm/tlb.h    |    4 ++--
>  arch/arm/include/asm/tlb.h      |    6 ++++--
>  arch/avr32/include/asm/tlb.h    |    4 ++--
>  arch/blackfin/include/asm/tlb.h |    4 ++--
>  arch/cris/include/asm/tlb.h     |    4 ++--
>  arch/ia64/include/asm/tlb.h     |    8 ++++----
>  arch/m68k/include/asm/tlb.h     |    4 ++--
>  arch/mips/include/asm/tlb.h     |    4 ++--
>  arch/parisc/include/asm/tlb.h   |    4 ++--
>  arch/powerpc/include/asm/tlb.h  |    4 ++--
>  arch/s390/include/asm/tlb.h     |    4 ++--
>  arch/sh/include/asm/tlb.h       |    4 ++--
>  arch/sparc/include/asm/tlb_32.h |    4 ++--
>  arch/sparc/include/asm/tlb_64.h |    4 ++--
>  arch/um/include/asm/tlb.h       |    4 ++--
>  arch/x86/include/asm/tlb.h      |    4 ++--
>  arch/xtensa/include/asm/tlb.h   |    8 ++++----
>  include/asm-frv/tlb.h           |    4 ++--
>  include/asm-m32r/tlb.h          |    4 ++--
>  include/asm-mn10300/tlb.h       |    4 ++--
>  mm/memory.c                     |   10 +++++++---
>  21 files changed, 53 insertions(+), 47 deletions(-)
...
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
>  
>  static unsigned long unmap_page_range(struct mmu_gather *tlb,
>  				struct vm_area_struct *vma,
> -				unsigned long addr, unsigned long end,
> +				unsigned long range_start, unsigned long end,
>  				long *zap_work, struct zap_details *details)
>  {
>  	pgd_t *pgd;
>  	unsigned long next;
> +	unsigned long addr = range_start;
> +	unsigned long range_end;
>  
>  	if (details && !details->check_mapping && !details->nonlinear_vma)
>  		details = NULL;
>  
>  	BUG_ON(addr >= end);
> -	tlb_start_vma(tlb, vma);
> +	BUG_ON(*zap_work <= 0);
> +	range_end = addr + min(end - addr, (unsigned long)*zap_work);
> +	tlb_start_vma(tlb, vma, range_start, range_end);
>  	pgd = pgd_offset(vma->vm_mm, addr);
>  	do {
>  		next = pgd_addr_end(addr, end);
> @@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
>  		next = zap_pud_range(tlb, vma, pgd, addr, next,
>  						zap_work, details);
>  	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
> -	tlb_end_vma(tlb, vma);
> +	tlb_end_vma(tlb, vma, range_start, range_end);
>  
>  	return addr;
>  }
> -- 

Sorry, I don't like this second-guessing of zap_work at all (okay,
we all hate zap_work, and would love to rework the tlb mmu_gather
stuff to be preemptible, but the file truncation case has so far
discouraged us).

Take a look at the levels below, in particular zap_pte_range(),
and you'll see that zap_work is just an approximate cap upon the
amount of work being done while zapping, and is decremented by
wildly different amounts if a pte (or swap entry) is there or not.

So the range_end you calculate will usually be misleadingly
different from the actual end of the range.

I don't see that you need to change the interface and other arches
at all.  What prevents ARM from noting the first and last addresses
freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
(see arch/um/include/asm/tlb.h for an example of that), then using
that in its tlb_end_vma() TLB flushing?

Admittedly you won't know the end for cache flusing in tlb_start_vma(),
but you haven't mentioned that one as a problem, and I expect you can
devise (ARM-specific) optimizations to avoid repetition there too.

Hugh

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Hugh Dickins]

On Mon, 9 Mar 2009, Aaro Koskinen wrote:

> Pass the range to be teared down with tlb_start_vma() and
> tlb_end_vma(). This allows architectures doing per-VMA handling to flush
> only the needed range instead of the full VMA region.
> 
> This patch changes the interface only, no changes in functionality.
> 
> Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> ---
>  arch/alpha/include/asm/tlb.h    |    4 ++--
>  arch/arm/include/asm/tlb.h      |    6 ++++--
>  arch/avr32/include/asm/tlb.h    |    4 ++--
>  arch/blackfin/include/asm/tlb.h |    4 ++--
>  arch/cris/include/asm/tlb.h     |    4 ++--
>  arch/ia64/include/asm/tlb.h     |    8 ++++----
>  arch/m68k/include/asm/tlb.h     |    4 ++--
>  arch/mips/include/asm/tlb.h     |    4 ++--
>  arch/parisc/include/asm/tlb.h   |    4 ++--
>  arch/powerpc/include/asm/tlb.h  |    4 ++--
>  arch/s390/include/asm/tlb.h     |    4 ++--
>  arch/sh/include/asm/tlb.h       |    4 ++--
>  arch/sparc/include/asm/tlb_32.h |    4 ++--
>  arch/sparc/include/asm/tlb_64.h |    4 ++--
>  arch/um/include/asm/tlb.h       |    4 ++--
>  arch/x86/include/asm/tlb.h      |    4 ++--
>  arch/xtensa/include/asm/tlb.h   |    8 ++++----
>  include/asm-frv/tlb.h           |    4 ++--
>  include/asm-m32r/tlb.h          |    4 ++--
>  include/asm-mn10300/tlb.h       |    4 ++--
>  mm/memory.c                     |   10 +++++++---
>  21 files changed, 53 insertions(+), 47 deletions(-)
...
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -896,17 +896,21 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
>  
>  static unsigned long unmap_page_range(struct mmu_gather *tlb,
>  				struct vm_area_struct *vma,
> -				unsigned long addr, unsigned long end,
> +				unsigned long range_start, unsigned long end,
>  				long *zap_work, struct zap_details *details)
>  {
>  	pgd_t *pgd;
>  	unsigned long next;
> +	unsigned long addr = range_start;
> +	unsigned long range_end;
>  
>  	if (details && !details->check_mapping && !details->nonlinear_vma)
>  		details = NULL;
>  
>  	BUG_ON(addr >= end);
> -	tlb_start_vma(tlb, vma);
> +	BUG_ON(*zap_work <= 0);
> +	range_end = addr + min(end - addr, (unsigned long)*zap_work);
> +	tlb_start_vma(tlb, vma, range_start, range_end);
>  	pgd = pgd_offset(vma->vm_mm, addr);
>  	do {
>  		next = pgd_addr_end(addr, end);
> @@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
>  		next = zap_pud_range(tlb, vma, pgd, addr, next,
>  						zap_work, details);
>  	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
> -	tlb_end_vma(tlb, vma);
> +	tlb_end_vma(tlb, vma, range_start, range_end);
>  
>  	return addr;
>  }
> -- 

Sorry, I don't like this second-guessing of zap_work at all (okay,
we all hate zap_work, and would love to rework the tlb mmu_gather
stuff to be preemptible, but the file truncation case has so far
discouraged us).

Take a look at the levels below, in particular zap_pte_range(),
and you'll see that zap_work is just an approximate cap upon the
amount of work being done while zapping, and is decremented by
wildly different amounts if a pte (or swap entry) is there or not.

So the range_end you calculate will usually be misleadingly
different from the actual end of the range.

I don't see that you need to change the interface and other arches
at all.  What prevents ARM from noting the first and last addresses
freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
(see arch/um/include/asm/tlb.h for an example of that), then using
that in its tlb_end_vma() TLB flushing?

Admittedly you won't know the end for cache flusing in tlb_start_vma(),
but you haven't mentioned that one as a problem, and I expect you can
devise (ARM-specific) optimizations to avoid repetition there too.

Hugh

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Hello,

Hugh Dickins wrote:
> On Mon, 9 Mar 2009, Aaro Koskinen wrote:
>> Pass the range to be teared down with tlb_start_vma() and
>> tlb_end_vma(). This allows architectures doing per-VMA handling to flush
>> only the needed range instead of the full VMA region.
[...]
>>  static unsigned long unmap_page_range(struct mmu_gather *tlb,
>>  				struct vm_area_struct *vma,
>> -				unsigned long addr, unsigned long end,
>> +				unsigned long range_start, unsigned long end,
>>  				long *zap_work, struct zap_details *details)
>>  {
>>  	pgd_t *pgd;
>>  	unsigned long next;
>> +	unsigned long addr = range_start;
>> +	unsigned long range_end;
>>  
>>  	if (details && !details->check_mapping && !details->nonlinear_vma)
>>  		details = NULL;
>>  
>>  	BUG_ON(addr >= end);
>> -	tlb_start_vma(tlb, vma);
>> +	BUG_ON(*zap_work <= 0);
>> +	range_end = addr + min(end - addr, (unsigned long)*zap_work);
>> +	tlb_start_vma(tlb, vma, range_start, range_end);
>>  	pgd = pgd_offset(vma->vm_mm, addr);
>>  	do {
>>  		next = pgd_addr_end(addr, end);
>> @@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
>>  		next = zap_pud_range(tlb, vma, pgd, addr, next,
>>  						zap_work, details);
>>  	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
>> -	tlb_end_vma(tlb, vma);
>> +	tlb_end_vma(tlb, vma, range_start, range_end);
>>  
>>  	return addr;
>>  }
> 
> Sorry, I don't like this second-guessing of zap_work at all (okay,
> we all hate zap_work, and would love to rework the tlb mmu_gather
> stuff to be preemptible, but the file truncation case has so far
> discouraged us).
> 
> Take a look at the levels below, in particular zap_pte_range(),
> and you'll see that zap_work is just an approximate cap upon the
> amount of work being done while zapping, and is decremented by
> wildly different amounts if a pte (or swap entry) is there or not.
> 
> So the range_end you calculate will usually be misleadingly
> different from the actual end of the range.

You are right. Somehow I assumed it would simply define the maximum 
range in bytes, but I now realize it does not. So the range calculation 
is totally wrong. For tlb_end_vma() the range end would be available in 
addr, though, but that is probably irrelevant because of what you said:

> I don't see that you need to change the interface and other arches
> at all.  What prevents ARM from noting the first and last addresses
> freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
> (see arch/um/include/asm/tlb.h for an example of that), then using
> that in its tlb_end_vma() TLB flushing?

This would probably work, thanks for pointing it out. I should have 
taken a better look of the full API, not just what was implemented in ARM.

So, there's a new ARM-only patch draft below based on this idea, adding 
also linux-arm-kernel again.

> Admittedly you won't know the end for cache flusing in tlb_start_vma(),
> but you haven't mentioned that one as a problem, and I expect you can
> devise (ARM-specific) optimizations to avoid repetition there too.

Yes, the execution time of tlb_start_vma() does not depend on the range 
size, so that is a lesser problem.

Thanks,

A.

---

From: Aaro Koskinen <Aaro.Koskinen@nokia.com>
Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
  arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
  1 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..2729fb9 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -36,6 +36,8 @@
  struct mmu_gather {
  	struct mm_struct	*mm;
  	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
  };

  DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int 
full_mm_flush)

  	tlb->mm = mm;
  	tlb->fullmm = full_mm_flush;
+	tlb->range_start = TASK_SIZE;
+	tlb->range_end = 0;

  	return tlb;
  }
@@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long 
start, unsigned long end)
  	put_cpu_var(mmu_gathers);
  }

-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long 
addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}

  /*
   * In the case of tlb vma handling, we can optimise these away in the
@@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct 
vm_area_struct *vma)
  static inline void
  tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
  {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0) {
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
  }

  #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Aaro Koskinen]

Hello,

Hugh Dickins wrote:
> On Mon, 9 Mar 2009, Aaro Koskinen wrote:
>> Pass the range to be teared down with tlb_start_vma() and
>> tlb_end_vma(). This allows architectures doing per-VMA handling to flush
>> only the needed range instead of the full VMA region.
[...]
>>  static unsigned long unmap_page_range(struct mmu_gather *tlb,
>>  				struct vm_area_struct *vma,
>> -				unsigned long addr, unsigned long end,
>> +				unsigned long range_start, unsigned long end,
>>  				long *zap_work, struct zap_details *details)
>>  {
>>  	pgd_t *pgd;
>>  	unsigned long next;
>> +	unsigned long addr = range_start;
>> +	unsigned long range_end;
>>  
>>  	if (details && !details->check_mapping && !details->nonlinear_vma)
>>  		details = NULL;
>>  
>>  	BUG_ON(addr >= end);
>> -	tlb_start_vma(tlb, vma);
>> +	BUG_ON(*zap_work <= 0);
>> +	range_end = addr + min(end - addr, (unsigned long)*zap_work);
>> +	tlb_start_vma(tlb, vma, range_start, range_end);
>>  	pgd = pgd_offset(vma->vm_mm, addr);
>>  	do {
>>  		next = pgd_addr_end(addr, end);
>> @@ -917,7 +921,7 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
>>  		next = zap_pud_range(tlb, vma, pgd, addr, next,
>>  						zap_work, details);
>>  	} while (pgd++, addr = next, (addr != end && *zap_work > 0));
>> -	tlb_end_vma(tlb, vma);
>> +	tlb_end_vma(tlb, vma, range_start, range_end);
>>  
>>  	return addr;
>>  }
> 
> Sorry, I don't like this second-guessing of zap_work at all (okay,
> we all hate zap_work, and would love to rework the tlb mmu_gather
> stuff to be preemptible, but the file truncation case has so far
> discouraged us).
> 
> Take a look at the levels below, in particular zap_pte_range(),
> and you'll see that zap_work is just an approximate cap upon the
> amount of work being done while zapping, and is decremented by
> wildly different amounts if a pte (or swap entry) is there or not.
> 
> So the range_end you calculate will usually be misleadingly
> different from the actual end of the range.

You are right. Somehow I assumed it would simply define the maximum 
range in bytes, but I now realize it does not. So the range calculation 
is totally wrong. For tlb_end_vma() the range end would be available in 
addr, though, but that is probably irrelevant because of what you said:

> I don't see that you need to change the interface and other arches
> at all.  What prevents ARM from noting the first and last addresses
> freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
> (see arch/um/include/asm/tlb.h for an example of that), then using
> that in its tlb_end_vma() TLB flushing?

This would probably work, thanks for pointing it out. I should have 
taken a better look of the full API, not just what was implemented in ARM.

So, there's a new ARM-only patch draft below based on this idea, adding 
also linux-arm-kernel again.

> Admittedly you won't know the end for cache flusing in tlb_start_vma(),
> but you haven't mentioned that one as a problem, and I expect you can
> devise (ARM-specific) optimizations to avoid repetition there too.

Yes, the execution time of tlb_start_vma() does not depend on the range 
size, so that is a lesser problem.

Thanks,

A.

---

From: Aaro Koskinen <Aaro.Koskinen@nokia.com>
Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---
  arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
  1 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..2729fb9 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -36,6 +36,8 @@
  struct mmu_gather {
  	struct mm_struct	*mm;
  	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
  };

  DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int 
full_mm_flush)

  	tlb->mm = mm;
  	tlb->fullmm = full_mm_flush;
+	tlb->range_start = TASK_SIZE;
+	tlb->range_end = 0;

  	return tlb;
  }
@@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long 
start, unsigned long end)
  	put_cpu_var(mmu_gathers);
  }

-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long 
addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}

  /*
   * In the case of tlb vma handling, we can optimise these away in the
@@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct 
vm_area_struct *vma)
  static inline void
  tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
  {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0) {
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
  }

  #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Hugh Dickins]

On Mon, 9 Mar 2009, Aaro Koskinen wrote:
> Hugh Dickins wrote:
> 
> > I don't see that you need to change the interface and other arches
> > at all.  What prevents ARM from noting the first and last addresses
> > freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
> > (see arch/um/include/asm/tlb.h for an example of that), then using
> > that in its tlb_end_vma() TLB flushing?
> 
> This would probably work, thanks for pointing it out. I should have taken a
> better look of the full API, not just what was implemented in ARM.
> 
> So, there's a new ARM-only patch draft below based on this idea, adding also
> linux-arm-kernel again.

This one is much better, thank you.  I would think it more natural
to do the initialization of range_start and range_end in your
tlb_start_vma() - to complement tlb_end_vma() where you deal with
the final result - rather than in two places you have sited it;
but that's somewhat a matter of taste, your patch should work as is.

Hugh

> 
> ---
> 
> From: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA
> 
> Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> ---
>  arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
>  1 files changed, 22 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
> index 857f1df..2729fb9 100644
> --- a/arch/arm/include/asm/tlb.h
> +++ b/arch/arm/include/asm/tlb.h
> @@ -36,6 +36,8 @@
>  struct mmu_gather {
>  	struct mm_struct	*mm;
>  	unsigned int		fullmm;
> +	unsigned long		range_start;
> +	unsigned long		range_end;
>  };
> 
>  DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
> @@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int
> full_mm_flush)
> 
>  	tlb->mm = mm;
>  	tlb->fullmm = full_mm_flush;
> +	tlb->range_start = TASK_SIZE;
> +	tlb->range_end = 0;
> 
>  	return tlb;
>  }
> @@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
> unsigned long end)
>  	put_cpu_var(mmu_gathers);
>  }
> 
> -#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
> +/*
> + * Memorize the range for the TLB flush.
> + */
> +static inline void
> +tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
> +{
> +	if (!tlb->fullmm) {
> +		if (addr < tlb->range_start)
> +			tlb->range_start = addr;
> +		if (addr + PAGE_SIZE > tlb->range_end)
> +			tlb->range_end = addr + PAGE_SIZE;
> +	}
> +}
> 
>  /*
>   * In the case of tlb vma handling, we can optimise these away in the
> @@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct
> *vma)
>  static inline void
>  tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
>  {
> -	if (!tlb->fullmm)
> -		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
> +	if (!tlb->fullmm && tlb->range_end > 0) {
> +		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
> +		tlb->range_start = TASK_SIZE;
> +		tlb->range_end = 0;
> +	}
>  }
> 
>  #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
> -- 
> 1.5.4.3

Re: [RFC PATCH 1/2] mm: tlb: Add range to tlb_start_vma() and tlb_end_vma()

[Hugh Dickins]

On Mon, 9 Mar 2009, Aaro Koskinen wrote:
> Hugh Dickins wrote:
> 
> > I don't see that you need to change the interface and other arches
> > at all.  What prevents ARM from noting the first and last addresses
> > freed in its struct mmu_gather when tlb_remove_tlb_entry() is called
> > (see arch/um/include/asm/tlb.h for an example of that), then using
> > that in its tlb_end_vma() TLB flushing?
> 
> This would probably work, thanks for pointing it out. I should have taken a
> better look of the full API, not just what was implemented in ARM.
> 
> So, there's a new ARM-only patch draft below based on this idea, adding also
> linux-arm-kernel again.

This one is much better, thank you.  I would think it more natural
to do the initialization of range_start and range_end in your
tlb_start_vma() - to complement tlb_end_vma() where you deal with
the final result - rather than in two places you have sited it;
but that's somewhat a matter of taste, your patch should work as is.

Hugh

> 
> ---
> 
> From: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> Subject: [RFC PATCH] [ARM] Flush only the needed range when unmapping VMA
> 
> Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> ---
>  arch/arm/include/asm/tlb.h |   25 ++++++++++++++++++++++---
>  1 files changed, 22 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
> index 857f1df..2729fb9 100644
> --- a/arch/arm/include/asm/tlb.h
> +++ b/arch/arm/include/asm/tlb.h
> @@ -36,6 +36,8 @@
>  struct mmu_gather {
>  	struct mm_struct	*mm;
>  	unsigned int		fullmm;
> +	unsigned long		range_start;
> +	unsigned long		range_end;
>  };
> 
>  DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
> @@ -47,6 +49,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int
> full_mm_flush)
> 
>  	tlb->mm = mm;
>  	tlb->fullmm = full_mm_flush;
> +	tlb->range_start = TASK_SIZE;
> +	tlb->range_end = 0;
> 
>  	return tlb;
>  }
> @@ -63,7 +67,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
> unsigned long end)
>  	put_cpu_var(mmu_gathers);
>  }
> 
> -#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
> +/*
> + * Memorize the range for the TLB flush.
> + */
> +static inline void
> +tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
> +{
> +	if (!tlb->fullmm) {
> +		if (addr < tlb->range_start)
> +			tlb->range_start = addr;
> +		if (addr + PAGE_SIZE > tlb->range_end)
> +			tlb->range_end = addr + PAGE_SIZE;
> +	}
> +}
> 
>  /*
>   * In the case of tlb vma handling, we can optimise these away in the
> @@ -80,8 +96,11 @@ tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct
> *vma)
>  static inline void
>  tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
>  {
> -	if (!tlb->fullmm)
> -		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
> +	if (!tlb->fullmm && tlb->range_end > 0) {
> +		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
> +		tlb->range_start = TASK_SIZE;
> +		tlb->range_end = 0;
> +	}
>  }
> 
>  #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
> -- 
> 1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Aaro Koskinen]

When unmapping N pages (e.g. shared memory) the amount of TLB flushes
done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at
maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a
noticeable performance penalty when unmapping a large VMA and the system
is spending its time in flush_tlb_range().

The problem is that tlb_end_vma() is always flushing the full VMA
range. The subrange that needs to be flushed can be calculated by
tlb_remove_tlb_entry(). This approach was suggested by Hugh Dickins,
and is also used by other arches.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: linux-mm@kvack.org
---

For earlier discussion, see:
http://marc.info/?t=123609820900002&r=1&w=2
http://marc.info/?t=123660375800003&r=1&w=2

 arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..321c83e 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the
@@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
+	if (!tlb->fullmm) {
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0)
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Hugh Dickins]

On Tue, 10 Mar 2009, Aaro Koskinen wrote:

> When unmapping N pages (e.g. shared memory) the amount of TLB flushes
> done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at
> maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a
> noticeable performance penalty when unmapping a large VMA and the system
> is spending its time in flush_tlb_range().
> 
> The problem is that tlb_end_vma() is always flushing the full VMA
> range. The subrange that needs to be flushed can be calculated by
> tlb_remove_tlb_entry(). This approach was suggested by Hugh Dickins,
> and is also used by other arches.
> 
> Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
> Cc: Hugh Dickins <hugh@veritas.com>
> Cc: linux-mm@kvack.org
> ---

Looks good to me:
Acked-by: Hugh Dickins <hugh@veritas.com>

> 
> For earlier discussion, see:
> http://marc.info/?t=123609820900002&r=1&w=2
> http://marc.info/?t=123660375800003&r=1&w=2
> 
>  arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
>  1 files changed, 21 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
> index 857f1df..321c83e 100644
> --- a/arch/arm/include/asm/tlb.h
> +++ b/arch/arm/include/asm/tlb.h
> @@ -36,6 +36,8 @@
>  struct mmu_gather {
>  	struct mm_struct	*mm;
>  	unsigned int		fullmm;
> +	unsigned long		range_start;
> +	unsigned long		range_end;
>  };
>  
>  DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
> @@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
>  	put_cpu_var(mmu_gathers);
>  }
>  
> -#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
> +/*
> + * Memorize the range for the TLB flush.
> + */
> +static inline void
> +tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
> +{
> +	if (!tlb->fullmm) {
> +		if (addr < tlb->range_start)
> +			tlb->range_start = addr;
> +		if (addr + PAGE_SIZE > tlb->range_end)
> +			tlb->range_end = addr + PAGE_SIZE;
> +	}
> +}
>  
>  /*
>   * In the case of tlb vma handling, we can optimise these away in the
> @@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
>  static inline void
>  tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
>  {
> -	if (!tlb->fullmm)
> +	if (!tlb->fullmm) {
>  		flush_cache_range(vma, vma->vm_start, vma->vm_end);
> +		tlb->range_start = TASK_SIZE;
> +		tlb->range_end = 0;
> +	}
>  }
>  
>  static inline void
>  tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
>  {
> -	if (!tlb->fullmm)
> -		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
> +	if (!tlb->fullmm && tlb->range_end > 0)
> +		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
>  }
>  
>  #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
> -- 
> 1.5.4.3
> 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Russell King - ARM Linux]

On Tue, Mar 10, 2009 at 03:01:33PM +0200, Aaro Koskinen wrote:
> When unmapping N pages (e.g. shared memory) the amount of TLB flushes
> done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at
> maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a
> noticeable performance penalty when unmapping a large VMA and the system
> is spending its time in flush_tlb_range().

It would be nice to have some figures for the speedup gained by this
optimisation - is there any chance you could provide a comparison?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Aaro Koskinen]

Hello,

ext Russell King - ARM Linux wrote:
> On Tue, Mar 10, 2009 at 03:01:33PM +0200, Aaro Koskinen wrote:
>> When unmapping N pages (e.g. shared memory) the amount of TLB flushes
>> done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at
>> maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a
>> noticeable performance penalty when unmapping a large VMA and the system
>> is spending its time in flush_tlb_range().
> 
> It would be nice to have some figures for the speedup gained by this
> optimisation - is there any chance you could provide a comparison?

Here's a test on an OMAP3 board, 2.6.28 with linux-omap fixes and 
PREEMPT enabled. Without the patch:

~ # for PAGES in 1000 2000 3000 4000 5000 6000 7000 8000; do time 
./shmtst $PAGES; done
shm segment size 4096000 bytes
real    0m 0.12s
user    0m 0.02s
sys     0m 0.10s
shm segment size 8192000 bytes
real    0m 0.36s
user    0m 0.00s
sys     0m 0.35s
shm segment size 12288000 bytes
real    0m 0.71s
user    0m 0.03s
sys     0m 0.67s
shm segment size 16384000 bytes
real    0m 1.17s
user    0m 0.07s
sys     0m 1.10s
shm segment size 20480000 bytes
real    0m 1.75s
user    0m 0.03s
sys     0m 1.71s
shm segment size 24576000 bytes
real    0m 2.44s
user    0m 0.03s
sys     0m 2.39s
shm segment size 28672000 bytes
real    0m 3.24s
user    0m 0.10s
sys     0m 3.14s
shm segment size 32768000 bytes
real    0m 4.16s
user    0m 0.11s
sys     0m 4.04s

With the patch:

~ # for PAGES in 1000 2000 3000 4000 5000 6000 7000 8000; do time 
./shmtst $PAGES; done
shm segment size 4096000 bytes
real    0m 0.07s
user    0m 0.01s
sys     0m 0.05s
shm segment size 8192000 bytes
real    0m 0.13s
user    0m 0.02s
sys     0m 0.10s
shm segment size 12288000 bytes
real    0m 0.20s
user    0m 0.00s
sys     0m 0.19s
shm segment size 16384000 bytes
real    0m 0.27s
user    0m 0.04s
sys     0m 0.22s
shm segment size 20480000 bytes
real    0m 0.33s
user    0m 0.02s
sys     0m 0.31s
shm segment size 24576000 bytes
real    0m 0.40s
user    0m 0.03s
sys     0m 0.36s
shm segment size 28672000 bytes
real    0m 0.47s
user    0m 0.03s
sys     0m 0.42s
shm segment size 32768000 bytes
real    0m 0.53s
user    0m 0.09s
sys     0m 0.43s

The test program:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <sys/ipc.h>
#include <sys/shm.h>

volatile char dummy;

int main (int argc, char *argv[])
{
         void *addr;
         int i;
         int shmid;
         int pages = 0;

         if (argc == 2)
                 pages = atoi(argv[1]);

         if (pages < 0) {
                 fprintf(stderr, "usage: %s <pages>\n", argv[0]);
                 return 0;
         }

         printf("shm segment size %d bytes\n", pages*4096);

         shmid = shmget(IPC_PRIVATE, pages*4096, IPC_CREAT | 0777);
         addr = shmat(shmid, NULL, 0);
         memset(addr, 0xBA, pages*4096);
         shmdt(addr);

         addr = shmat(shmid, NULL, 0);
         for (i = 0; i < pages*4096; i += 4096)
                 dummy += *((char *)addr + i);
         shmdt(addr);

         addr = shmat(shmid, NULL, 0);
         shmctl(shmid, IPC_RMID, 0);
         shmdt(addr);

         return 0;
}

A.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Russell King - ARM Linux]

On Fri, Mar 13, 2009 at 01:36:33PM +0200, Aaro Koskinen wrote:
> shm segment size 4096000 bytes
> real    0m 0.12s
> user    0m 0.02s
> sys     0m 0.10s
> shm segment size 8192000 bytes
> real    0m 0.36s
> user    0m 0.00s
> sys     0m 0.35s
...
> shm segment size 4096000 bytes
> real    0m 0.07s
> user    0m 0.01s
> sys     0m 0.05s
> shm segment size 8192000 bytes
> real    0m 0.13s
> user    0m 0.02s
> sys     0m 0.10s

Yes, that's quite a speedup.  Can you provide some words in the patch
description which indicate that we have about a 3x speed increase for
8M mappings, which only gets better for larger mappings?

Given that this patch is now only touching ARM, is there any need to
indicate that it was CC'd to Hugh when it gets committed?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH] [ARM] Flush only the needed range when unmapping a VMA

[Aaro Koskinen]

ext Russell King - ARM Linux wrote:
> Yes, that's quite a speedup.  Can you provide some words in the patch
> description which indicate that we have about a 3x speed increase for
> 8M mappings, which only gets better for larger mappings?

Ok, I will resend the patch with updated description.

> Given that this patch is now only touching ARM, is there any need to
> indicate that it was CC'd to Hugh when it gets committed?

Probably no...

A.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH v2] [ARM] Flush only the needed range when unmapping a VMA

[Aaro Koskinen]

When unmapping N pages (e.g. shared memory) the amount of TLB flushes
done can be (N*PAGE_SIZE/ZAP_BLOCK_SIZE)*N although it should be N at
maximum. With PREEMPT kernel ZAP_BLOCK_SIZE is 8 pages, so there is a
noticeable performance penalty when unmapping a large VMA and the system
is spending its time in flush_tlb_range().

The problem is that tlb_end_vma() is always flushing the full VMA
range. The subrange that needs to be flushed can be calculated by
tlb_remove_tlb_entry(). This approach was suggested by Hugh Dickins,
and is also used by other arches.

The speed increase is roughly 3x for 8M mappings and for larger mappings
even more.

Signed-off-by: Aaro Koskinen <Aaro.Koskinen@nokia.com>
---

v2: Patch description updated, no other changes.

 arch/arm/include/asm/tlb.h |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 857f1df..321c83e 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -36,6 +36,8 @@
 struct mmu_gather {
 	struct mm_struct	*mm;
 	unsigned int		fullmm;
+	unsigned long		range_start;
+	unsigned long		range_end;
 };
 
 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -63,7 +65,19 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 	put_cpu_var(mmu_gathers);
 }
 
-#define tlb_remove_tlb_entry(tlb,ptep,address)	do { } while (0)
+/*
+ * Memorize the range for the TLB flush.
+ */
+static inline void
+tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
+{
+	if (!tlb->fullmm) {
+		if (addr < tlb->range_start)
+			tlb->range_start = addr;
+		if (addr + PAGE_SIZE > tlb->range_end)
+			tlb->range_end = addr + PAGE_SIZE;
+	}
+}
 
 /*
  * In the case of tlb vma handling, we can optimise these away in the
@@ -73,15 +87,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 static inline void
 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
+	if (!tlb->fullmm) {
 		flush_cache_range(vma, vma->vm_start, vma->vm_end);
+		tlb->range_start = TASK_SIZE;
+		tlb->range_end = 0;
+	}
 }
 
 static inline void
 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
-	if (!tlb->fullmm)
-		flush_tlb_range(vma, vma->vm_start, vma->vm_end);
+	if (!tlb->fullmm && tlb->range_end > 0)
+		flush_tlb_range(vma, tlb->range_start, tlb->range_end);
 }
 
 #define tlb_remove_page(tlb,page)	free_page_and_swap_cache(page)
-- 
1.5.4.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>