All of lore.kernel.org
 help / color / mirror / Atom feed
From: Liam Howlett <liam.howlett@oracle.com>
To: "maple-tree@lists.infradead.org" <maple-tree@lists.infradead.org>,
	"linux-mm@kvack.org" <linux-mm@kvack.org>,
	"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
	Andrew Morton <akpm@linux-foundation.org>,
	"damon @ lists . linux . dev" <damon@lists.linux.dev>,
	SeongJae Park <sj@kernel.org>,
	David Hildenbrand <david@redhat.com>
Subject: [PATCH v10 65/69] nommu: remove uses of VMA linked list
Date: Tue, 21 Jun 2022 20:47:13 +0000	[thread overview]
Message-ID: <20220621204632.3370049-66-Liam.Howlett@oracle.com> (raw)
In-Reply-To: <20220621204632.3370049-1-Liam.Howlett@oracle.com>

From: "Matthew Wilcox (Oracle)" <willy@infradead.org>

Use the maple tree or VMA iterator instead.  This is faster and will allow
us to shrink the VMA.

Link: https://lkml.kernel.org/r/20220504011345.662299-50-Liam.Howlett@oracle.com
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Will Deacon <will@kernel.org>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
 mm/nommu.c | 135 +++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 101 insertions(+), 34 deletions(-)

diff --git a/mm/nommu.c b/mm/nommu.c
index 4ae252a785dc..f6b187090d95 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -557,26 +557,14 @@ void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas)
 	mas_store_prealloc(mas, NULL);
 }
 
-/*
- * add a VMA into a process's mm_struct in the appropriate place in the list
- * and tree and add to the address space's page tree also if not an anonymous
- * page
- * - should be called with mm->mmap_lock held writelocked
- */
-static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
+static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
 {
-	struct address_space *mapping;
-	struct vm_area_struct *prev;
-	MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
-
-	BUG_ON(!vma->vm_region);
-
 	mm->map_count++;
 	vma->vm_mm = mm;
 
 	/* add the VMA to the mapping */
 	if (vma->vm_file) {
-		mapping = vma->vm_file->f_mapping;
+		struct address_space *mapping = vma->vm_file->f_mapping;
 
 		i_mmap_lock_write(mapping);
 		flush_dcache_mmap_lock(mapping);
@@ -584,21 +572,52 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
 		flush_dcache_mmap_unlock(mapping);
 		i_mmap_unlock_write(mapping);
 	}
+}
 
-	prev = mas_prev(&mas, 0);
-	mas_reset(&mas);
+/*
+ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm().
+ * @mas: The maple state with preallocations.
+ * @mm: The mm_struct
+ * @vma: The vma to add
+ *
+ */
+static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm,
+			      struct vm_area_struct *vma)
+{
+	struct vm_area_struct *prev;
+
+	BUG_ON(!vma->vm_region);
+
+	setup_vma_to_mm(vma, mm);
+
+	prev = mas_prev(mas, 0);
+	mas_reset(mas);
 	/* add the VMA to the tree */
-	vma_mas_store(vma, &mas);
+	vma_mas_store(vma, mas);
 	__vma_link_list(mm, vma, prev);
 }
 
 /*
- * delete a VMA from its owning mm_struct and address space
+ * add a VMA into a process's mm_struct in the appropriate place in the list
+ * and tree and add to the address space's page tree also if not an anonymous
+ * page
+ * - should be called with mm->mmap_lock held writelocked
  */
-static void delete_vma_from_mm(struct vm_area_struct *vma)
+static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
 {
-	MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
+	MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
+
+	if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+		pr_warn("Allocation of vma tree for process %d failed\n",
+		       current->pid);
+		return -ENOMEM;
+	}
+	mas_add_vma_to_mm(&mas, mm, vma);
+	return 0;
+}
 
+static void cleanup_vma_from_mm(struct vm_area_struct *vma)
+{
 	vma->vm_mm->map_count--;
 	/* remove the VMA from the mapping */
 	if (vma->vm_file) {
@@ -611,10 +630,25 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
 		flush_dcache_mmap_unlock(mapping);
 		i_mmap_unlock_write(mapping);
 	}
+}
+/*
+ * delete a VMA from its owning mm_struct and address space
+ */
+static int delete_vma_from_mm(struct vm_area_struct *vma)
+{
+	MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
+
+	if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+		pr_warn("Allocation of vma tree for process %d failed\n",
+		       current->pid);
+		return -ENOMEM;
+	}
+	cleanup_vma_from_mm(vma);
 
 	/* remove from the MM's tree and list */
 	vma_mas_remove(vma, &mas);
 	__vma_unlink_list(vma->vm_mm, vma);
+	return 0;
 }
 
 /*
@@ -1024,6 +1058,7 @@ unsigned long do_mmap(struct file *file,
 	vm_flags_t vm_flags;
 	unsigned long capabilities, result;
 	int ret;
+	MA_STATE(mas, &current->mm->mm_mt, 0, 0);
 
 	*populate = 0;
 
@@ -1042,6 +1077,7 @@ unsigned long do_mmap(struct file *file,
 	 * now know into VMA flags */
 	vm_flags = determine_vm_flags(file, prot, flags, capabilities);
 
+
 	/* we're going to need to record the mapping */
 	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
 	if (!region)
@@ -1051,6 +1087,9 @@ unsigned long do_mmap(struct file *file,
 	if (!vma)
 		goto error_getting_vma;
 
+	if (mas_preallocate(&mas, vma, GFP_KERNEL))
+		goto error_maple_preallocate;
+
 	region->vm_usage = 1;
 	region->vm_flags = vm_flags;
 	region->vm_pgoff = pgoff;
@@ -1191,7 +1230,7 @@ unsigned long do_mmap(struct file *file,
 	current->mm->total_vm += len >> PAGE_SHIFT;
 
 share:
-	add_vma_to_mm(current->mm, vma);
+	mas_add_vma_to_mm(&mas, current->mm, vma);
 
 	/* we flush the region from the icache only when the first executable
 	 * mapping of it is made  */
@@ -1217,6 +1256,7 @@ unsigned long do_mmap(struct file *file,
 
 sharing_violation:
 	up_write(&nommu_region_sem);
+	mas_destroy(&mas);
 	pr_warn("Attempt to share mismatched mappings\n");
 	ret = -EINVAL;
 	goto error;
@@ -1233,6 +1273,14 @@ unsigned long do_mmap(struct file *file,
 			len, current->pid);
 	show_free_areas(0, NULL);
 	return -ENOMEM;
+
+error_maple_preallocate:
+	kmem_cache_free(vm_region_jar, region);
+	vm_area_free(vma);
+	pr_warn("Allocation of vma tree for process %d failed\n", current->pid);
+	show_free_areas(0, NULL);
+	return -ENOMEM;
+
 }
 
 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
@@ -1298,6 +1346,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
 	struct vm_area_struct *new;
 	struct vm_region *region;
 	unsigned long npages;
+	MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
 
 	/* we're only permitted to split anonymous regions (these should have
 	 * only a single usage on the region) */
@@ -1333,7 +1382,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
 	if (new->vm_ops && new->vm_ops->open)
 		new->vm_ops->open(new);
 
-	delete_vma_from_mm(vma);
 	down_write(&nommu_region_sem);
 	delete_nommu_region(vma->vm_region);
 	if (new_below) {
@@ -1346,8 +1394,17 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
 	add_nommu_region(vma->vm_region);
 	add_nommu_region(new->vm_region);
 	up_write(&nommu_region_sem);
-	add_vma_to_mm(mm, vma);
-	add_vma_to_mm(mm, new);
+	if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+		pr_warn("Allocation of vma tree for process %d failed\n",
+		       current->pid);
+		return -ENOMEM;
+	}
+
+	setup_vma_to_mm(vma, mm);
+	setup_vma_to_mm(new, mm);
+	mas_set_range(&mas, vma->vm_start, vma->vm_end - 1);
+	mas_store(&mas, vma);
+	vma_mas_store(new, &mas);
 	return 0;
 }
 
@@ -1363,12 +1420,14 @@ static int shrink_vma(struct mm_struct *mm,
 
 	/* adjust the VMA's pointers, which may reposition it in the MM's tree
 	 * and list */
-	delete_vma_from_mm(vma);
+	if (delete_vma_from_mm(vma))
+		return -ENOMEM;
 	if (from > vma->vm_start)
 		vma->vm_end = from;
 	else
 		vma->vm_start = to;
-	add_vma_to_mm(mm, vma);
+	if (add_vma_to_mm(mm, vma))
+		return -ENOMEM;
 
 	/* cut the backing region down to size */
 	region = vma->vm_region;
@@ -1396,9 +1455,10 @@ static int shrink_vma(struct mm_struct *mm,
  */
 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
 {
+	MA_STATE(mas, &mm->mm_mt, start, start);
 	struct vm_area_struct *vma;
 	unsigned long end;
-	int ret;
+	int ret = 0;
 
 	len = PAGE_ALIGN(len);
 	if (len == 0)
@@ -1407,7 +1467,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
 	end = start + len;
 
 	/* find the first potentially overlapping VMA */
-	vma = find_vma(mm, start);
+	vma = mas_find(&mas, end - 1);
 	if (!vma) {
 		static int limit;
 		if (limit < 5) {
@@ -1426,7 +1486,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
 				return -EINVAL;
 			if (end == vma->vm_end)
 				goto erase_whole_vma;
-			vma = vma->vm_next;
+			vma = mas_next(&mas, end - 1);
 		} while (vma);
 		return -EINVAL;
 	} else {
@@ -1448,9 +1508,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
 	}
 
 erase_whole_vma:
-	delete_vma_from_mm(vma);
+	if (delete_vma_from_mm(vma))
+		ret = -ENOMEM;
 	delete_vma(mm, vma);
-	return 0;
+	return ret;
 }
 
 int vm_munmap(unsigned long addr, size_t len)
@@ -1475,6 +1536,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
  */
 void exit_mmap(struct mm_struct *mm)
 {
+	VMA_ITERATOR(vmi, mm, 0);
 	struct vm_area_struct *vma;
 
 	if (!mm)
@@ -1482,13 +1544,18 @@ void exit_mmap(struct mm_struct *mm)
 
 	mm->total_vm = 0;
 
-	while ((vma = mm->mmap)) {
-		mm->mmap = vma->vm_next;
-		delete_vma_from_mm(vma);
+	/*
+	 * Lock the mm to avoid assert complaining even though this is the only
+	 * user of the mm
+	 */
+	mmap_write_lock(mm);
+	for_each_vma(vmi, vma) {
+		cleanup_vma_from_mm(vma);
 		delete_vma(mm, vma);
 		cond_resched();
 	}
 	__mt_destroy(&mm->mm_mt);
+	mmap_write_unlock(mm);
 }
 
 int vm_brk(unsigned long addr, unsigned long len)
-- 
2.35.1

  parent reply	other threads:[~2022-06-21 20:48 UTC|newest]

Thread overview: 87+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-06-21 20:46 [PATCH v10 00/69] Introducing the Maple Tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 01/69] Maple Tree: add new data structure Liam Howlett
2022-06-21 20:46 ` [PATCH v10 04/69] radix tree test suite: add allocation counts and size to kmem_cache Liam Howlett
2022-06-21 20:46 ` [PATCH v10 02/69] radix tree test suite: add pr_err define Liam Howlett
2022-06-21 20:46 ` [PATCH v10 03/69] radix tree test suite: add kmem_cache_set_non_kernel() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 05/69] radix tree test suite: add support for slab bulk APIs Liam Howlett
2022-06-21 20:46 ` [PATCH v10 06/69] radix tree test suite: add lockdep_is_held to header Liam Howlett
2022-06-21 20:46 ` [PATCH v10 07/69] lib/test_maple_tree: add testing for maple tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 08/69] mm: start tracking VMAs with " Liam Howlett
2022-06-21 20:46 ` [PATCH v10 10/69] mmap: use the VMA iterator in count_vma_pages_range() Liam Howlett
2022-06-21 21:13   ` David Hildenbrand
2022-06-24 13:10     ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 09/69] mm: add VMA iterator Liam Howlett
2022-06-21 21:10   ` David Hildenbrand
2022-06-23 17:03     ` Matthew Wilcox
2022-06-24 14:31       ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 11/69] mm/mmap: use the maple tree in find_vma() instead of the rbtree Liam Howlett
2022-06-21 21:04   ` David Hildenbrand
2022-06-24 13:05     ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 13/69] mm/mmap: use maple tree for unmapped_area{_topdown} Liam Howlett
2022-06-23 17:25   ` David Hildenbrand
2022-07-14  0:47     ` Liam Howlett
2022-07-09  7:29   ` Alexander Gordeev
2022-07-12  1:49     ` Liam Howlett
2022-07-19 14:20   ` Sven Schnelle
2022-07-19 14:54     ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 14/69] kernel/fork: use maple tree for dup_mmap() during forking Liam Howlett
2022-06-23 18:42   ` David Hildenbrand
2022-07-13 19:05     ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 12/69] mm/mmap: use the maple tree for find_vma_prev() instead of the rbtree Liam Howlett
2022-06-21 21:17   ` David Hildenbrand
2022-06-21 20:46 ` [PATCH v10 17/69] mm: remove rb tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 16/69] proc: remove VMA rbtree use from nommu Liam Howlett
2022-06-21 20:46 ` [PATCH v10 15/69] damon: convert __damon_va_three_regions to use the VMA iterator Liam Howlett
2022-06-23 18:47   ` David Hildenbrand
2022-06-21 20:46 ` [PATCH v10 20/69] mm: optimize find_exact_vma() to use vma_lookup() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 19/69] xen: use vma_lookup() in privcmd_ioctl_mmap() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 18/69] mmap: change zeroing of maple tree in __vma_adjust() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 23/69] mm: use maple tree operations for find_vma_intersection() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 22/69] mm/mmap: change do_brk_flags() to expand existing VMA and add do_brk_munmap() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 21/69] mm/khugepaged: optimize collapse_pte_mapped_thp() by using vma_lookup() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 25/69] mm: remove vmacache Liam Howlett
2022-06-21 20:46 ` [PATCH v10 24/69] mm/mmap: use advanced maple tree API for mmap_region() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 26/69] mm: convert vma_lookup() to use mtree_load() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 27/69] mm/mmap: move mmap_region() below do_munmap() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 28/69] mm/mmap: reorganize munmap to use maple states Liam Howlett
2022-06-21 20:47 ` [PATCH v10 31/69] arm64: Change elfcore for_each_mte_vma() to use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 29/69] mm/mmap: change do_brk_munmap() to use do_mas_align_munmap() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 30/69] arm64: remove mmap linked list from vdso Liam Howlett
2022-06-21 20:47 ` [PATCH v10 34/69] s390: remove vma linked list walks Liam Howlett
2022-06-21 20:47 ` [PATCH v10 35/69] x86: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 33/69] powerpc: remove mmap " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 32/69] parisc: remove mmap linked list from cache handling Liam Howlett
2022-06-21 20:47 ` [PATCH v10 38/69] optee: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 39/69] um: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 36/69] xtensa: remove vma linked list walks Liam Howlett
2022-06-21 20:47 ` [PATCH v10 37/69] cxl: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 41/69] exec: use VMA iterator instead of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 40/69] coredump: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 43/69] fs/proc/task_mmu: stop using linked list and highest_vm_end Liam Howlett
2022-06-21 20:47 ` [PATCH v10 42/69] fs/proc/base: use maple tree iterators in place of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 45/69] ipc/shm: use VMA iterator instead " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 44/69] userfaultfd: use maple tree iterator to iterate VMAs Liam Howlett
2022-06-21 20:47 ` [PATCH v10 47/69] perf: use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 46/69] acct: use VMA iterator instead of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 48/69] sched: use maple tree iterator to walk VMAs Liam Howlett
2022-06-21 20:47 ` [PATCH v10 49/69] fork: use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 50/69] bpf: remove VMA linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 51/69] mm/gup: use maple tree navigation instead of " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 52/69] mm/khugepaged: stop using vma " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 53/69] mm/ksm: use vma iterators instead of " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 55/69] mm/memcontrol: stop using mm->highest_vm_end Liam Howlett
2022-06-21 20:47 ` [PATCH v10 54/69] mm/madvise: use vma_find() instead of vma linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 57/69] mm/mlock: use vma iterator and maple state " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 58/69] mm/mprotect: use maple tree navigation " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 59/69] mm/mremap: use vma_find_intersection() " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 56/69] mm/mempolicy: use vma iterator & maple state " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 62/69] mm/pagewalk: use vma_find() " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 60/69] mm/msync: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 61/69] mm/oom_kill: use maple tree iterators " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 63/69] mm/swapfile: use vma iterator " Liam Howlett
2022-06-21 20:47 ` Liam Howlett [this message]
2022-06-21 20:47 ` [PATCH v10 66/69] riscv: use vma iterator for vdso Liam Howlett
2022-06-21 20:47 ` [PATCH v10 64/69] i915: use the VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 67/69] mm: remove the vma linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 69/69] mm/mmap.c: pass in mapping to __vma_link_file() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 68/69] mm/mmap: drop range_has_overlap() function Liam Howlett

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220621204632.3370049-66-Liam.Howlett@oracle.com \
    --to=liam.howlett@oracle.com \
    --cc=akpm@linux-foundation.org \
    --cc=damon@lists.linux.dev \
    --cc=david@redhat.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=maple-tree@lists.infradead.org \
    --cc=sj@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.