From: Liam Howlett <liam.howlett@oracle.com>
To: "maple-tree@lists.infradead.org" <maple-tree@lists.infradead.org>,
"linux-mm@kvack.org" <linux-mm@kvack.org>,
"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
Andrew Morton <akpm@linux-foundation.org>,
"damon @ lists . linux . dev" <damon@lists.linux.dev>,
SeongJae Park <sj@kernel.org>,
David Hildenbrand <david@redhat.com>
Subject: [PATCH v10 65/69] nommu: remove uses of VMA linked list
Date: Tue, 21 Jun 2022 20:47:13 +0000 [thread overview]
Message-ID: <20220621204632.3370049-66-Liam.Howlett@oracle.com> (raw)
In-Reply-To: <20220621204632.3370049-1-Liam.Howlett@oracle.com>
From: "Matthew Wilcox (Oracle)" <willy@infradead.org>
Use the maple tree or VMA iterator instead. This is faster and will allow
us to shrink the VMA.
Link: https://lkml.kernel.org/r/20220504011345.662299-50-Liam.Howlett@oracle.com
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Howells <dhowells@redhat.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Will Deacon <will@kernel.org>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
mm/nommu.c | 135 +++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 101 insertions(+), 34 deletions(-)
diff --git a/mm/nommu.c b/mm/nommu.c
index 4ae252a785dc..f6b187090d95 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -557,26 +557,14 @@ void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas)
mas_store_prealloc(mas, NULL);
}
-/*
- * add a VMA into a process's mm_struct in the appropriate place in the list
- * and tree and add to the address space's page tree also if not an anonymous
- * page
- * - should be called with mm->mmap_lock held writelocked
- */
-static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
+static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
{
- struct address_space *mapping;
- struct vm_area_struct *prev;
- MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
-
- BUG_ON(!vma->vm_region);
-
mm->map_count++;
vma->vm_mm = mm;
/* add the VMA to the mapping */
if (vma->vm_file) {
- mapping = vma->vm_file->f_mapping;
+ struct address_space *mapping = vma->vm_file->f_mapping;
i_mmap_lock_write(mapping);
flush_dcache_mmap_lock(mapping);
@@ -584,21 +572,52 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
+}
- prev = mas_prev(&mas, 0);
- mas_reset(&mas);
+/*
+ * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm().
+ * @mas: The maple state with preallocations.
+ * @mm: The mm_struct
+ * @vma: The vma to add
+ *
+ */
+static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm,
+ struct vm_area_struct *vma)
+{
+ struct vm_area_struct *prev;
+
+ BUG_ON(!vma->vm_region);
+
+ setup_vma_to_mm(vma, mm);
+
+ prev = mas_prev(mas, 0);
+ mas_reset(mas);
/* add the VMA to the tree */
- vma_mas_store(vma, &mas);
+ vma_mas_store(vma, mas);
__vma_link_list(mm, vma, prev);
}
/*
- * delete a VMA from its owning mm_struct and address space
+ * add a VMA into a process's mm_struct in the appropriate place in the list
+ * and tree and add to the address space's page tree also if not an anonymous
+ * page
+ * - should be called with mm->mmap_lock held writelocked
*/
-static void delete_vma_from_mm(struct vm_area_struct *vma)
+static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
{
- MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
+
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ return -ENOMEM;
+ }
+ mas_add_vma_to_mm(&mas, mm, vma);
+ return 0;
+}
+static void cleanup_vma_from_mm(struct vm_area_struct *vma)
+{
vma->vm_mm->map_count--;
/* remove the VMA from the mapping */
if (vma->vm_file) {
@@ -611,10 +630,25 @@ static void delete_vma_from_mm(struct vm_area_struct *vma)
flush_dcache_mmap_unlock(mapping);
i_mmap_unlock_write(mapping);
}
+}
+/*
+ * delete a VMA from its owning mm_struct and address space
+ */
+static int delete_vma_from_mm(struct vm_area_struct *vma)
+{
+ MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0);
+
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ return -ENOMEM;
+ }
+ cleanup_vma_from_mm(vma);
/* remove from the MM's tree and list */
vma_mas_remove(vma, &mas);
__vma_unlink_list(vma->vm_mm, vma);
+ return 0;
}
/*
@@ -1024,6 +1058,7 @@ unsigned long do_mmap(struct file *file,
vm_flags_t vm_flags;
unsigned long capabilities, result;
int ret;
+ MA_STATE(mas, ¤t->mm->mm_mt, 0, 0);
*populate = 0;
@@ -1042,6 +1077,7 @@ unsigned long do_mmap(struct file *file,
* now know into VMA flags */
vm_flags = determine_vm_flags(file, prot, flags, capabilities);
+
/* we're going to need to record the mapping */
region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
if (!region)
@@ -1051,6 +1087,9 @@ unsigned long do_mmap(struct file *file,
if (!vma)
goto error_getting_vma;
+ if (mas_preallocate(&mas, vma, GFP_KERNEL))
+ goto error_maple_preallocate;
+
region->vm_usage = 1;
region->vm_flags = vm_flags;
region->vm_pgoff = pgoff;
@@ -1191,7 +1230,7 @@ unsigned long do_mmap(struct file *file,
current->mm->total_vm += len >> PAGE_SHIFT;
share:
- add_vma_to_mm(current->mm, vma);
+ mas_add_vma_to_mm(&mas, current->mm, vma);
/* we flush the region from the icache only when the first executable
* mapping of it is made */
@@ -1217,6 +1256,7 @@ unsigned long do_mmap(struct file *file,
sharing_violation:
up_write(&nommu_region_sem);
+ mas_destroy(&mas);
pr_warn("Attempt to share mismatched mappings\n");
ret = -EINVAL;
goto error;
@@ -1233,6 +1273,14 @@ unsigned long do_mmap(struct file *file,
len, current->pid);
show_free_areas(0, NULL);
return -ENOMEM;
+
+error_maple_preallocate:
+ kmem_cache_free(vm_region_jar, region);
+ vm_area_free(vma);
+ pr_warn("Allocation of vma tree for process %d failed\n", current->pid);
+ show_free_areas(0, NULL);
+ return -ENOMEM;
+
}
unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
@@ -1298,6 +1346,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
struct vm_area_struct *new;
struct vm_region *region;
unsigned long npages;
+ MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end);
/* we're only permitted to split anonymous regions (these should have
* only a single usage on the region) */
@@ -1333,7 +1382,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
if (new->vm_ops && new->vm_ops->open)
new->vm_ops->open(new);
- delete_vma_from_mm(vma);
down_write(&nommu_region_sem);
delete_nommu_region(vma->vm_region);
if (new_below) {
@@ -1346,8 +1394,17 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
add_nommu_region(vma->vm_region);
add_nommu_region(new->vm_region);
up_write(&nommu_region_sem);
- add_vma_to_mm(mm, vma);
- add_vma_to_mm(mm, new);
+ if (mas_preallocate(&mas, vma, GFP_KERNEL)) {
+ pr_warn("Allocation of vma tree for process %d failed\n",
+ current->pid);
+ return -ENOMEM;
+ }
+
+ setup_vma_to_mm(vma, mm);
+ setup_vma_to_mm(new, mm);
+ mas_set_range(&mas, vma->vm_start, vma->vm_end - 1);
+ mas_store(&mas, vma);
+ vma_mas_store(new, &mas);
return 0;
}
@@ -1363,12 +1420,14 @@ static int shrink_vma(struct mm_struct *mm,
/* adjust the VMA's pointers, which may reposition it in the MM's tree
* and list */
- delete_vma_from_mm(vma);
+ if (delete_vma_from_mm(vma))
+ return -ENOMEM;
if (from > vma->vm_start)
vma->vm_end = from;
else
vma->vm_start = to;
- add_vma_to_mm(mm, vma);
+ if (add_vma_to_mm(mm, vma))
+ return -ENOMEM;
/* cut the backing region down to size */
region = vma->vm_region;
@@ -1396,9 +1455,10 @@ static int shrink_vma(struct mm_struct *mm,
*/
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
{
+ MA_STATE(mas, &mm->mm_mt, start, start);
struct vm_area_struct *vma;
unsigned long end;
- int ret;
+ int ret = 0;
len = PAGE_ALIGN(len);
if (len == 0)
@@ -1407,7 +1467,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
end = start + len;
/* find the first potentially overlapping VMA */
- vma = find_vma(mm, start);
+ vma = mas_find(&mas, end - 1);
if (!vma) {
static int limit;
if (limit < 5) {
@@ -1426,7 +1486,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
return -EINVAL;
if (end == vma->vm_end)
goto erase_whole_vma;
- vma = vma->vm_next;
+ vma = mas_next(&mas, end - 1);
} while (vma);
return -EINVAL;
} else {
@@ -1448,9 +1508,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
}
erase_whole_vma:
- delete_vma_from_mm(vma);
+ if (delete_vma_from_mm(vma))
+ ret = -ENOMEM;
delete_vma(mm, vma);
- return 0;
+ return ret;
}
int vm_munmap(unsigned long addr, size_t len)
@@ -1475,6 +1536,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
*/
void exit_mmap(struct mm_struct *mm)
{
+ VMA_ITERATOR(vmi, mm, 0);
struct vm_area_struct *vma;
if (!mm)
@@ -1482,13 +1544,18 @@ void exit_mmap(struct mm_struct *mm)
mm->total_vm = 0;
- while ((vma = mm->mmap)) {
- mm->mmap = vma->vm_next;
- delete_vma_from_mm(vma);
+ /*
+ * Lock the mm to avoid assert complaining even though this is the only
+ * user of the mm
+ */
+ mmap_write_lock(mm);
+ for_each_vma(vmi, vma) {
+ cleanup_vma_from_mm(vma);
delete_vma(mm, vma);
cond_resched();
}
__mt_destroy(&mm->mm_mt);
+ mmap_write_unlock(mm);
}
int vm_brk(unsigned long addr, unsigned long len)
--
2.35.1
next prev parent reply other threads:[~2022-06-21 20:54 UTC|newest]
Thread overview: 87+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-21 20:46 [PATCH v10 00/69] Introducing the Maple Tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 01/69] Maple Tree: add new data structure Liam Howlett
2022-06-21 20:46 ` [PATCH v10 04/69] radix tree test suite: add allocation counts and size to kmem_cache Liam Howlett
2022-06-21 20:46 ` [PATCH v10 02/69] radix tree test suite: add pr_err define Liam Howlett
2022-06-21 20:46 ` [PATCH v10 03/69] radix tree test suite: add kmem_cache_set_non_kernel() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 05/69] radix tree test suite: add support for slab bulk APIs Liam Howlett
2022-06-21 20:46 ` [PATCH v10 06/69] radix tree test suite: add lockdep_is_held to header Liam Howlett
2022-06-21 20:46 ` [PATCH v10 07/69] lib/test_maple_tree: add testing for maple tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 08/69] mm: start tracking VMAs with " Liam Howlett
2022-06-21 20:46 ` [PATCH v10 10/69] mmap: use the VMA iterator in count_vma_pages_range() Liam Howlett
2022-06-21 21:13 ` David Hildenbrand
2022-06-24 13:10 ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 09/69] mm: add VMA iterator Liam Howlett
2022-06-21 21:10 ` David Hildenbrand
2022-06-23 17:03 ` Matthew Wilcox
2022-06-24 14:31 ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 11/69] mm/mmap: use the maple tree in find_vma() instead of the rbtree Liam Howlett
2022-06-21 21:04 ` David Hildenbrand
2022-06-24 13:05 ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 13/69] mm/mmap: use maple tree for unmapped_area{_topdown} Liam Howlett
2022-06-23 17:25 ` David Hildenbrand
2022-07-14 0:47 ` Liam Howlett
2022-07-09 7:29 ` Alexander Gordeev
2022-07-12 1:49 ` Liam Howlett
2022-07-19 14:20 ` Sven Schnelle
2022-07-19 14:54 ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 14/69] kernel/fork: use maple tree for dup_mmap() during forking Liam Howlett
2022-06-23 18:42 ` David Hildenbrand
2022-07-13 19:05 ` Liam Howlett
2022-06-21 20:46 ` [PATCH v10 12/69] mm/mmap: use the maple tree for find_vma_prev() instead of the rbtree Liam Howlett
2022-06-21 21:17 ` David Hildenbrand
2022-06-21 20:46 ` [PATCH v10 17/69] mm: remove rb tree Liam Howlett
2022-06-21 20:46 ` [PATCH v10 16/69] proc: remove VMA rbtree use from nommu Liam Howlett
2022-06-21 20:46 ` [PATCH v10 15/69] damon: convert __damon_va_three_regions to use the VMA iterator Liam Howlett
2022-06-23 18:47 ` David Hildenbrand
2022-06-21 20:46 ` [PATCH v10 20/69] mm: optimize find_exact_vma() to use vma_lookup() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 19/69] xen: use vma_lookup() in privcmd_ioctl_mmap() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 18/69] mmap: change zeroing of maple tree in __vma_adjust() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 23/69] mm: use maple tree operations for find_vma_intersection() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 22/69] mm/mmap: change do_brk_flags() to expand existing VMA and add do_brk_munmap() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 21/69] mm/khugepaged: optimize collapse_pte_mapped_thp() by using vma_lookup() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 25/69] mm: remove vmacache Liam Howlett
2022-06-21 20:46 ` [PATCH v10 24/69] mm/mmap: use advanced maple tree API for mmap_region() Liam Howlett
2022-06-21 20:46 ` [PATCH v10 26/69] mm: convert vma_lookup() to use mtree_load() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 27/69] mm/mmap: move mmap_region() below do_munmap() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 28/69] mm/mmap: reorganize munmap to use maple states Liam Howlett
2022-06-21 20:47 ` [PATCH v10 31/69] arm64: Change elfcore for_each_mte_vma() to use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 29/69] mm/mmap: change do_brk_munmap() to use do_mas_align_munmap() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 30/69] arm64: remove mmap linked list from vdso Liam Howlett
2022-06-21 20:47 ` [PATCH v10 34/69] s390: remove vma linked list walks Liam Howlett
2022-06-21 20:47 ` [PATCH v10 35/69] x86: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 33/69] powerpc: remove mmap " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 32/69] parisc: remove mmap linked list from cache handling Liam Howlett
2022-06-21 20:47 ` [PATCH v10 38/69] optee: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 39/69] um: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 36/69] xtensa: remove vma linked list walks Liam Howlett
2022-06-21 20:47 ` [PATCH v10 37/69] cxl: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 41/69] exec: use VMA iterator instead of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 40/69] coredump: remove vma linked list walk Liam Howlett
2022-06-21 20:47 ` [PATCH v10 43/69] fs/proc/task_mmu: stop using linked list and highest_vm_end Liam Howlett
2022-06-21 20:47 ` [PATCH v10 42/69] fs/proc/base: use maple tree iterators in place of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 45/69] ipc/shm: use VMA iterator instead " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 44/69] userfaultfd: use maple tree iterator to iterate VMAs Liam Howlett
2022-06-21 20:47 ` [PATCH v10 47/69] perf: use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 46/69] acct: use VMA iterator instead of linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 48/69] sched: use maple tree iterator to walk VMAs Liam Howlett
2022-06-21 20:47 ` [PATCH v10 49/69] fork: use VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 50/69] bpf: remove VMA linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 51/69] mm/gup: use maple tree navigation instead of " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 52/69] mm/khugepaged: stop using vma " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 53/69] mm/ksm: use vma iterators instead of " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 55/69] mm/memcontrol: stop using mm->highest_vm_end Liam Howlett
2022-06-21 20:47 ` [PATCH v10 54/69] mm/madvise: use vma_find() instead of vma linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 57/69] mm/mlock: use vma iterator and maple state " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 58/69] mm/mprotect: use maple tree navigation " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 59/69] mm/mremap: use vma_find_intersection() " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 56/69] mm/mempolicy: use vma iterator & maple state " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 62/69] mm/pagewalk: use vma_find() " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 60/69] mm/msync: " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 61/69] mm/oom_kill: use maple tree iterators " Liam Howlett
2022-06-21 20:47 ` [PATCH v10 63/69] mm/swapfile: use vma iterator " Liam Howlett
2022-06-21 20:47 ` Liam Howlett [this message]
2022-06-21 20:47 ` [PATCH v10 66/69] riscv: use vma iterator for vdso Liam Howlett
2022-06-21 20:47 ` [PATCH v10 64/69] i915: use the VMA iterator Liam Howlett
2022-06-21 20:47 ` [PATCH v10 67/69] mm: remove the vma linked list Liam Howlett
2022-06-21 20:47 ` [PATCH v10 69/69] mm/mmap.c: pass in mapping to __vma_link_file() Liam Howlett
2022-06-21 20:47 ` [PATCH v10 68/69] mm/mmap: drop range_has_overlap() function Liam Howlett
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220621204632.3370049-66-Liam.Howlett@oracle.com \
--to=liam.howlett@oracle.com \
--cc=akpm@linux-foundation.org \
--cc=damon@lists.linux.dev \
--cc=david@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=maple-tree@lists.infradead.org \
--cc=sj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).