From: Liam Howlett <liam.howlett@oracle.com>
To: "maple-tree@lists.infradead.org" <maple-tree@lists.infradead.org>,
"linux-mm@kvack.org" <linux-mm@kvack.org>,
"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
Andrew Morton <akpm@linux-foundation.org>
Cc: Liam Howlett <liam.howlett@oracle.com>
Subject: [PATCH v14 44/70] userfaultfd: use maple tree iterator to iterate VMAs
Date: Tue, 6 Sep 2022 19:48:57 +0000 [thread overview]
Message-ID: <20220906194824.2110408-45-Liam.Howlett@oracle.com> (raw)
In-Reply-To: <20220906194824.2110408-1-Liam.Howlett@oracle.com>
From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>
Don't use the mm_struct linked list or the vma->vm_next in prep for
removal.
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
---
fs/userfaultfd.c | 62 ++++++++++++++++++++++++-----------
include/linux/userfaultfd_k.h | 7 ++--
mm/mmap.c | 2 +-
3 files changed, 46 insertions(+), 25 deletions(-)
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 4de91ba9e85e..091d95ddf9a0 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -611,14 +611,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
if (release_new_ctx) {
struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm;
+ VMA_ITERATOR(vmi, mm, 0);
/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for_each_vma(vmi, vma) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~__VM_UFFD_FLAGS;
}
+ }
mmap_write_unlock(mm);
userfaultfd_ctx_put(release_new_ctx);
@@ -799,11 +801,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
return false;
}
-int userfaultfd_unmap_prep(struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- struct list_head *unmaps)
+int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct list_head *unmaps)
{
- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
+ VMA_ITERATOR(vmi, mm, start);
+ struct vm_area_struct *vma;
+
+ for_each_vma_range(vmi, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
@@ -853,6 +857,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
/* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
WRITE_ONCE(ctx->released, true);
@@ -869,7 +874,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
*/
mmap_write_lock(mm);
prev = NULL;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
cond_resched();
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & __VM_UFFD_FLAGS));
@@ -883,10 +888,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
NULL_VM_UFFD_CTX, anon_vma_name(vma));
- if (prev)
+ if (prev) {
+ mas_pause(&mas);
vma = prev;
- else
+ } else {
prev = vma;
+ }
+
vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
}
@@ -1268,6 +1276,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
bool found;
bool basic_ioctls;
unsigned long start, end, vma_end;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
user_uffdio_register = (struct uffdio_register __user *) arg;
@@ -1310,7 +1319,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
goto out;
mmap_write_lock(mm);
- vma = find_vma_prev(mm, start, &prev);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;
@@ -1335,7 +1345,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
*/
found = false;
basic_ioctls = false;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
@@ -1395,8 +1405,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
}
BUG_ON(!found);
- if (vma->vm_start < start)
- prev = vma;
+ mas_set(&mas, start);
+ prev = mas_prev(&mas, 0);
+ if (prev != vma)
+ mas_next(&mas, ULONG_MAX);
ret = 0;
do {
@@ -1426,6 +1438,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
((struct vm_userfaultfd_ctx){ ctx }),
anon_vma_name(vma));
if (prev) {
+ /* vma_merge() invalidated the mas */
+ mas_pause(&mas);
vma = prev;
goto next;
}
@@ -1433,11 +1447,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
ret = split_vma(mm, vma, start, 1);
if (ret)
break;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
if (vma->vm_end > end) {
ret = split_vma(mm, vma, end, 0);
if (ret)
break;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
next:
/*
@@ -1454,8 +1472,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
- } while (vma && vma->vm_start < end);
+ vma = mas_next(&mas, end - 1);
+ } while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
@@ -1499,6 +1517,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
bool found;
unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
@@ -1517,7 +1536,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
goto out;
mmap_write_lock(mm);
- vma = find_vma_prev(mm, start, &prev);
+ mas_set(&mas, start);
+ vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;
@@ -1542,7 +1562,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
*/
found = false;
ret = -EINVAL;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
@@ -1562,8 +1582,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
}
BUG_ON(!found);
- if (vma->vm_start < start)
- prev = vma;
+ mas_set(&mas, start);
+ prev = mas_prev(&mas, 0);
+ if (prev != vma)
+ mas_next(&mas, ULONG_MAX);
ret = 0;
do {
@@ -1632,8 +1654,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
- } while (vma && vma->vm_start < end);
+ vma = mas_next(&mas, end - 1);
+ } while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index e1b8a915e9e9..f07e6998bb68 100644
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start,
unsigned long end);
-extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- struct list_head *uf);
+extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
+ unsigned long end, struct list_head *uf);
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf);
@@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma,
return true;
}
-static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
+static inline int userfaultfd_unmap_prep(struct mm_struct *mm,
unsigned long start, unsigned long end,
struct list_head *uf)
{
diff --git a/mm/mmap.c b/mm/mmap.c
index 8b7e9d5afd38..aabd4f986ccf 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2545,7 +2545,7 @@ do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
* split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for.
*/
- error = userfaultfd_unmap_prep(vma, start, end, uf);
+ error = userfaultfd_unmap_prep(mm, start, end, uf);
if (error)
goto userfaultfd_error;
--
2.35.1
next prev parent reply other threads:[~2022-09-06 19:49 UTC|newest]
Thread overview: 89+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-06 19:48 [PATCH v14 00/70] Introducing the Maple Tree Liam Howlett
2022-09-06 19:48 ` [PATCH v14 02/70] radix tree test suite: add pr_err define Liam Howlett
2022-09-06 19:48 ` [PATCH v14 01/70] Maple Tree: add new data structure Liam Howlett
2022-09-12 8:42 ` Bagas Sanjaya
2022-09-14 10:16 ` Liam Howlett
2022-09-06 19:48 ` [PATCH v14 04/70] radix tree test suite: add allocation counts and size to kmem_cache Liam Howlett
2022-09-06 19:48 ` [PATCH v14 03/70] radix tree test suite: add kmem_cache_set_non_kernel() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 05/70] radix tree test suite: add support for slab bulk APIs Liam Howlett
2022-09-06 19:48 ` [PATCH v14 06/70] radix tree test suite: add lockdep_is_held to header Liam Howlett
2022-09-06 19:48 ` [PATCH v14 07/70] lib/test_maple_tree: add testing for maple tree Liam Howlett
2022-10-11 12:16 ` Geert Uytterhoeven
2022-10-11 12:56 ` Matthew Wilcox
2022-10-11 12:59 ` Geert Uytterhoeven
2022-10-11 16:13 ` Liam Howlett
2022-09-06 19:48 ` [PATCH v14 08/70] mm: start tracking VMAs with " Liam Howlett
2022-09-06 19:48 ` [PATCH v14 11/70] mm/mmap: use the maple tree in find_vma() instead of the rbtree Liam Howlett
2022-09-06 19:48 ` [PATCH v14 10/70] mmap: use the VMA iterator in count_vma_pages_range() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 09/70] mm: add VMA iterator Liam Howlett
2022-09-06 19:48 ` [PATCH v14 14/70] kernel/fork: use maple tree for dup_mmap() during forking Liam Howlett
2022-09-06 19:48 ` [PATCH v14 12/70] mm/mmap: use the maple tree for find_vma_prev() instead of the rbtree Liam Howlett
2022-09-06 19:48 ` [PATCH v14 13/70] mm/mmap: use maple tree for unmapped_area{_topdown} Liam Howlett
2022-09-06 19:48 ` [PATCH v14 17/70] mm: remove rb tree Liam Howlett
2022-09-06 19:48 ` [PATCH v14 16/70] proc: remove VMA rbtree use from nommu Liam Howlett
2022-09-06 19:48 ` [PATCH v14 15/70] damon: convert __damon_va_three_regions to use the VMA iterator Liam Howlett
2022-09-06 19:48 ` [PATCH v14 20/70] mm: optimize find_exact_vma() to use vma_lookup() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 18/70] mmap: change zeroing of maple tree in __vma_adjust() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 19/70] xen: use vma_lookup() in privcmd_ioctl_mmap() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 22/70] mm/mmap: change do_brk_flags() to expand existing VMA and add do_brk_munmap() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 21/70] mm/khugepaged: optimize collapse_pte_mapped_thp() by using vma_lookup() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 23/70] mm: use maple tree operations for find_vma_intersection() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 24/70] mm/mmap: use advanced maple tree API for mmap_region() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 25/70] mm: remove vmacache Liam Howlett
2022-09-06 19:48 ` [PATCH v14 26/70] mm: convert vma_lookup() to use mtree_load() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 28/70] mm/mmap: reorganize munmap to use maple states Liam Howlett
2022-09-06 19:48 ` [PATCH v14 27/70] mm/mmap: move mmap_region() below do_munmap() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 29/70] mm/mmap: change do_brk_munmap() to use do_mas_align_munmap() Liam Howlett
2022-09-06 19:48 ` [PATCH v14 30/70] arm64: remove mmap linked list from vdso Liam Howlett
2022-09-06 19:48 ` [PATCH v14 31/70] arm64: Change elfcore for_each_mte_vma() to use VMA iterator Liam Howlett
2022-09-06 19:48 ` [PATCH v14 32/70] parisc: remove mmap linked list from cache handling Liam Howlett
2022-09-06 19:48 ` [PATCH v14 33/70] powerpc: remove mmap linked list walks Liam Howlett
2022-09-06 19:48 ` [PATCH v14 34/70] s390: remove vma " Liam Howlett
2022-09-06 19:48 ` [PATCH v14 35/70] x86: " Liam Howlett
2022-09-06 19:48 ` [PATCH v14 38/70] optee: remove vma linked list walk Liam Howlett
2022-09-06 19:48 ` [PATCH v14 36/70] xtensa: remove vma linked list walks Liam Howlett
2022-09-06 19:48 ` [PATCH v14 37/70] cxl: remove vma linked list walk Liam Howlett
2022-09-06 19:48 ` [PATCH v14 39/70] um: " Liam Howlett
2022-09-06 19:48 ` [PATCH v14 41/70] exec: use VMA iterator instead of linked list Liam Howlett
2022-09-06 19:48 ` [PATCH v14 40/70] coredump: remove vma linked list walk Liam Howlett
2022-09-06 19:48 ` [PATCH v14 42/70] fs/proc/base: use the vma iterators in place of linked list Liam Howlett
2022-09-06 19:48 ` [PATCH v14 43/70] fs/proc/task_mmu: stop using linked list and highest_vm_end Liam Howlett
2022-09-06 19:48 ` Liam Howlett [this message]
2022-09-06 19:48 ` [PATCH v14 45/70] ipc/shm: use VMA iterator instead of linked list Liam Howlett
2022-09-06 19:48 ` [PATCH v14 46/70] acct: " Liam Howlett
2022-09-06 19:48 ` [PATCH v14 47/70] perf: use VMA iterator Liam Howlett
2022-09-06 19:48 ` [PATCH v14 50/70] bpf: remove VMA linked list Liam Howlett
2022-09-06 19:48 ` [PATCH v14 49/70] fork: use VMA iterator Liam Howlett
2022-09-06 19:48 ` [PATCH v14 48/70] sched: use maple tree iterator to walk VMAs Liam Howlett
2022-09-06 19:49 ` [PATCH v14 52/70] mm/khugepaged: stop using vma linked list Liam Howlett
2022-09-06 19:49 ` [PATCH v14 51/70] mm/gup: use maple tree navigation instead of " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 55/70] mm/memcontrol: stop using mm->highest_vm_end Liam Howlett
2022-09-06 19:49 ` [PATCH v14 54/70] mm/madvise: use vma_find() instead of vma linked list Liam Howlett
2022-09-06 19:49 ` [PATCH v14 53/70] mm/ksm: use vma iterators " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 57/70] mm/mlock: use vma iterator and maple state " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 56/70] mm/mempolicy: use vma iterator & " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 58/70] mm/mprotect: use maple tree navigation instead of VMA " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 59/70] mm/mremap: use vma_find_intersection() instead of vma " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 60/70] mm/msync: use vma_find() " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 61/70] mm/oom_kill: use vma iterators " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 64/70] i915: use the VMA iterator Liam Howlett
2022-09-06 19:49 ` [PATCH v14 63/70] mm/swapfile: use vma iterator instead of vma linked list Liam Howlett
2022-09-06 19:49 ` [PATCH v14 62/70] mm/pagewalk: use vma_find() " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 65/70] nommu: remove uses of VMA " Liam Howlett
2022-09-06 19:49 ` [PATCH v14 67/70] mm/vmscan: Use vma iterator instead of vm_next Liam Howlett
2022-09-12 6:55 ` Yu Zhao
2022-09-12 7:13 ` Yu Zhao
2022-09-12 7:14 ` Liam Howlett
2022-09-12 19:45 ` Andrew Morton
2022-09-12 20:01 ` Yu Zhao
2022-09-12 21:03 ` Andrew Morton
2022-09-13 3:39 ` Yu Zhao
2022-09-06 19:49 ` [PATCH v14 66/70] riscv: use vma iterator for vdso Liam Howlett
2022-09-06 19:49 ` [PATCH v14 70/70] mm/mmap.c: pass in mapping to __vma_link_file() Liam Howlett
2022-09-06 19:49 ` [PATCH v14 68/70] mm: remove the vma linked list Liam Howlett
2022-09-06 19:49 ` [PATCH v14 69/70] mm/mmap: drop range_has_overlap() function Liam Howlett
2022-09-06 23:51 ` [PATCH v14 00/70] Introducing the Maple Tree Andrew Morton
2022-09-07 0:24 ` Liam Howlett
2022-09-12 0:20 ` Andrew Morton
2022-09-15 18:03 ` Yu Zhao
2022-09-17 8:24 ` Yu Zhao
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220906194824.2110408-45-Liam.Howlett@oracle.com \
--to=liam.howlett@oracle.com \
--cc=akpm@linux-foundation.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=maple-tree@lists.infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).