From: Liam Howlett <liam.howlett@oracle.com>
To: "maple-tree@lists.infradead.org" <maple-tree@lists.infradead.org>,
"linux-mm@kvack.org" <linux-mm@kvack.org>,
"linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
Andrew Morton <akpm@linux-foundation.org>
Subject: [PATCH v9 56/69] mm/mempolicy: use vma iterator & maple state instead of vma linked list
Date: Wed, 4 May 2022 01:14:03 +0000 [thread overview]
Message-ID: <20220504011345.662299-41-Liam.Howlett@oracle.com> (raw)
In-Reply-To: <20220504011345.662299-1-Liam.Howlett@oracle.com>
From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>
Reworked the way mbind_range() finds the first VMA to reuse the maple
state and limit the number of tree walks needed.
Note, this drops the VM_BUG_ON(!vma) call, which would catch a start
address higher than the last VMA. The code was written in a way that
allowed no VMA updates to occur and still return success. There should be
no functional change to this scenario with the new code.
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
mm/mempolicy.c | 56 ++++++++++++++++++++++++++++----------------------
1 file changed, 32 insertions(+), 24 deletions(-)
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 0288ffaea064..df4487767259 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -380,9 +380,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{
struct vm_area_struct *vma;
+ VMA_ITERATOR(vmi, mm, 0);
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for_each_vma(vmi, vma)
mpol_rebind_policy(vma->vm_policy, new);
mmap_write_unlock(mm);
}
@@ -649,7 +650,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma,
static int queue_pages_test_walk(unsigned long start, unsigned long end,
struct mm_walk *walk)
{
- struct vm_area_struct *vma = walk->vma;
+ struct vm_area_struct *next, *vma = walk->vma;
struct queue_pages *qp = walk->private;
unsigned long endvma = vma->vm_end;
unsigned long flags = qp->flags;
@@ -664,9 +665,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end,
/* hole at head side of range */
return -EFAULT;
}
+ next = find_vma(vma->vm_mm, vma->vm_end);
if (!(flags & MPOL_MF_DISCONTIG_OK) &&
((vma->vm_end < qp->end) &&
- (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
+ (!next || vma->vm_end < next->vm_start)))
/* hole at middle or tail of range */
return -EFAULT;
@@ -780,26 +782,24 @@ static int vma_replace_policy(struct vm_area_struct *vma,
static int mbind_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct mempolicy *new_pol)
{
+ MA_STATE(mas, &mm->mm_mt, start - 1, start - 1);
struct vm_area_struct *prev;
struct vm_area_struct *vma;
int err = 0;
pgoff_t pgoff;
- unsigned long vmstart;
- unsigned long vmend;
-
- vma = find_vma(mm, start);
- VM_BUG_ON(!vma);
- prev = vma->vm_prev;
- if (start > vma->vm_start)
- prev = vma;
+ prev = mas_find_rev(&mas, 0);
+ if (prev && (start < prev->vm_end))
+ vma = prev;
+ else
+ vma = mas_next(&mas, end - 1);
- for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) {
- vmstart = max(start, vma->vm_start);
- vmend = min(end, vma->vm_end);
+ for (; vma; vma = mas_next(&mas, end - 1)) {
+ unsigned long vmstart = max(start, vma->vm_start);
+ unsigned long vmend = min(end, vma->vm_end);
if (mpol_equal(vma_policy(vma), new_pol))
- continue;
+ goto next;
pgoff = vma->vm_pgoff +
((vmstart - vma->vm_start) >> PAGE_SHIFT);
@@ -808,6 +808,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
new_pol, vma->vm_userfaultfd_ctx,
anon_vma_name(vma));
if (prev) {
+ /* vma_merge() invalidated the mas */
+ mas_pause(&mas);
vma = prev;
goto replace;
}
@@ -815,19 +817,25 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
err = split_vma(vma->vm_mm, vma, vmstart, 1);
if (err)
goto out;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
if (vma->vm_end != vmend) {
err = split_vma(vma->vm_mm, vma, vmend, 0);
if (err)
goto out;
+ /* split_vma() invalidated the mas */
+ mas_pause(&mas);
}
- replace:
+replace:
err = vma_replace_policy(vma, new_pol);
if (err)
goto out;
+next:
+ prev = vma;
}
- out:
+out:
return err;
}
@@ -1042,6 +1050,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
int flags)
{
nodemask_t nmask;
+ struct vm_area_struct *vma;
LIST_HEAD(pagelist);
int err = 0;
struct migration_target_control mtc = {
@@ -1057,8 +1066,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
* need migration. Between passing in the full user address
* space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
*/
+ vma = find_vma(mm, 0);
VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
- queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
+ queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
flags | MPOL_MF_DISCONTIG_OK, &pagelist);
if (!list_empty(&pagelist)) {
@@ -1188,14 +1198,13 @@ static struct page *new_page(struct page *page, unsigned long start)
struct folio *dst, *src = page_folio(page);
struct vm_area_struct *vma;
unsigned long address;
+ VMA_ITERATOR(vmi, current->mm, start);
gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL;
- vma = find_vma(current->mm, start);
- while (vma) {
+ for_each_vma(vmi, vma) {
address = page_address_in_vma(page, vma);
if (address != -EFAULT)
break;
- vma = vma->vm_next;
}
if (folio_test_hugetlb(src))
@@ -1473,6 +1482,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le
unsigned long vmend;
unsigned long end;
int err = -ENOENT;
+ VMA_ITERATOR(vmi, mm, start);
start = untagged_addr(start);
if (start & ~PAGE_MASK)
@@ -1498,9 +1508,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le
if (end == start)
return 0;
mmap_write_lock(mm);
- vma = find_vma(mm, start);
- for (; vma && vma->vm_start < end; vma = vma->vm_next) {
-
+ for_each_vma_range(vmi, vma, end) {
vmstart = max(start, vma->vm_start);
vmend = min(end, vma->vm_end);
new = mpol_dup(vma_policy(vma));
--
2.35.1
next prev parent reply other threads:[~2022-05-04 1:17 UTC|newest]
Thread overview: 83+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-05-04 0:26 [PATCH 0/1] Prepare for maple tree Liam Howlett
2022-05-04 0:26 ` [PATCH 1/1] mips: rename mt_init to mips_mt_init Liam Howlett
2022-05-12 9:54 ` David Hildenbrand
2022-05-04 1:12 ` [PATCH v9 15/69] damon: Convert __damon_va_three_regions to use the VMA iterator Liam Howlett
2022-05-10 10:44 ` SeongJae Park
2022-05-10 16:27 ` Liam Howlett
2022-05-10 19:13 ` Andrew Morton
2022-05-04 1:13 ` [PATCH v9 16/69] proc: remove VMA rbtree use from nommu Liam Howlett
2022-05-04 1:13 ` [PATCH v9 17/69] mm: remove rb tree Liam Howlett
2022-05-04 1:13 ` [PATCH v9 19/69] xen: use vma_lookup() in privcmd_ioctl_mmap() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 20/69] mm: optimize find_exact_vma() to use vma_lookup() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 18/69] mmap: change zeroing of maple tree in __vma_adjust() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 23/69] mm: use maple tree operations for find_vma_intersection() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 21/69] mm/khugepaged: optimize collapse_pte_mapped_thp() by using vma_lookup() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 22/69] mm/mmap: change do_brk_flags() to expand existing VMA and add do_brk_munmap() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 24/69] mm/mmap: use advanced maple tree API for mmap_region() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 25/69] mm: remove vmacache Liam Howlett
2022-05-04 1:13 ` [PATCH v9 27/69] mm/mmap: move mmap_region() below do_munmap() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 26/69] mm: convert vma_lookup() to use mtree_load() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 28/69] mm/mmap: reorganize munmap to use maple states Liam Howlett
2022-06-06 12:09 ` Qian Cai
2022-06-06 16:19 ` Liam Howlett
2022-06-06 16:40 ` Qian Cai
2022-06-11 20:11 ` Yu Zhao
2022-06-11 21:49 ` Yu Zhao
2022-06-12 1:09 ` Liam Howlett
2022-06-15 14:25 ` Liam Howlett
2022-06-15 18:07 ` Yu Zhao
2022-06-15 18:55 ` Liam Howlett
2022-06-15 19:05 ` Yu Zhao
2022-06-15 21:16 ` Yu Zhao
2022-06-16 1:50 ` Liam Howlett
2022-06-16 1:58 ` Yu Zhao
2022-06-16 2:56 ` Liam Howlett
2022-06-16 3:02 ` Yu Zhao
2022-06-16 5:45 ` Yu Zhao
2022-06-16 5:55 ` Yu Zhao
2022-06-16 18:26 ` Liam Howlett
2022-06-16 18:34 ` Yu Zhao
2022-06-17 13:49 ` Liam Howlett
2022-05-04 1:13 ` [PATCH v9 29/69] mm/mmap: change do_brk_munmap() to use do_mas_align_munmap() Liam Howlett
2022-05-04 1:13 ` [PATCH v9 30/69] arm64: remove mmap linked list from vdso Liam Howlett
2022-05-04 1:13 ` [PATCH v9 31/69] arm64: Change elfcore for_each_mte_vma() to use VMA iterator Liam Howlett
2022-05-04 1:13 ` [PATCH v9 33/69] powerpc: remove mmap linked list walks Liam Howlett
2022-05-04 1:13 ` [PATCH v9 32/69] parisc: remove mmap linked list from cache handling Liam Howlett
2022-05-04 1:13 ` [PATCH v9 35/69] x86: remove vma linked list walks Liam Howlett
2022-05-04 1:13 ` [PATCH v9 36/69] xtensa: " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 34/69] s390: " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 38/69] optee: remove vma linked list walk Liam Howlett
2022-05-04 1:13 ` [PATCH v9 39/69] um: " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 37/69] cxl: " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 40/69] coredump: " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 41/69] exec: use VMA iterator instead of linked list Liam Howlett
2022-05-04 1:13 ` [PATCH v9 42/69] fs/proc/base: use maple tree iterators in place " Liam Howlett
2022-05-04 1:13 ` [PATCH v9 43/69] fs/proc/task_mmu: stop using linked list and highest_vm_end Liam Howlett
2022-05-04 1:13 ` [PATCH v9 45/69] ipc/shm: use VMA iterator instead of linked list Liam Howlett
2022-05-04 1:13 ` [PATCH v9 44/69] userfaultfd: use maple tree iterator to iterate VMAs Liam Howlett
2022-05-04 1:14 ` [PATCH v9 47/69] perf: use VMA iterator Liam Howlett
2022-05-04 1:14 ` [PATCH v9 46/69] acct: use VMA iterator instead of linked list Liam Howlett
2022-05-04 1:14 ` [PATCH v9 48/69] sched: use maple tree iterator to walk VMAs Liam Howlett
2022-05-04 1:14 ` [PATCH v9 50/69] bpf: remove VMA linked list Liam Howlett
2022-05-04 1:14 ` [PATCH v9 49/69] fork: use VMA iterator Liam Howlett
2022-05-04 1:14 ` [PATCH v9 52/69] mm/khugepaged: stop using vma linked list Liam Howlett
2022-05-04 1:14 ` [PATCH v9 51/69] mm/gup: use maple tree navigation instead of " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 53/69] mm/ksm: use vma iterators instead of vma " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 54/69] mm/madvise: use vma_find() " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 55/69] mm/memcontrol: stop using mm->highest_vm_end Liam Howlett
2022-05-04 1:14 ` Liam Howlett [this message]
2022-05-04 1:14 ` [PATCH v9 59/69] mm/mremap: use vma_find_intersection() instead of vma linked list Liam Howlett
2022-05-04 1:14 ` [PATCH v9 57/69] mm/mlock: use vma iterator and maple state " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 58/69] mm/mprotect: use maple tree navigation " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 60/69] mm/msync: use vma_find() " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 61/69] mm/oom_kill: use maple tree iterators " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 62/69] mm/pagewalk: use vma_find() " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 63/69] mm/swapfile: use vma iterator " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 65/69] nommu: remove uses of VMA " Liam Howlett
2022-05-04 1:14 ` [PATCH v9 64/69] i915: use the VMA iterator Liam Howlett
2022-05-04 1:14 ` [PATCH v9 66/69] riscv: use vma iterator for vdso Liam Howlett
2022-05-04 1:14 ` [PATCH v9 68/69] mm/mmap: drop range_has_overlap() function Liam Howlett
2022-05-04 1:14 ` [PATCH v9 67/69] mm: remove the vma linked list Liam Howlett
2022-05-13 13:30 ` Qian Cai
2022-05-13 14:17 ` Liam Howlett
2022-05-04 1:14 ` [PATCH v9 69/69] mm/mmap.c: pass in mapping to __vma_link_file() Liam Howlett
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220504011345.662299-41-Liam.Howlett@oracle.com \
--to=liam.howlett@oracle.com \
--cc=akpm@linux-foundation.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=maple-tree@lists.infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).