From: Andrew Morton <akpm@linux-foundation.org>
To: akpm@linux-foundation.org, aneesh.kumar@linux.ibm.com,
christophe.leroy@csgroup.eu, hughd@google.com,
joel@joelfernandes.org, kaleshsingh@google.com,
kirill.shutemov@linux.intel.com, linux-mm@kvack.org,
mm-commits@vger.kernel.org, mpe@ellerman.id.au,
npiggin@gmail.com, sfr@canb.auug.org.au,
torvalds@linux-foundation.org
Subject: [patch 48/54] mm/mremap: convert huge PUD move to separate helper
Date: Wed, 07 Jul 2021 18:10:06 -0700 [thread overview]
Message-ID: <20210708011006.TSpgSAvxj%akpm@linux-foundation.org> (raw)
In-Reply-To: <20210707175950.eceddb86c6c555555d4730e2@linux-foundation.org>
From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Subject: mm/mremap: convert huge PUD move to separate helper
With TRANSPARENT_HUGEPAGE_PUD enabled the kernel can find huge PUD
entries. Add a helper to move huge PUD entries on mremap().
This will be used by a later patch to optimize mremap of PUD_SIZE aligned
level 4 PTE mapped address
This also make sure we support mremap on huge PUD entries even with
CONFIG_HAVE_MOVE_PUD disabled.
[aneesh.kumar@linux.ibm.com: fix build failure with clang-10]
Link: https://lore.kernel.org/lkml/YMuOSnJsL9qkxweY@archlinux-ax161
Link: https://lkml.kernel.org/r/20210619134310.89098-1-aneesh.kumar@linux.ibm.com
Link: https://lkml.kernel.org/r/20210616045239.370802-4-aneesh.kumar@linux.ibm.com
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Hugh Dickins <hughd@google.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
---
mm/mremap.c | 80 +++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 73 insertions(+), 7 deletions(-)
--- a/mm/mremap.c~mm-mremap-convert-huge-pud-move-to-separate-helper
+++ a/mm/mremap.c
@@ -324,10 +324,61 @@ static inline bool move_normal_pud(struc
}
#endif
+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
+ unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
+{
+ spinlock_t *old_ptl, *new_ptl;
+ struct mm_struct *mm = vma->vm_mm;
+ pud_t pud;
+
+ /*
+ * The destination pud shouldn't be established, free_pgtables()
+ * should have released it.
+ */
+ if (WARN_ON_ONCE(!pud_none(*new_pud)))
+ return false;
+
+ /*
+ * We don't have to worry about the ordering of src and dst
+ * ptlocks because exclusive mmap_lock prevents deadlock.
+ */
+ old_ptl = pud_lock(vma->vm_mm, old_pud);
+ new_ptl = pud_lockptr(mm, new_pud);
+ if (new_ptl != old_ptl)
+ spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
+
+ /* Clear the pud */
+ pud = *old_pud;
+ pud_clear(old_pud);
+
+ VM_BUG_ON(!pud_none(*new_pud));
+
+ /* Set the new pud */
+ /* mark soft_ditry when we add pud level soft dirty support */
+ set_pud_at(mm, new_addr, new_pud, pud);
+ flush_pud_tlb_range(vma, old_addr, old_addr + HPAGE_PUD_SIZE);
+ if (new_ptl != old_ptl)
+ spin_unlock(new_ptl);
+ spin_unlock(old_ptl);
+
+ return true;
+}
+#else
+static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr,
+ unsigned long new_addr, pud_t *old_pud, pud_t *new_pud)
+{
+ WARN_ON_ONCE(1);
+ return false;
+
+}
+#endif
+
enum pgt_entry {
NORMAL_PMD,
HPAGE_PMD,
NORMAL_PUD,
+ HPAGE_PUD,
};
/*
@@ -347,6 +398,7 @@ static __always_inline unsigned long get
mask = PMD_MASK;
size = PMD_SIZE;
break;
+ case HPAGE_PUD:
case NORMAL_PUD:
mask = PUD_MASK;
size = PUD_SIZE;
@@ -395,6 +447,12 @@ static bool move_pgt_entry(enum pgt_entr
move_huge_pmd(vma, old_addr, new_addr, old_entry,
new_entry);
break;
+ case HPAGE_PUD:
+ moved = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
+ move_huge_pud(vma, old_addr, new_addr, old_entry,
+ new_entry);
+ break;
+
default:
WARN_ON_ONCE(1);
break;
@@ -414,6 +472,7 @@ unsigned long move_page_tables(struct vm
unsigned long extent, old_end;
struct mmu_notifier_range range;
pmd_t *old_pmd, *new_pmd;
+ pud_t *old_pud, *new_pud;
old_end = old_addr + len;
flush_cache_range(vma, old_addr, old_end);
@@ -429,15 +488,22 @@ unsigned long move_page_tables(struct vm
* PUD level if possible.
*/
extent = get_extent(NORMAL_PUD, old_addr, old_end, new_addr);
- if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
- pud_t *old_pud, *new_pud;
- old_pud = get_old_pud(vma->vm_mm, old_addr);
- if (!old_pud)
+ old_pud = get_old_pud(vma->vm_mm, old_addr);
+ if (!old_pud)
+ continue;
+ new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr);
+ if (!new_pud)
+ break;
+ if (pud_trans_huge(*old_pud) || pud_devmap(*old_pud)) {
+ if (extent == HPAGE_PUD_SIZE) {
+ move_pgt_entry(HPAGE_PUD, vma, old_addr, new_addr,
+ old_pud, new_pud, need_rmap_locks);
+ /* We ignore and continue on error? */
continue;
- new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr);
- if (!new_pud)
- break;
+ }
+ } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) {
+
if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr,
old_pud, new_pud, need_rmap_locks))
continue;
_
next prev parent reply other threads:[~2021-07-08 1:11 UTC|newest]
Thread overview: 71+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-08 0:59 incoming Andrew Morton
2021-07-08 1:07 ` [patch 01/54] lib/test: fix spelling mistakes Andrew Morton
2021-07-08 1:07 ` [patch 02/54] lib: " Andrew Morton
2021-07-08 1:07 ` [patch 03/54] lib: fix spelling mistakes in header files Andrew Morton
2021-07-08 1:07 ` [patch 04/54] hexagon: handle {,SOFT}IRQENTRY_TEXT in linker script Andrew Morton
2021-07-08 1:07 ` [patch 05/54] hexagon: use common DISCARDS macro Andrew Morton
2021-07-08 1:07 ` [patch 06/54] hexagon: select ARCH_WANT_LD_ORPHAN_WARN Andrew Morton
2021-07-08 1:07 ` [patch 07/54] mm/slub: use stackdepot to save stack trace in objects Andrew Morton
2021-07-16 7:39 ` Christoph Hellwig
2021-07-16 8:57 ` Vlastimil Babka
2021-07-16 9:12 ` Christoph Hellwig
2021-07-16 20:12 ` Linus Torvalds
2021-07-16 22:37 ` Vlastimil Babka
2021-07-17 17:34 ` Randy Dunlap
2021-07-18 7:29 ` Vlastimil Babka
2021-07-18 14:17 ` Randy Dunlap
2021-07-08 1:07 ` [patch 08/54] mmap: make mlock_future_check() global Andrew Morton
2021-07-08 1:07 ` [patch 09/54] riscv/Kconfig: make direct map manipulation options depend on MMU Andrew Morton
2021-07-08 1:07 ` [patch 10/54] set_memory: allow querying whether set_direct_map_*() is actually enabled Andrew Morton
2021-07-08 1:08 ` [patch 11/54] mm: introduce memfd_secret system call to create "secret" memory areas Andrew Morton
2021-07-08 3:13 ` Linus Torvalds
2021-07-08 5:21 ` Mike Rapoport
2021-07-08 18:38 ` Linus Torvalds
2021-07-08 20:13 ` Hagen Paul Pfeifer
2021-07-09 15:44 ` Mike Rapoport
2021-07-08 1:08 ` [patch 12/54] PM: hibernate: disable when there are active secretmem users Andrew Morton
2021-07-08 3:15 ` Linus Torvalds
2021-07-08 5:30 ` Mike Rapoport
2021-07-08 1:08 ` [patch 13/54] arch, mm: wire up memfd_secret system call where relevant Andrew Morton
2021-07-08 1:08 ` [patch 14/54] secretmem: test: add basic selftest for memfd_secret(2) Andrew Morton
2021-07-08 1:08 ` [patch 15/54] mm: fix spelling mistakes in header files Andrew Morton
2021-07-08 1:08 ` [patch 16/54] mm: add setup_initial_init_mm() helper Andrew Morton
2021-07-08 1:08 ` [patch 17/54] arc: convert to setup_initial_init_mm() Andrew Morton
2021-07-08 1:08 ` [patch 18/54] arm: " Andrew Morton
2021-07-08 1:08 ` [patch 19/54] arm64: " Andrew Morton
2021-07-08 1:08 ` [patch 20/54] csky: " Andrew Morton
2021-07-08 1:08 ` [patch 21/54] h8300: " Andrew Morton
2021-07-08 1:08 ` [patch 22/54] m68k: " Andrew Morton
2021-07-08 1:08 ` [patch 23/54] nds32: " Andrew Morton
2021-07-08 1:08 ` [patch 24/54] nios2: " Andrew Morton
2021-07-08 1:08 ` [patch 25/54] openrisc: " Andrew Morton
2021-07-08 1:08 ` [patch 26/54] powerpc: " Andrew Morton
2021-07-08 4:46 ` Christophe Leroy
2021-07-08 1:08 ` [patch 27/54] riscv: " Andrew Morton
2021-07-08 1:08 ` [patch 28/54] s390: " Andrew Morton
2021-07-08 1:09 ` [patch 29/54] sh: " Andrew Morton
2021-07-08 1:09 ` [patch 30/54] x86: " Andrew Morton
2021-07-08 1:09 ` [patch 31/54] buildid: only consider GNU notes for build ID parsing Andrew Morton
2021-07-08 1:09 ` [patch 32/54] buildid: add API to parse build ID out of buffer Andrew Morton
2021-07-08 1:09 ` [patch 33/54] buildid: stash away kernels build ID on init Andrew Morton
2021-07-08 1:09 ` [patch 34/54] dump_stack: add vmlinux build ID to stack traces Andrew Morton
2021-07-08 1:09 ` [patch 35/54] module: add printk formats to add module build ID to stacktraces Andrew Morton
2021-07-08 1:09 ` [patch 36/54] arm64: stacktrace: use %pSb for backtrace printing Andrew Morton
2021-07-08 1:09 ` [patch 37/54] x86/dumpstack: use %pSb/%pBb " Andrew Morton
2021-07-08 1:09 ` [patch 38/54] scripts/decode_stacktrace.sh: support debuginfod Andrew Morton
2021-07-08 1:09 ` [patch 39/54] scripts/decode_stacktrace.sh: silence stderr messages from addr2line/nm Andrew Morton
2021-07-08 1:09 ` [patch 40/54] scripts/decode_stacktrace.sh: indicate 'auto' can be used for base path Andrew Morton
2021-07-08 1:09 ` [patch 41/54] buildid: mark some arguments const Andrew Morton
2021-07-08 1:09 ` [patch 42/54] buildid: fix kernel-doc notation Andrew Morton
2021-07-08 1:09 ` [patch 43/54] kdump: use vmlinux_build_id to simplify Andrew Morton
2021-07-08 1:09 ` [patch 44/54] mm: rename pud_page_vaddr to pud_pgtable and make it return pmd_t * Andrew Morton
2021-07-08 1:09 ` [patch 45/54] mm: rename p4d_page_vaddr to p4d_pgtable and make it return pud_t * Andrew Morton
2021-07-08 1:09 ` [patch 46/54] selftest/mremap_test: update the test to handle pagesize other than 4K Andrew Morton
2021-07-08 1:10 ` [patch 47/54] selftest/mremap_test: avoid crash with static build Andrew Morton
2021-07-08 1:10 ` Andrew Morton [this message]
2021-07-08 1:10 ` [patch 49/54] mm/mremap: don't enable optimized PUD move if page table levels is 2 Andrew Morton
2021-07-08 1:10 ` [patch 50/54] mm/mremap: use pmd/pud_poplulate to update page table entries Andrew Morton
2021-07-08 1:10 ` [patch 51/54] mm/mremap: hold the rmap lock in write mode when moving " Andrew Morton
2021-07-08 1:10 ` [patch 52/54] mm/mremap: allow arch runtime override Andrew Morton
2021-07-08 1:10 ` [patch 53/54] powerpc/book3s64/mm: update flush_tlb_range to flush page walk cache Andrew Morton
2021-07-08 1:10 ` [patch 54/54] powerpc/mm: enable HAVE_MOVE_PMD support Andrew Morton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210708011006.TSpgSAvxj%akpm@linux-foundation.org \
--to=akpm@linux-foundation.org \
--cc=aneesh.kumar@linux.ibm.com \
--cc=christophe.leroy@csgroup.eu \
--cc=hughd@google.com \
--cc=joel@joelfernandes.org \
--cc=kaleshsingh@google.com \
--cc=kirill.shutemov@linux.intel.com \
--cc=linux-mm@kvack.org \
--cc=mm-commits@vger.kernel.org \
--cc=mpe@ellerman.id.au \
--cc=npiggin@gmail.com \
--cc=sfr@canb.auug.org.au \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).