From: peterx@redhat.com
To: linux-mm@kvack.org, linux-kernel@vger.kernel.org
Cc: James Houghton <jthoughton@google.com>,
David Hildenbrand <david@redhat.com>,
Yang Shi <shy828301@gmail.com>,
peterx@redhat.com, Andrew Jones <andrew.jones@linux.dev>,
linux-riscv@lists.infradead.org,
Andrea Arcangeli <aarcange@redhat.com>,
Christoph Hellwig <hch@infradead.org>,
Matthew Wilcox <willy@infradead.org>,
"Aneesh Kumar K . V" <aneesh.kumar@kernel.org>,
linux-arm-kernel@lists.infradead.org,
Jason Gunthorpe <jgg@nvidia.com>,
Axel Rasmussen <axelrasmussen@google.com>,
Rik van Riel <riel@surriel.com>,
John Hubbard <jhubbard@nvidia.com>,
"Kirill A . Shutemov" <kirill@shutemov.name>,
Vlastimil Babka <vbabka@suse.cz>,
Lorenzo Stoakes <lstoakes@gmail.com>,
Muchun Song <muchun.song@linux.dev>,
Andrew Morton <akpm@linux-foundation.org>,
linuxppc-dev@lists.ozlabs.org, Mike Rapoport <rppt@kernel.org>,
Mike Kravetz <mike.kravetz@oracle.com>
Subject: [PATCH v4 08/13] mm/gup: Handle hugetlb for no_page_table()
Date: Wed, 27 Mar 2024 11:23:27 -0400 [thread overview]
Message-ID: <20240327152332.950956-9-peterx@redhat.com> (raw)
In-Reply-To: <20240327152332.950956-1-peterx@redhat.com>
From: Peter Xu <peterx@redhat.com>
no_page_table() is not yet used for hugetlb code paths. Make it prepared.
The major difference here is hugetlb will return -EFAULT as long as page
cache does not exist, even if VM_SHARED. See hugetlb_follow_page_mask().
Pass "address" into no_page_table() too, as hugetlb will need it.
Reviewed-by: Christoph Hellwig <hch@infradead.org>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Peter Xu <peterx@redhat.com>
---
mm/gup.c | 44 ++++++++++++++++++++++++++------------------
1 file changed, 26 insertions(+), 18 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index c2881772216b..ef46a7053e16 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -501,19 +501,27 @@ static inline void mm_set_has_pinned_flag(unsigned long *mm_flags)
#ifdef CONFIG_MMU
static struct page *no_page_table(struct vm_area_struct *vma,
- unsigned int flags)
+ unsigned int flags, unsigned long address)
{
+ if (!(flags & FOLL_DUMP))
+ return NULL;
+
/*
- * When core dumping an enormous anonymous area that nobody
- * has touched so far, we don't want to allocate unnecessary pages or
+ * When core dumping, we don't want to allocate unnecessary pages or
* page tables. Return error instead of NULL to skip handle_mm_fault,
* then get_dump_page() will return NULL to leave a hole in the dump.
* But we can only make this optimization where a hole would surely
* be zero-filled if handle_mm_fault() actually did handle it.
*/
- if ((flags & FOLL_DUMP) &&
- (vma_is_anonymous(vma) || !vma->vm_ops->fault))
+ if (is_vm_hugetlb_page(vma)) {
+ struct hstate *h = hstate_vma(vma);
+
+ if (!hugetlbfs_pagecache_present(h, vma, address))
+ return ERR_PTR(-EFAULT);
+ } else if ((vma_is_anonymous(vma) || !vma->vm_ops->fault)) {
return ERR_PTR(-EFAULT);
+ }
+
return NULL;
}
@@ -593,7 +601,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
if (!ptep)
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
pte = ptep_get(ptep);
if (!pte_present(pte))
goto no_page;
@@ -685,7 +693,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
}
static struct page *follow_pmd_mask(struct vm_area_struct *vma,
@@ -701,27 +709,27 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
pmd = pmd_offset(pudp, address);
pmdval = pmdp_get_lockless(pmd);
if (pmd_none(pmdval))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
if (!pmd_present(pmdval))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
if (pmd_devmap(pmdval)) {
ptl = pmd_lock(mm, pmd);
page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
spin_unlock(ptl);
if (page)
return page;
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
}
if (likely(!pmd_trans_huge(pmdval)))
return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
ptl = pmd_lock(mm, pmd);
if (unlikely(!pmd_present(*pmd))) {
spin_unlock(ptl);
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
}
if (unlikely(!pmd_trans_huge(*pmd))) {
spin_unlock(ptl);
@@ -752,17 +760,17 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
pud = pud_offset(p4dp, address);
if (pud_none(*pud))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
if (pud_devmap(*pud)) {
ptl = pud_lock(mm, pud);
page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
spin_unlock(ptl);
if (page)
return page;
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
}
if (unlikely(pud_bad(*pud)))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
return follow_pmd_mask(vma, address, pud, flags, ctx);
}
@@ -777,10 +785,10 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
p4dp = p4d_offset(pgdp, address);
p4d = READ_ONCE(*p4dp);
if (!p4d_present(p4d))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
BUILD_BUG_ON(p4d_leaf(p4d));
if (unlikely(p4d_bad(p4d)))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
return follow_pud_mask(vma, address, p4dp, flags, ctx);
}
@@ -830,7 +838,7 @@ static struct page *follow_page_mask(struct vm_area_struct *vma,
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
- return no_page_table(vma, flags);
+ return no_page_table(vma, flags, address);
return follow_p4d_mask(vma, address, pgd, flags, ctx);
}
--
2.44.0
next prev parent reply other threads:[~2024-03-27 15:30 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-03-27 15:23 [PATCH v4 00/13] mm/gup: Unify hugetlb, part 2 peterx
2024-03-27 15:23 ` [PATCH v4 01/13] mm/Kconfig: CONFIG_PGTABLE_HAS_HUGE_LEAVES peterx
2024-03-27 15:23 ` [PATCH v4 02/13] mm/hugetlb: Declare hugetlbfs_pagecache_present() non-static peterx
2024-03-27 15:23 ` [PATCH v4 03/13] mm: Make HPAGE_PXD_* macros even if !THP peterx
2024-03-27 15:23 ` [PATCH v4 04/13] mm: Introduce vma_pgtable_walk_{begin|end}() peterx
2024-03-27 15:23 ` [PATCH v4 05/13] mm/arch: Provide pud_pfn() fallback peterx
2024-04-02 19:05 ` Nathan Chancellor
2024-04-02 22:43 ` Peter Xu
2024-04-02 22:53 ` Jason Gunthorpe
2024-04-02 23:35 ` Peter Xu
2024-04-03 12:08 ` Jason Gunthorpe
2024-04-03 12:26 ` Christophe Leroy
2024-04-03 13:07 ` Jason Gunthorpe
2024-04-03 13:17 ` Christophe Leroy
2024-04-03 13:33 ` Jason Gunthorpe
2024-04-03 18:25 ` Peter Xu
2024-04-04 11:24 ` Jason Gunthorpe
2024-04-04 12:00 ` Peter Xu
2024-03-27 15:23 ` [PATCH v4 06/13] mm/gup: Drop folio_fast_pin_allowed() in hugepd processing peterx
2024-03-28 10:10 ` David Hildenbrand
2024-03-28 19:01 ` Andrew Morton
2024-03-27 15:23 ` [PATCH v4 07/13] mm/gup: Refactor record_subpages() to find 1st small page peterx
2024-03-27 15:23 ` peterx [this message]
2024-03-27 15:23 ` [PATCH v4 09/13] mm/gup: Cache *pudp in follow_pud_mask() peterx
2024-03-27 15:23 ` [PATCH v4 10/13] mm/gup: Handle huge pud for follow_pud_mask() peterx
2024-03-27 15:23 ` [PATCH v4 11/13] mm/gup: Handle huge pmd for follow_pmd_mask() peterx
2024-03-27 15:23 ` [PATCH v4 12/13] mm/gup: Handle hugepd for follow_page() peterx
2024-03-27 15:23 ` [PATCH v4 13/13] mm/gup: Handle hugetlb in the generic follow_page_mask code peterx
2024-04-02 14:48 ` Ryan Roberts
2024-04-02 15:26 ` David Hildenbrand
2024-04-02 16:00 ` Matthew Wilcox
2024-04-02 16:18 ` Ryan Roberts
2024-04-02 16:26 ` Peter Xu
2024-04-02 16:40 ` David Hildenbrand
2024-04-02 16:20 ` Peter Xu
2024-04-02 16:39 ` David Hildenbrand
2024-04-02 17:57 ` Peter Xu
2024-04-02 18:43 ` David Hildenbrand
2024-04-02 16:46 ` Ryan Roberts
2024-04-02 17:58 ` Peter Xu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240327152332.950956-9-peterx@redhat.com \
--to=peterx@redhat.com \
--cc=aarcange@redhat.com \
--cc=akpm@linux-foundation.org \
--cc=andrew.jones@linux.dev \
--cc=aneesh.kumar@kernel.org \
--cc=axelrasmussen@google.com \
--cc=david@redhat.com \
--cc=hch@infradead.org \
--cc=jgg@nvidia.com \
--cc=jhubbard@nvidia.com \
--cc=jthoughton@google.com \
--cc=kirill@shutemov.name \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-riscv@lists.infradead.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=lstoakes@gmail.com \
--cc=mike.kravetz@oracle.com \
--cc=muchun.song@linux.dev \
--cc=riel@surriel.com \
--cc=rppt@kernel.org \
--cc=shy828301@gmail.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).