From: Hugh Dickins <hughd@google.com>
To: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>,
Ning Qu <quning@gmail.com>,
Andrew Morton <akpm@linux-foundation.org>,
linux-kernel@vger.kernel.org, linux-mm@kvack.org
Subject: [PATCH 13/24] huge tmpfs: extend get_user_pages_fast to shmem pmd
Date: Fri, 20 Feb 2015 20:12:22 -0800 (PST) [thread overview]
Message-ID: <alpine.LSU.2.11.1502202011070.14414@eggly.anvils> (raw)
In-Reply-To: <alpine.LSU.2.11.1502201941340.14414@eggly.anvils>
Factor out one small part of the shmem pmd handling: the arch-specific
get_user_pages_fast() has special code to cope with the peculiar
refcounting on anonymous THP tail pages (and on hugetlbfs tail pages):
which must be avoided in the straightforward shmem pmd case.
Signed-off-by: Hugh Dickins <hughd@google.com>
---
arch/mips/mm/gup.c | 17 ++++++++++++-----
arch/s390/mm/gup.c | 22 +++++++++++++++++++++-
arch/sparc/mm/gup.c | 22 +++++++++++++++++++++-
arch/x86/mm/gup.c | 17 ++++++++++++-----
mm/gup.c | 22 +++++++++++++++++++++-
5 files changed, 87 insertions(+), 13 deletions(-)
--- thpfs.orig/arch/mips/mm/gup.c 2015-02-08 18:54:22.000000000 -0800
+++ thpfs/arch/mips/mm/gup.c 2015-02-20 19:34:26.971957306 -0800
@@ -64,7 +64,8 @@ static inline void get_head_page_multipl
{
VM_BUG_ON(page != compound_head(page));
VM_BUG_ON(page_count(page) == 0);
- atomic_add(nr, &page->_count);
+ if (nr)
+ atomic_add(nr, &page->_count);
SetPageReferenced(page);
}
@@ -85,13 +86,19 @@ static int gup_huge_pmd(pmd_t pmd, unsig
head = pte_page(pte);
page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
do {
- VM_BUG_ON(compound_head(page) != head);
- pages[*nr] = page;
- if (PageTail(page))
+ if (PageTail(page)) {
+ VM_BUG_ON(compound_head(page) != head);
get_huge_page_tail(page);
+ refs++;
+ } else {
+ /*
+ * Handle head or huge tmpfs with normal refcounting.
+ */
+ get_page(page);
+ }
+ pages[*nr] = page;
(*nr)++;
page++;
- refs++;
} while (addr += PAGE_SIZE, addr != end);
get_head_page_multiple(head, refs);
--- thpfs.orig/arch/s390/mm/gup.c 2014-01-19 18:40:07.000000000 -0800
+++ thpfs/arch/s390/mm/gup.c 2015-02-20 19:34:26.971957306 -0800
@@ -61,10 +61,30 @@ static inline int gup_huge_pmd(pmd_t *pm
return 0;
VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
- refs = 0;
head = pmd_page(pmd);
page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+ if (!PageHead(head)) {
+ /*
+ * Handle a huge tmpfs team with normal refcounting.
+ */
+ do {
+ if (!page_cache_get_speculative(page))
+ return 0;
+ if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
+ put_page(page);
+ return 0;
+ }
+ pages[*nr] = page;
+ (*nr)++;
+ page++;
+ } while (addr += PAGE_SIZE, addr != end);
+ return 1;
+ }
+
tail = page;
+ refs = 0;
+
do {
VM_BUG_ON(compound_head(page) != head);
pages[*nr] = page;
--- thpfs.orig/arch/sparc/mm/gup.c 2014-12-07 14:21:05.000000000 -0800
+++ thpfs/arch/sparc/mm/gup.c 2015-02-20 19:34:26.975957297 -0800
@@ -79,10 +79,30 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd
if (write && !pmd_write(pmd))
return 0;
- refs = 0;
head = pmd_page(pmd);
page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+ if (!PageHead(head)) {
+ /*
+ * Handle a huge tmpfs team with normal refcounting.
+ */
+ do {
+ if (!page_cache_get_speculative(page))
+ return 0;
+ if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
+ put_page(page);
+ return 0;
+ }
+ pages[*nr] = page;
+ (*nr)++;
+ page++;
+ } while (addr += PAGE_SIZE, addr != end);
+ return 1;
+ }
+
tail = page;
+ refs = 0;
+
do {
VM_BUG_ON(compound_head(page) != head);
pages[*nr] = page;
--- thpfs.orig/arch/x86/mm/gup.c 2015-02-08 18:54:22.000000000 -0800
+++ thpfs/arch/x86/mm/gup.c 2015-02-20 19:34:26.975957297 -0800
@@ -110,7 +110,8 @@ static inline void get_head_page_multipl
{
VM_BUG_ON_PAGE(page != compound_head(page), page);
VM_BUG_ON_PAGE(page_count(page) == 0, page);
- atomic_add(nr, &page->_count);
+ if (nr)
+ atomic_add(nr, &page->_count);
SetPageReferenced(page);
}
@@ -135,13 +136,19 @@ static noinline int gup_huge_pmd(pmd_t p
head = pte_page(pte);
page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
do {
- VM_BUG_ON_PAGE(compound_head(page) != head, page);
- pages[*nr] = page;
- if (PageTail(page))
+ if (PageTail(page)) {
+ VM_BUG_ON_PAGE(compound_head(page) != head, page);
get_huge_page_tail(page);
+ refs++;
+ } else {
+ /*
+ * Handle head or huge tmpfs with normal refcounting.
+ */
+ get_page(page);
+ }
+ pages[*nr] = page;
(*nr)++;
page++;
- refs++;
} while (addr += PAGE_SIZE, addr != end);
get_head_page_multiple(head, refs);
--- thpfs.orig/mm/gup.c 2015-02-08 18:54:22.000000000 -0800
+++ thpfs/mm/gup.c 2015-02-20 19:34:26.975957297 -0800
@@ -795,10 +795,30 @@ static int gup_huge_pmd(pmd_t orig, pmd_
if (write && !pmd_write(orig))
return 0;
- refs = 0;
head = pmd_page(orig);
page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+
+ if (!PageHead(head)) {
+ /*
+ * Handle a huge tmpfs team with normal refcounting.
+ */
+ do {
+ if (!page_cache_get_speculative(page))
+ return 0;
+ if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
+ put_page(page);
+ return 0;
+ }
+ pages[*nr] = page;
+ (*nr)++;
+ page++;
+ } while (addr += PAGE_SIZE, addr != end);
+ return 1;
+ }
+
tail = page;
+ refs = 0;
+
do {
VM_BUG_ON_PAGE(compound_head(page) != head, page);
pages[*nr] = page;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2015-02-21 4:12 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-02-21 3:49 [PATCH 00/24] huge tmpfs: an alternative approach to THPageCache Hugh Dickins
2015-02-21 3:51 ` [PATCH 01/24] mm: update_lru_size warn and reset bad lru_size Hugh Dickins
2015-02-23 9:30 ` Kirill A. Shutemov
2015-03-23 2:44 ` Hugh Dickins
2015-02-21 3:54 ` [PATCH 02/24] mm: update_lru_size do the __mod_zone_page_state Hugh Dickins
2015-02-21 3:56 ` [PATCH 03/24] mm: use __SetPageSwapBacked and don't ClearPageSwapBacked Hugh Dickins
2015-02-25 10:53 ` Mel Gorman
2015-03-23 3:01 ` Hugh Dickins
2015-02-21 3:58 ` [PATCH 04/24] mm: make page migration's newpage handling more robust Hugh Dickins
2015-02-21 4:00 ` [PATCH 05/24] tmpfs: preliminary minor tidyups Hugh Dickins
2015-02-21 4:01 ` [PATCH 06/24] huge tmpfs: prepare counts in meminfo, vmstat and SysRq-m Hugh Dickins
2015-02-21 4:03 ` [PATCH 07/24] huge tmpfs: include shmem freeholes in available memory counts Hugh Dickins
2015-02-21 4:05 ` [PATCH 08/24] huge tmpfs: prepare huge=N mount option and /proc/sys/vm/shmem_huge Hugh Dickins
2015-02-21 4:06 ` [PATCH 09/24] huge tmpfs: try to allocate huge pages, split into a team Hugh Dickins
2015-02-21 4:07 ` [PATCH 10/24] huge tmpfs: avoid team pages in a few places Hugh Dickins
2015-02-21 4:09 ` [PATCH 11/24] huge tmpfs: shrinker to migrate and free underused holes Hugh Dickins
2015-03-19 16:56 ` Konstantin Khlebnikov
2015-03-23 4:40 ` Hugh Dickins
2015-03-23 12:50 ` Kirill A. Shutemov
2015-03-23 13:50 ` Kirill A. Shutemov
2015-03-24 12:57 ` Kirill A. Shutemov
2015-03-25 0:41 ` Hugh Dickins
2015-02-21 4:11 ` [PATCH 12/24] huge tmpfs: get_unmapped_area align and fault supply huge page Hugh Dickins
2015-02-21 4:12 ` Hugh Dickins [this message]
2015-02-21 4:13 ` [PATCH 14/24] huge tmpfs: extend vma_adjust_trans_huge to shmem pmd Hugh Dickins
2015-02-21 4:15 ` [PATCH 15/24] huge tmpfs: rework page_referenced_one and try_to_unmap_one Hugh Dickins
2015-02-21 4:16 ` [PATCH 16/24] huge tmpfs: fix problems from premature exposure of pagetable Hugh Dickins
2015-07-01 10:53 ` Kirill A. Shutemov
2015-02-21 4:18 ` [PATCH 17/24] huge tmpfs: map shmem by huge page pmd or by page team ptes Hugh Dickins
2015-02-21 4:20 ` [PATCH 18/24] huge tmpfs: mmap_sem is unlocked when truncation splits huge pmd Hugh Dickins
2015-02-21 4:22 ` [PATCH 19/24] huge tmpfs: disband split huge pmds on race or memory failure Hugh Dickins
2015-02-21 4:23 ` [PATCH 20/24] huge tmpfs: use Unevictable lru with variable hpage_nr_pages() Hugh Dickins
2015-02-21 4:25 ` [PATCH 21/24] huge tmpfs: fix Mlocked meminfo, tracking huge and unhuge mlocks Hugh Dickins
2015-02-21 4:27 ` [PATCH 22/24] huge tmpfs: fix Mapped meminfo, tracking huge and unhuge mappings Hugh Dickins
2015-02-21 4:29 ` [PATCH 23/24] kvm: plumb return of hva when resolving page fault Hugh Dickins
2015-02-21 4:31 ` [PATCH 24/24] kvm: teach kvm to map page teams as huge pages Hugh Dickins
2015-02-23 13:48 ` [PATCH 00/24] huge tmpfs: an alternative approach to THPageCache Kirill A. Shutemov
2015-03-23 2:25 ` Hugh Dickins
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=alpine.LSU.2.11.1502202011070.14414@eggly.anvils \
--to=hughd@google.com \
--cc=aarcange@redhat.com \
--cc=akpm@linux-foundation.org \
--cc=kirill.shutemov@linux.intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=quning@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).