From: "Matthew Wilcox (Oracle)" <willy@infradead.org>
To: Andrew Morton <akpm@linux-foundation.org>
Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>,
linux-mm@kvack.org, Hugh Dickins <hughd@google.com>
Subject: [PATCH 02/22] mm: Convert head_subpages_mapcount() into folio_nr_pages_mapped()
Date: Sat, 31 Dec 2022 21:45:50 +0000 [thread overview]
Message-ID: <20221231214610.2800682-3-willy@infradead.org> (raw)
In-Reply-To: <20221231214610.2800682-1-willy@infradead.org>
Calling this 'mapcount' is confusing since mapcount is usually the number
of times something is mapped; instead this is the number of mapped pages.
It's also better to enforce that this is a folio rather than a head page.
Move folio_nr_pages_mapped() into mm/internal.h since this is not
something we want device drivers or filesystems poking at. Get rid of
folio_subpages_mapcount_ptr() and use folio->_nr_pages_mapped directly.
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
---
include/linux/mm.h | 22 ++--------------------
include/linux/mm_types.h | 12 +++---------
mm/debug.c | 4 ++--
mm/hugetlb.c | 4 ++--
mm/internal.h | 18 ++++++++++++++++++
mm/rmap.c | 9 +++++----
6 files changed, 32 insertions(+), 37 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index ec801f24ef61..7ee1938278f5 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -838,24 +838,6 @@ static inline int head_compound_mapcount(struct page *head)
return atomic_read(compound_mapcount_ptr(head)) + 1;
}
-/*
- * If a 16GB hugetlb page were mapped by PTEs of all of its 4kB sub-pages,
- * its subpages_mapcount would be 0x400000: choose the COMPOUND_MAPPED bit
- * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently
- * leaves subpages_mapcount at 0, but avoid surprise if it participates later.
- */
-#define COMPOUND_MAPPED 0x800000
-#define SUBPAGES_MAPPED (COMPOUND_MAPPED - 1)
-
-/*
- * Number of sub-pages mapped by PTE, does not include compound mapcount.
- * Must be called only on head of compound page.
- */
-static inline int head_subpages_mapcount(struct page *head)
-{
- return atomic_read(subpages_mapcount_ptr(head)) & SUBPAGES_MAPPED;
-}
-
/*
* The atomic page->_mapcount, starts from -1: so that transitions
* both from it and to it can be tracked, using atomic_inc_and_test
@@ -915,9 +897,9 @@ static inline bool folio_large_is_mapped(struct folio *folio)
{
/*
* Reading folio_mapcount_ptr() below could be omitted if hugetlb
- * participated in incrementing subpages_mapcount when compound mapped.
+ * participated in incrementing nr_pages_mapped when compound mapped.
*/
- return atomic_read(folio_subpages_mapcount_ptr(folio)) > 0 ||
+ return atomic_read(&folio->_nr_pages_mapped) > 0 ||
atomic_read(folio_mapcount_ptr(folio)) >= 0;
}
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 5d9bf1f79e96..fc44d5bab7b8 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -307,7 +307,7 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
* @_folio_dtor: Which destructor to use for this folio.
* @_folio_order: Do not use directly, call folio_order().
* @_compound_mapcount: Do not use directly, call folio_entire_mapcount().
- * @_subpages_mapcount: Do not use directly, call folio_mapcount().
+ * @_nr_pages_mapped: Do not use directly, call folio_mapcount().
* @_pincount: Do not use directly, call folio_maybe_dma_pinned().
* @_folio_nr_pages: Do not use directly, call folio_nr_pages().
* @_flags_2: For alignment. Do not use.
@@ -361,7 +361,7 @@ struct folio {
unsigned char _folio_dtor;
unsigned char _folio_order;
atomic_t _compound_mapcount;
- atomic_t _subpages_mapcount;
+ atomic_t _nr_pages_mapped;
atomic_t _pincount;
#ifdef CONFIG_64BIT
unsigned int _folio_nr_pages;
@@ -404,7 +404,7 @@ FOLIO_MATCH(compound_head, _head_1);
FOLIO_MATCH(compound_dtor, _folio_dtor);
FOLIO_MATCH(compound_order, _folio_order);
FOLIO_MATCH(compound_mapcount, _compound_mapcount);
-FOLIO_MATCH(subpages_mapcount, _subpages_mapcount);
+FOLIO_MATCH(subpages_mapcount, _nr_pages_mapped);
FOLIO_MATCH(compound_pincount, _pincount);
#ifdef CONFIG_64BIT
FOLIO_MATCH(compound_nr, _folio_nr_pages);
@@ -427,12 +427,6 @@ static inline atomic_t *folio_mapcount_ptr(struct folio *folio)
return &tail->compound_mapcount;
}
-static inline atomic_t *folio_subpages_mapcount_ptr(struct folio *folio)
-{
- struct page *tail = &folio->page + 1;
- return &tail->subpages_mapcount;
-}
-
static inline atomic_t *compound_mapcount_ptr(struct page *page)
{
return &page[1].compound_mapcount;
diff --git a/mm/debug.c b/mm/debug.c
index 893c9dbf76ca..8e58e8dab0b2 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -94,10 +94,10 @@ static void __dump_page(struct page *page)
page, page_ref_count(head), mapcount, mapping,
page_to_pgoff(page), page_to_pfn(page));
if (compound) {
- pr_warn("head:%p order:%u compound_mapcount:%d subpages_mapcount:%d pincount:%d\n",
+ pr_warn("head:%p order:%u compound_mapcount:%d nr_pages_mapped:%d pincount:%d\n",
head, compound_order(head),
head_compound_mapcount(head),
- head_subpages_mapcount(head),
+ folio_nr_pages_mapped(folio),
atomic_read(&folio->_pincount));
}
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index c01493ceeb8d..55e744abb962 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1479,7 +1479,7 @@ static void __destroy_compound_gigantic_folio(struct folio *folio,
struct page *p;
atomic_set(folio_mapcount_ptr(folio), 0);
- atomic_set(folio_subpages_mapcount_ptr(folio), 0);
+ atomic_set(&folio->_nr_pages_mapped, 0);
atomic_set(&folio->_pincount, 0);
for (i = 1; i < nr_pages; i++) {
@@ -2001,7 +2001,7 @@ static bool __prep_compound_gigantic_folio(struct folio *folio,
set_compound_head(p, &folio->page);
}
atomic_set(folio_mapcount_ptr(folio), -1);
- atomic_set(folio_subpages_mapcount_ptr(folio), 0);
+ atomic_set(&folio->_nr_pages_mapped, 0);
atomic_set(&folio->_pincount, 0);
return true;
diff --git a/mm/internal.h b/mm/internal.h
index bcf75a8b032d..f3bb12e77980 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -52,6 +52,24 @@ struct folio_batch;
void page_writeback_init(void);
+/*
+ * If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages,
+ * its nr_pages_mapped would be 0x400000: choose the COMPOUND_MAPPED bit
+ * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently
+ * leaves nr_pages_mapped at 0, but avoid surprise if it participates later.
+ */
+#define COMPOUND_MAPPED 0x800000
+#define FOLIO_PAGES_MAPPED (COMPOUND_MAPPED - 1)
+
+/*
+ * How many individual pages have an elevated _mapcount. Excludes
+ * the folio's entire_mapcount.
+ */
+static inline int folio_nr_pages_mapped(struct folio *folio)
+{
+ return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED;
+}
+
static inline void *folio_raw_mapping(struct folio *folio)
{
unsigned long mapping = (unsigned long)folio->mapping;
diff --git a/mm/rmap.c b/mm/rmap.c
index b616870a09be..09f4d260a46c 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1087,12 +1087,13 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
int total_compound_mapcount(struct page *head)
{
+ struct folio *folio = (struct folio *)head;
int mapcount = head_compound_mapcount(head);
int nr_subpages;
int i;
/* In the common case, avoid the loop when no subpages mapped by PTE */
- if (head_subpages_mapcount(head) == 0)
+ if (folio_nr_pages_mapped(folio) == 0)
return mapcount;
/*
* Add all the PTE mappings of those subpages mapped by PTE.
@@ -1243,7 +1244,7 @@ void page_add_anon_rmap(struct page *page,
nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);
if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {
nr_pmdmapped = thp_nr_pages(page);
- nr = nr_pmdmapped - (nr & SUBPAGES_MAPPED);
+ nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
/* Raced ahead of a remove and another add? */
if (unlikely(nr < 0))
nr = 0;
@@ -1349,7 +1350,7 @@ void page_add_file_rmap(struct page *page,
nr = atomic_add_return_relaxed(COMPOUND_MAPPED, mapped);
if (likely(nr < COMPOUND_MAPPED + COMPOUND_MAPPED)) {
nr_pmdmapped = thp_nr_pages(page);
- nr = nr_pmdmapped - (nr & SUBPAGES_MAPPED);
+ nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
/* Raced ahead of a remove and another add? */
if (unlikely(nr < 0))
nr = 0;
@@ -1414,7 +1415,7 @@ void page_remove_rmap(struct page *page,
nr = atomic_sub_return_relaxed(COMPOUND_MAPPED, mapped);
if (likely(nr < COMPOUND_MAPPED)) {
nr_pmdmapped = thp_nr_pages(page);
- nr = nr_pmdmapped - (nr & SUBPAGES_MAPPED);
+ nr = nr_pmdmapped - (nr & FOLIO_PAGES_MAPPED);
/* Raced ahead of another remove and an add? */
if (unlikely(nr < 0))
nr = 0;
--
2.35.1
next prev parent reply other threads:[~2022-12-31 21:46 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-12-31 21:45 [PATCH 00/22] Get rid of first tail page fields Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 01/22] mm: Remove folio_pincount_ptr() and head_compound_pincount() Matthew Wilcox (Oracle)
2022-12-31 21:45 ` Matthew Wilcox (Oracle) [this message]
2022-12-31 21:45 ` [PATCH 03/22] doc: Clarify refcount section by referring to folios & pages Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 04/22] mm: Convert total_compound_mapcount() to folio_total_mapcount() Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 05/22] mm: Convert page_remove_rmap() to use a folio internally Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 06/22] mm: Convert page_add_anon_rmap() " Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 07/22] mm: Convert page_add_file_rmap() " Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 08/22] mm: Add folio_add_new_anon_rmap() Matthew Wilcox (Oracle)
2022-12-31 23:29 ` kernel test robot
2022-12-31 23:59 ` kernel test robot
2022-12-31 21:45 ` [PATCH 09/22] page_alloc: Use folio fields directly Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 10/22] mm: Use a folio in hugepage_add_anon_rmap() and hugepage_add_new_anon_rmap() Matthew Wilcox (Oracle)
2022-12-31 21:45 ` [PATCH 11/22] mm: Use entire_mapcount in __page_dup_rmap() Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 12/22] mm/debug: Remove call to head_compound_mapcount() Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 13/22] hugetlb: Remove uses of folio_mapcount_ptr Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 14/22] mm: Convert page_mapcount() to use folio_entire_mapcount() Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 15/22] mm: Remove head_compound_mapcount() and _ptr functions Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 16/22] mm: Reimplement compound_order() Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 17/22] mm: Reimplement compound_nr() Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 18/22] mm: Convert set_compound_page_dtor() and set_compound_order() to folios Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 19/22] mm: Convert is_transparent_hugepage() to use a folio Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 20/22] mm: Convert destroy_large_folio() to use folio_dtor Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 21/22] hugetlb: Remove uses of compound_dtor and compound_nr Matthew Wilcox (Oracle)
2022-12-31 21:46 ` [PATCH 22/22] mm: Remove 'First tail page' members from struct page Matthew Wilcox (Oracle)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221231214610.2800682-3-willy@infradead.org \
--to=willy@infradead.org \
--cc=akpm@linux-foundation.org \
--cc=hughd@google.com \
--cc=linux-mm@kvack.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).