From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, akpm@linux-foundation.org, vbabka@suse.cz,
mhocko@suse.com, david@redhat.com, osalvador@suse.de,
dan.j.williams@intel.com, sashal@kernel.org,
tyhicks@linux.microsoft.com, iamjoonsoo.kim@lge.com,
mike.kravetz@oracle.com, rostedt@goodmis.org, mingo@redhat.com,
jgg@ziepe.ca, peterz@infradead.org, mgorman@suse.de,
willy@infradead.org, rientjes@google.com, jhubbard@nvidia.com,
linux-doc@vger.kernel.org, ira.weiny@intel.com,
linux-kselftest@vger.kernel.org, jmorris@namei.org
Subject: [PATCH v8 07/14] mm: honor PF_MEMALLOC_PIN for all movable pages
Date: Mon, 25 Jan 2021 14:47:44 -0500 [thread overview]
Message-ID: <20210125194751.1275316-8-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20210125194751.1275316-1-pasha.tatashin@soleen.com>
PF_MEMALLOC_PIN is only honored for CMA pages, extend
this flag to work for any allocations from ZONE_MOVABLE by removing
__GFP_MOVABLE from gfp_mask when this flag is passed in the current
context.
Add is_pinnable_page() to return true if page is in a pinnable page.
A pinnable page is not in ZONE_MOVABLE and not of MIGRATE_CMA type.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
Acked-by: Michal Hocko <mhocko@suse.com>
---
include/linux/mm.h | 11 +++++++++++
include/linux/sched/mm.h | 6 +++++-
mm/hugetlb.c | 2 +-
mm/page_alloc.c | 20 +++++++++-----------
4 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a5d618d08506..0990a76d5e6f 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1117,6 +1117,17 @@ static inline bool is_zone_device_page(const struct page *page)
}
#endif
+static inline bool is_zone_movable_page(const struct page *page)
+{
+ return page_zonenum(page) == ZONE_MOVABLE;
+}
+
+/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */
+static inline bool is_pinnable_page(struct page *page)
+{
+ return !is_zone_movable_page(page) && !is_migrate_cma_page(page);
+}
+
#ifdef CONFIG_DEV_PAGEMAP_OPS
void free_devmap_managed_page(struct page *page);
DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 5f4dd3274734..a55277b0d475 100644
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -150,12 +150,13 @@ static inline bool in_vfork(struct task_struct *tsk)
* Applies per-task gfp context to the given allocation flags.
* PF_MEMALLOC_NOIO implies GFP_NOIO
* PF_MEMALLOC_NOFS implies GFP_NOFS
+ * PF_MEMALLOC_PIN implies !GFP_MOVABLE
*/
static inline gfp_t current_gfp_context(gfp_t flags)
{
unsigned int pflags = READ_ONCE(current->flags);
- if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS))) {
+ if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_PIN))) {
/*
* NOIO implies both NOIO and NOFS and it is a weaker context
* so always make sure it makes precedence
@@ -164,6 +165,9 @@ static inline gfp_t current_gfp_context(gfp_t flags)
flags &= ~(__GFP_IO | __GFP_FS);
else if (pflags & PF_MEMALLOC_NOFS)
flags &= ~__GFP_FS;
+
+ if (pflags & PF_MEMALLOC_PIN)
+ flags &= ~__GFP_MOVABLE;
}
return flags;
}
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 2d79e515a7a3..77098492a2fd 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1052,7 +1052,7 @@ static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
bool pin = !!(current->flags & PF_MEMALLOC_PIN);
list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
- if (pin && is_migrate_cma_page(page))
+ if (pin && !is_pinnable_page(page))
continue;
if (PageHWPoison(page))
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c93e801a45e9..3f17c73ad582 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3807,16 +3807,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
return alloc_flags;
}
-static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
- unsigned int alloc_flags)
+/* Must be called after current_gfp_context() which can change gfp_mask */
+static inline unsigned int gpf_to_alloc_flags(gfp_t gfp_mask,
+ unsigned int alloc_flags)
{
#ifdef CONFIG_CMA
- unsigned int pflags = current->flags;
-
- if (!(pflags & PF_MEMALLOC_PIN) &&
- gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
+ if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
alloc_flags |= ALLOC_CMA;
-
#endif
return alloc_flags;
}
@@ -4472,7 +4469,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
} else if (unlikely(rt_task(current)) && !in_interrupt())
alloc_flags |= ALLOC_HARDER;
- alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);
+ alloc_flags = gpf_to_alloc_flags(gfp_mask, alloc_flags);
return alloc_flags;
}
@@ -4774,7 +4771,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
if (reserve_flags)
- alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);
+ alloc_flags = gpf_to_alloc_flags(gfp_mask, reserve_flags);
/*
* Reset the nodemask and zonelist iterators if memory policies can be
@@ -4943,7 +4940,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
if (should_fail_alloc_page(gfp_mask, order))
return false;
- *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);
+ *alloc_flags = gpf_to_alloc_flags(gfp_mask, *alloc_flags);
/* Dirty zone balancing only done in the fast path */
ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
@@ -4985,7 +4982,8 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
* Apply scoped allocation constraints. This is mainly about GFP_NOFS
* resp. GFP_NOIO which has to be inherited for all allocation requests
* from a particular context which has been marked by
- * memalloc_no{fs,io}_{save,restore}.
+ * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures
+ * movable zones are not used during allocation.
*/
gfp_mask = current_gfp_context(gfp_mask);
alloc_mask = gfp_mask;
--
2.25.1
next prev parent reply other threads:[~2021-01-25 19:48 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-25 19:47 [PATCH v8 00/14] prohibit pinning pages in ZONE_MOVABLE Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 01/14] mm/gup: don't pin migrated cma pages in movable zone Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 02/14] mm/gup: check every subpage of a compound page during isolation Pavel Tatashin
2021-02-03 13:22 ` Joao Martins
2021-02-03 14:51 ` Pavel Tatashin
2021-02-03 15:32 ` Joao Martins
2021-02-03 16:46 ` Joao Martins
2021-02-03 14:53 ` Jason Gunthorpe
2021-02-03 16:13 ` Joao Martins
2021-02-03 18:00 ` Jason Gunthorpe
2021-01-25 19:47 ` [PATCH v8 03/14] mm/gup: return an error on migration failure Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 04/14] mm/gup: check for isolation errors Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 05/14] mm cma: rename PF_MEMALLOC_NOCMA to PF_MEMALLOC_PIN Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 06/14] mm: apply per-task gfp constraints in fast path Pavel Tatashin
2021-01-25 19:47 ` Pavel Tatashin [this message]
[not found] ` <20210131130903.4875-1-lecopzer@gmail.com>
2021-01-31 16:03 ` [PATCH v8 07/14] mm: honor PF_MEMALLOC_PIN for all movable pages Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 08/14] mm/gup: do not migrate zero page Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 09/14] mm/gup: migrate pinned pages out of movable zone Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 10/14] memory-hotplug.rst: add a note about ZONE_MOVABLE and page pinning Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 11/14] mm/gup: change index type to long as it counts pages Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 12/14] mm/gup: longterm pin migration cleanup Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 13/14] selftests/vm: gup_test: fix test flag Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 14/14] selftests/vm: gup_test: test faulting in kernel, and verify pinnable pages Pavel Tatashin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210125194751.1275316-8-pasha.tatashin@soleen.com \
--to=pasha.tatashin@soleen.com \
--cc=akpm@linux-foundation.org \
--cc=dan.j.williams@intel.com \
--cc=david@redhat.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=ira.weiny@intel.com \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=jmorris@namei.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mgorman@suse.de \
--cc=mhocko@suse.com \
--cc=mike.kravetz@oracle.com \
--cc=mingo@redhat.com \
--cc=osalvador@suse.de \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=rostedt@goodmis.org \
--cc=sashal@kernel.org \
--cc=tyhicks@linux.microsoft.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).