From: Pavel Tatashin <pasha.tatashin@soleen.com>
To: pasha.tatashin@soleen.com, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, akpm@linux-foundation.org, vbabka@suse.cz,
mhocko@suse.com, david@redhat.com, osalvador@suse.de,
dan.j.williams@intel.com, sashal@kernel.org,
tyhicks@linux.microsoft.com, iamjoonsoo.kim@lge.com,
mike.kravetz@oracle.com, rostedt@goodmis.org, mingo@redhat.com,
jgg@ziepe.ca, peterz@infradead.org, mgorman@suse.de,
willy@infradead.org, rientjes@google.com, jhubbard@nvidia.com,
linux-doc@vger.kernel.org, ira.weiny@intel.com,
linux-kselftest@vger.kernel.org, jmorris@namei.org
Subject: [PATCH v8 12/14] mm/gup: longterm pin migration cleanup
Date: Mon, 25 Jan 2021 14:47:49 -0500 [thread overview]
Message-ID: <20210125194751.1275316-13-pasha.tatashin@soleen.com> (raw)
In-Reply-To: <20210125194751.1275316-1-pasha.tatashin@soleen.com>
When pages are longterm pinned, we must migrated them out of movable zone.
The function that migrates them has a hidden loop with goto. The loop is
to retry on isolation failures, and after successful migration.
Make this code better by moving this loop to the caller.
Signed-off-by: Pavel Tatashin <pasha.tatashin@soleen.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
---
mm/gup.c | 93 ++++++++++++++++++++++----------------------------------
1 file changed, 37 insertions(+), 56 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index f98af75dab0f..fabfe2a5c627 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1549,27 +1549,28 @@ struct page *get_dump_page(unsigned long addr)
#endif /* CONFIG_ELF_CORE */
#ifdef CONFIG_MIGRATION
-static long check_and_migrate_movable_pages(struct mm_struct *mm,
- unsigned long start,
- unsigned long nr_pages,
+/*
+ * Check whether all pages are pinnable, if so return number of pages. If some
+ * pages are not pinnable, migrate them, and unpin all pages. Return zero if
+ * pages were migrated, or if some pages were not successfully isolated.
+ * Return negative error if migration fails.
+ */
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages,
- struct vm_area_struct **vmas,
unsigned int gup_flags)
{
- unsigned long i, isolation_error_count;
- bool drain_allow;
+ unsigned long i;
+ unsigned long isolation_error_count = 0;
+ bool drain_allow = true;
LIST_HEAD(movable_page_list);
- long ret = nr_pages;
- struct page *prev_head, *head;
+ long ret = 0;
+ struct page *prev_head = NULL;
+ struct page *head;
struct migration_target_control mtc = {
.nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_NOWARN,
};
-check_again:
- prev_head = NULL;
- isolation_error_count = 0;
- drain_allow = true;
for (i = 0; i < nr_pages; i++) {
head = compound_head(pages[i]);
if (head == prev_head)
@@ -1607,47 +1608,27 @@ static long check_and_migrate_movable_pages(struct mm_struct *mm,
* in the correct zone.
*/
if (list_empty(&movable_page_list) && !isolation_error_count)
- return ret;
+ return nr_pages;
+ if (gup_flags & FOLL_PIN) {
+ unpin_user_pages(pages, nr_pages);
+ } else {
+ for (i = 0; i < nr_pages; i++)
+ put_page(pages[i]);
+ }
if (!list_empty(&movable_page_list)) {
- /*
- * drop the above get_user_pages reference.
- */
- if (gup_flags & FOLL_PIN)
- unpin_user_pages(pages, nr_pages);
- else
- for (i = 0; i < nr_pages; i++)
- put_page(pages[i]);
-
ret = migrate_pages(&movable_page_list, alloc_migration_target,
NULL, (unsigned long)&mtc, MIGRATE_SYNC,
MR_LONGTERM_PIN);
- if (ret) {
- if (!list_empty(&movable_page_list))
- putback_movable_pages(&movable_page_list);
- return ret > 0 ? -ENOMEM : ret;
- }
-
- /* We unpinned pages before migration, pin them again */
- ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
- NULL, gup_flags);
- if (ret <= 0)
- return ret;
- nr_pages = ret;
+ if (ret && !list_empty(&movable_page_list))
+ putback_movable_pages(&movable_page_list);
}
- /*
- * check again because pages were unpinned, and we also might have
- * had isolation errors and need more pages to migrate.
- */
- goto check_again;
+ return ret > 0 ? -ENOMEM : ret;
}
#else
-static long check_and_migrate_movable_pages(struct mm_struct *mm,
- unsigned long start,
- unsigned long nr_pages,
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages,
- struct vm_area_struct **vmas,
unsigned int gup_flags)
{
return nr_pages;
@@ -1665,22 +1646,22 @@ static long __gup_longterm_locked(struct mm_struct *mm,
struct vm_area_struct **vmas,
unsigned int gup_flags)
{
- unsigned long flags = 0;
+ unsigned int flags;
long rc;
- if (gup_flags & FOLL_LONGTERM)
- flags = memalloc_pin_save();
-
- rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL,
- gup_flags);
+ if (!(gup_flags & FOLL_LONGTERM))
+ return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+ NULL, gup_flags);
+ flags = memalloc_pin_save();
+ do {
+ rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+ NULL, gup_flags);
+ if (rc <= 0)
+ break;
+ rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
+ } while (!rc);
+ memalloc_pin_restore(flags);
- if (gup_flags & FOLL_LONGTERM) {
- if (rc > 0)
- rc = check_and_migrate_movable_pages(mm, start, rc,
- pages, vmas,
- gup_flags);
- memalloc_pin_restore(flags);
- }
return rc;
}
--
2.25.1
next prev parent reply other threads:[~2021-01-25 19:48 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-25 19:47 [PATCH v8 00/14] prohibit pinning pages in ZONE_MOVABLE Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 01/14] mm/gup: don't pin migrated cma pages in movable zone Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 02/14] mm/gup: check every subpage of a compound page during isolation Pavel Tatashin
2021-02-03 13:22 ` Joao Martins
2021-02-03 14:51 ` Pavel Tatashin
2021-02-03 15:32 ` Joao Martins
2021-02-03 16:46 ` Joao Martins
2021-02-03 14:53 ` Jason Gunthorpe
2021-02-03 16:13 ` Joao Martins
2021-02-03 18:00 ` Jason Gunthorpe
2021-01-25 19:47 ` [PATCH v8 03/14] mm/gup: return an error on migration failure Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 04/14] mm/gup: check for isolation errors Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 05/14] mm cma: rename PF_MEMALLOC_NOCMA to PF_MEMALLOC_PIN Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 06/14] mm: apply per-task gfp constraints in fast path Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 07/14] mm: honor PF_MEMALLOC_PIN for all movable pages Pavel Tatashin
[not found] ` <20210131130903.4875-1-lecopzer@gmail.com>
2021-01-31 16:03 ` Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 08/14] mm/gup: do not migrate zero page Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 09/14] mm/gup: migrate pinned pages out of movable zone Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 10/14] memory-hotplug.rst: add a note about ZONE_MOVABLE and page pinning Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 11/14] mm/gup: change index type to long as it counts pages Pavel Tatashin
2021-01-25 19:47 ` Pavel Tatashin [this message]
2021-01-25 19:47 ` [PATCH v8 13/14] selftests/vm: gup_test: fix test flag Pavel Tatashin
2021-01-25 19:47 ` [PATCH v8 14/14] selftests/vm: gup_test: test faulting in kernel, and verify pinnable pages Pavel Tatashin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210125194751.1275316-13-pasha.tatashin@soleen.com \
--to=pasha.tatashin@soleen.com \
--cc=akpm@linux-foundation.org \
--cc=dan.j.williams@intel.com \
--cc=david@redhat.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=ira.weiny@intel.com \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=jmorris@namei.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mgorman@suse.de \
--cc=mhocko@suse.com \
--cc=mike.kravetz@oracle.com \
--cc=mingo@redhat.com \
--cc=osalvador@suse.de \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=rostedt@goodmis.org \
--cc=sashal@kernel.org \
--cc=tyhicks@linux.microsoft.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).