From: Zi Yan <zi.yan@sent.com> To: linux-kernel@vger.kernel.org, linux-mm@kvack.org Cc: kirill.shutemov@linux.intel.com, akpm@linux-foundation.org, minchan@kernel.org, vbabka@suse.cz, mgorman@techsingularity.net, mhocko@kernel.org, n-horiguchi@ah.jp.nec.com, khandual@linux.vnet.ibm.com, zi.yan@cs.rutgers.edu, dnellans@nvidia.com Subject: [PATCH v4 09/11] mm: mempolicy: mbind and migrate_pages support thp migration Date: Mon, 13 Mar 2017 11:45:05 -0400 [thread overview] Message-ID: <20170313154507.3647-10-zi.yan@sent.com> (raw) In-Reply-To: <20170313154507.3647-1-zi.yan@sent.com> From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> This patch enables thp migration for mbind(2) and migrate_pages(2). Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> --- ChangeLog v1 -> v2: - support pte-mapped and doubly-mapped thp --- mm/mempolicy.c | 108 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index aa242da77fda..d880dc6e9c6b 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -94,6 +94,7 @@ #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/printk.h> +#include <linux/swapops.h> #include <asm/tlbflush.h> #include <linux/uaccess.h> @@ -486,6 +487,49 @@ static inline bool queue_pages_node_check(struct page *page, return node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT); } +static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + int ret = 0; + struct page *page; + struct queue_pages *qp = walk->private; + unsigned long flags; + + if (unlikely(is_pmd_migration_entry(*pmd))) { + ret = 1; + goto unlock; + } + page = pmd_page(*pmd); + if (is_huge_zero_page(page)) { + spin_unlock(ptl); + __split_huge_pmd(walk->vma, pmd, addr, false, NULL); + goto out; + } + if (!thp_migration_supported()) { + get_page(page); + spin_unlock(ptl); + lock_page(page); + ret = split_huge_page(page); + unlock_page(page); + put_page(page); + goto out; + } + if (queue_pages_node_check(page, qp)) { + ret = 1; + goto unlock; + } + + ret = 1; + flags = qp->flags; + /* go to thp migration */ + if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) + migrate_page_add(page, qp->pagelist, flags); +unlock: + spin_unlock(ptl); +out: + return ret; +} + /* * Scan through pages checking if pages follow certain conditions, * and move them to the pagelist if they do. @@ -497,30 +541,15 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, struct page *page; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; - int nid, ret; + int ret; pte_t *pte; spinlock_t *ptl; - if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); - if (pmd_trans_huge(*pmd)) { - page = pmd_page(*pmd); - if (is_huge_zero_page(page)) { - spin_unlock(ptl); - __split_huge_pmd(vma, pmd, addr, false, NULL); - } else { - get_page(page); - spin_unlock(ptl); - lock_page(page); - ret = split_huge_page(page); - unlock_page(page); - put_page(page); - if (ret) - return 0; - } - } else { - spin_unlock(ptl); - } + ptl = pmd_trans_huge_lock(pmd, vma); + if (ptl) { + ret = queue_pages_pmd(pmd, ptl, addr, end, walk); + if (ret) + return 0; } if (pmd_trans_unstable(pmd)) @@ -541,7 +570,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, continue; if (queue_pages_node_check(page, qp)) continue; - if (PageTransCompound(page)) { + if (PageTransCompound(page) && !thp_migration_supported()) { get_page(page); pte_unmap_unlock(pte, ptl); lock_page(page); @@ -959,19 +988,21 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, #ifdef CONFIG_MIGRATION /* - * page migration + * page migration, thp tail pages can be passed. */ static void migrate_page_add(struct page *page, struct list_head *pagelist, unsigned long flags) { + struct page *head = compound_head(page); /* * Avoid migrating a page that is shared with others. */ - if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) { - if (!isolate_lru_page(page)) { - list_add_tail(&page->lru, pagelist); - inc_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(head) == 1) { + if (!isolate_lru_page(head)) { + list_add_tail(&head->lru, pagelist); + mod_node_page_state(page_pgdat(head), + NR_ISOLATED_ANON + page_is_file_cache(head), + hpage_nr_pages(head)); } } } @@ -981,7 +1012,17 @@ static struct page *new_node_page(struct page *page, unsigned long node, int **x if (PageHuge(page)) return alloc_huge_page_node(page_hstate(compound_head(page)), node); - else + else if (thp_migration_supported() && PageTransHuge(page)) { + struct page *thp; + + thp = alloc_pages_node(node, + (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_RECLAIM, + HPAGE_PMD_ORDER); + if (!thp) + return NULL; + prep_transhuge_page(thp); + return thp; + } else return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0); } @@ -1147,6 +1188,15 @@ static struct page *new_page(struct page *page, unsigned long start, int **x) if (PageHuge(page)) { BUG_ON(!vma); return alloc_huge_page_noerr(vma, address, 1); + } else if (thp_migration_supported() && PageTransHuge(page)) { + struct page *thp; + + thp = alloc_hugepage_vma(GFP_TRANSHUGE, vma, address, + HPAGE_PMD_ORDER); + if (!thp) + return NULL; + prep_transhuge_page(thp); + return thp; } /* * if !vma, alloc_page_vma() will use task or system default policy -- 2.11.0
WARNING: multiple messages have this Message-ID (diff)
From: Zi Yan <zi.yan@sent.com> To: linux-kernel@vger.kernel.org, linux-mm@kvack.org Cc: kirill.shutemov@linux.intel.com, akpm@linux-foundation.org, minchan@kernel.org, vbabka@suse.cz, mgorman@techsingularity.net, mhocko@kernel.org, n-horiguchi@ah.jp.nec.com, khandual@linux.vnet.ibm.com, zi.yan@cs.rutgers.edu, dnellans@nvidia.com Subject: [PATCH v4 09/11] mm: mempolicy: mbind and migrate_pages support thp migration Date: Mon, 13 Mar 2017 11:45:05 -0400 [thread overview] Message-ID: <20170313154507.3647-10-zi.yan@sent.com> (raw) In-Reply-To: <20170313154507.3647-1-zi.yan@sent.com> From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> This patch enables thp migration for mbind(2) and migrate_pages(2). Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> --- ChangeLog v1 -> v2: - support pte-mapped and doubly-mapped thp --- mm/mempolicy.c | 108 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/mm/mempolicy.c b/mm/mempolicy.c index aa242da77fda..d880dc6e9c6b 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -94,6 +94,7 @@ #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/printk.h> +#include <linux/swapops.h> #include <asm/tlbflush.h> #include <linux/uaccess.h> @@ -486,6 +487,49 @@ static inline bool queue_pages_node_check(struct page *page, return node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT); } +static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + int ret = 0; + struct page *page; + struct queue_pages *qp = walk->private; + unsigned long flags; + + if (unlikely(is_pmd_migration_entry(*pmd))) { + ret = 1; + goto unlock; + } + page = pmd_page(*pmd); + if (is_huge_zero_page(page)) { + spin_unlock(ptl); + __split_huge_pmd(walk->vma, pmd, addr, false, NULL); + goto out; + } + if (!thp_migration_supported()) { + get_page(page); + spin_unlock(ptl); + lock_page(page); + ret = split_huge_page(page); + unlock_page(page); + put_page(page); + goto out; + } + if (queue_pages_node_check(page, qp)) { + ret = 1; + goto unlock; + } + + ret = 1; + flags = qp->flags; + /* go to thp migration */ + if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) + migrate_page_add(page, qp->pagelist, flags); +unlock: + spin_unlock(ptl); +out: + return ret; +} + /* * Scan through pages checking if pages follow certain conditions, * and move them to the pagelist if they do. @@ -497,30 +541,15 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, struct page *page; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; - int nid, ret; + int ret; pte_t *pte; spinlock_t *ptl; - if (pmd_trans_huge(*pmd)) { - ptl = pmd_lock(walk->mm, pmd); - if (pmd_trans_huge(*pmd)) { - page = pmd_page(*pmd); - if (is_huge_zero_page(page)) { - spin_unlock(ptl); - __split_huge_pmd(vma, pmd, addr, false, NULL); - } else { - get_page(page); - spin_unlock(ptl); - lock_page(page); - ret = split_huge_page(page); - unlock_page(page); - put_page(page); - if (ret) - return 0; - } - } else { - spin_unlock(ptl); - } + ptl = pmd_trans_huge_lock(pmd, vma); + if (ptl) { + ret = queue_pages_pmd(pmd, ptl, addr, end, walk); + if (ret) + return 0; } if (pmd_trans_unstable(pmd)) @@ -541,7 +570,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr, continue; if (queue_pages_node_check(page, qp)) continue; - if (PageTransCompound(page)) { + if (PageTransCompound(page) && !thp_migration_supported()) { get_page(page); pte_unmap_unlock(pte, ptl); lock_page(page); @@ -959,19 +988,21 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, #ifdef CONFIG_MIGRATION /* - * page migration + * page migration, thp tail pages can be passed. */ static void migrate_page_add(struct page *page, struct list_head *pagelist, unsigned long flags) { + struct page *head = compound_head(page); /* * Avoid migrating a page that is shared with others. */ - if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) { - if (!isolate_lru_page(page)) { - list_add_tail(&page->lru, pagelist); - inc_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); + if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(head) == 1) { + if (!isolate_lru_page(head)) { + list_add_tail(&head->lru, pagelist); + mod_node_page_state(page_pgdat(head), + NR_ISOLATED_ANON + page_is_file_cache(head), + hpage_nr_pages(head)); } } } @@ -981,7 +1012,17 @@ static struct page *new_node_page(struct page *page, unsigned long node, int **x if (PageHuge(page)) return alloc_huge_page_node(page_hstate(compound_head(page)), node); - else + else if (thp_migration_supported() && PageTransHuge(page)) { + struct page *thp; + + thp = alloc_pages_node(node, + (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_RECLAIM, + HPAGE_PMD_ORDER); + if (!thp) + return NULL; + prep_transhuge_page(thp); + return thp; + } else return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0); } @@ -1147,6 +1188,15 @@ static struct page *new_page(struct page *page, unsigned long start, int **x) if (PageHuge(page)) { BUG_ON(!vma); return alloc_huge_page_noerr(vma, address, 1); + } else if (thp_migration_supported() && PageTransHuge(page)) { + struct page *thp; + + thp = alloc_hugepage_vma(GFP_TRANSHUGE, vma, address, + HPAGE_PMD_ORDER); + if (!thp) + return NULL; + prep_transhuge_page(thp); + return thp; } /* * if !vma, alloc_page_vma() will use task or system default policy -- 2.11.0 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@kvack.org. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2017-03-13 15:47 UTC|newest] Thread overview: 52+ messages / expand[flat|nested] mbox.gz Atom feed top 2017-03-13 15:44 [PATCH v4 00/11] mm: page migration enhancement for thp Zi Yan 2017-03-13 15:44 ` Zi Yan 2017-03-13 15:44 ` [PATCH v4 01/11] mm: x86: move _PAGE_SWP_SOFT_DIRTY from bit 7 to bit 1 Zi Yan 2017-03-13 15:44 ` Zi Yan 2017-03-24 18:23 ` Tim Chen 2017-03-24 18:23 ` Tim Chen 2017-03-24 18:30 ` Zi Yan 2017-03-24 18:30 ` Zi Yan 2017-03-13 15:44 ` [PATCH v4 02/11] mm: mempolicy: add queue_pages_node_check() Zi Yan 2017-03-13 15:44 ` Zi Yan 2017-03-13 15:44 ` [PATCH v4 03/11] mm: thp: introduce separate TTU flag for thp freezing Zi Yan 2017-03-13 15:44 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 04/11] mm: thp: introduce CONFIG_ARCH_ENABLE_THP_MIGRATION Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-24 14:10 ` Kirill A. Shutemov 2017-03-24 14:10 ` Kirill A. Shutemov 2017-03-24 14:21 ` Zi Yan 2017-03-24 14:21 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 05/11] mm: thp: enable thp migration in generic path Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-14 21:19 ` kbuild test robot 2017-03-14 21:55 ` Zi Yan 2017-03-14 21:55 ` Zi Yan 2017-03-15 9:01 ` Geert Uytterhoeven 2017-03-15 9:01 ` Geert Uytterhoeven 2017-03-15 16:00 ` Zi Yan 2017-03-15 16:00 ` Zi Yan 2017-03-14 21:26 ` kbuild test robot 2017-03-24 14:28 ` Kirill A. Shutemov 2017-03-24 14:28 ` Kirill A. Shutemov 2017-03-24 15:30 ` Zi Yan 2017-03-24 15:30 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 06/11] mm: thp: check pmd migration entry in common path Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-24 14:50 ` Kirill A. Shutemov 2017-03-24 14:50 ` Kirill A. Shutemov 2017-03-24 16:09 ` Zi Yan 2017-03-24 16:09 ` Zi Yan 2017-03-24 16:50 ` Kirill A. Shutemov 2017-03-24 16:50 ` Kirill A. Shutemov 2017-03-24 17:09 ` Zi Yan 2017-03-24 17:09 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 07/11] mm: soft-dirty: keep soft-dirty bits over thp migration Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 08/11] mm: hwpoison: soft offline supports " Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-13 15:45 ` Zi Yan [this message] 2017-03-13 15:45 ` [PATCH v4 09/11] mm: mempolicy: mbind and migrate_pages support " Zi Yan 2017-03-13 15:45 ` [PATCH v4 10/11] mm: migrate: move_pages() supports " Zi Yan 2017-03-13 15:45 ` Zi Yan 2017-03-13 15:45 ` [PATCH v4 11/11] mm: memory_hotplug: memory hotremove " Zi Yan 2017-03-13 15:45 ` Zi Yan
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20170313154507.3647-10-zi.yan@sent.com \ --to=zi.yan@sent.com \ --cc=akpm@linux-foundation.org \ --cc=dnellans@nvidia.com \ --cc=khandual@linux.vnet.ibm.com \ --cc=kirill.shutemov@linux.intel.com \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=mgorman@techsingularity.net \ --cc=mhocko@kernel.org \ --cc=minchan@kernel.org \ --cc=n-horiguchi@ah.jp.nec.com \ --cc=vbabka@suse.cz \ --cc=zi.yan@cs.rutgers.edu \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.