From: Matthew Wilcox <willy@infradead.org>
To: linux-mm@kvack.org, linux-fsdevel@vger.kernel.org
Cc: Matthew Wilcox <mawilcox@microsoft.com>, Jan Kara <jack@suse.cz>,
Jeff Layton <jlayton@redhat.com>,
Lukas Czerner <lczerner@redhat.com>,
Ross Zwisler <ross.zwisler@linux.intel.com>,
Christoph Hellwig <hch@lst.de>,
Goldwyn Rodrigues <rgoldwyn@suse.com>,
Nicholas Piggin <npiggin@gmail.com>,
Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>,
linux-nilfs@vger.kernel.org, Jaegeuk Kim <jaegeuk@kernel.org>,
Chao Yu <yuchao0@huawei.com>,
linux-f2fs-devel@lists.sourceforge.net,
Oleg Drokin <oleg.drokin@intel.com>,
Andreas Dilger <andreas.dilger@intel.com>,
James Simmons <jsimmons@infradead.org>,
Mike Kravetz <mike.kravetz@oracle.com>
Subject: [PATCH v10 33/62] mm: Convert collapse_shmem to XArray
Date: Thu, 29 Mar 2018 20:42:16 -0700 [thread overview]
Message-ID: <20180330034245.10462-34-willy@infradead.org> (raw)
In-Reply-To: <20180330034245.10462-1-willy@infradead.org>
From: Matthew Wilcox <mawilcox@microsoft.com>
I found another victim of the radix tree being hard to use. Because
there was no call to radix_tree_preload(), khugepaged was allocating
radix_tree_nodes using GFP_ATOMIC.
I also converted a local_irq_save()/restore() pair to
disable()/enable().
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
---
mm/khugepaged.c | 158 +++++++++++++++++++++++---------------------------------
1 file changed, 65 insertions(+), 93 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index a38764f666db..193bacac91a8 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1289,17 +1289,17 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
*
* Basic scheme is simple, details are more complex:
* - allocate and freeze a new huge page;
- * - scan over radix tree replacing old pages the new one
+ * - scan page cache replacing old pages with the new one
* + swap in pages if necessary;
* + fill in gaps;
- * + keep old pages around in case if rollback is required;
- * - if replacing succeed:
+ * + keep old pages around in case rollback is required;
+ * - if replacing succeeds:
* + copy data over;
* + free old pages;
* + unfreeze huge page;
* - if replacing failed;
* + put all pages back and unfreeze them;
- * + restore gaps in the radix-tree;
+ * + restore gaps in the page cache;
* + free huge page;
*/
static void collapse_shmem(struct mm_struct *mm,
@@ -1307,12 +1307,11 @@ static void collapse_shmem(struct mm_struct *mm,
struct page **hpage, int node)
{
gfp_t gfp;
- struct page *page, *new_page, *tmp;
+ struct page *new_page;
struct mem_cgroup *memcg;
pgoff_t index, end = start + HPAGE_PMD_NR;
LIST_HEAD(pagelist);
- struct radix_tree_iter iter;
- void **slot;
+ XA_STATE(xas, &mapping->i_pages, start);
int nr_none = 0, result = SCAN_SUCCEED;
VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
@@ -1339,48 +1338,48 @@ static void collapse_shmem(struct mm_struct *mm,
__SetPageLocked(new_page);
BUG_ON(!page_ref_freeze(new_page, 1));
-
/*
- * At this point the new_page is 'frozen' (page_count() is zero), locked
- * and not up-to-date. It's safe to insert it into radix tree, because
- * nobody would be able to map it or use it in other way until we
- * unfreeze it.
+ * At this point the new_page is 'frozen' (page_count() is zero),
+ * locked and not up-to-date. It's safe to insert it into the page
+ * cache, because nobody would be able to map it or use it in other
+ * way until we unfreeze it.
*/
- index = start;
- xa_lock_irq(&mapping->i_pages);
- radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
- int n = min(iter.index, end) - index;
-
- /*
- * Handle holes in the radix tree: charge it from shmem and
- * insert relevant subpage of new_page into the radix-tree.
- */
- if (n && !shmem_charge(mapping->host, n)) {
- result = SCAN_FAIL;
+ /* This will be less messy when we use multi-index entries */
+ do {
+ xas_lock_irq(&xas);
+ xas_create_range(&xas, end - 1);
+ if (!xas_error(&xas))
break;
- }
- nr_none += n;
- for (; index < min(iter.index, end); index++) {
- radix_tree_insert(&mapping->i_pages, index,
- new_page + (index % HPAGE_PMD_NR));
- }
+ xas_unlock_irq(&xas);
+ if (!xas_nomem(&xas, GFP_KERNEL))
+ goto out;
+ } while (1);
- /* We are done. */
- if (index >= end)
- break;
+ for (index = start; index < end; index++) {
+ struct page *page = xas_next(&xas);
+
+ VM_BUG_ON(index != xas.xa_index);
+ if (!page) {
+ if (!shmem_charge(mapping->host, 1)) {
+ result = SCAN_FAIL;
+ break;
+ }
+ xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
+ nr_none++;
+ continue;
+ }
- page = radix_tree_deref_slot_protected(slot,
- &mapping->i_pages.xa_lock);
if (xa_is_value(page) || !PageUptodate(page)) {
- xa_unlock_irq(&mapping->i_pages);
+ xas_unlock_irq(&xas);
/* swap in or instantiate fallocated page */
if (shmem_getpage(mapping->host, index, &page,
SGP_NOHUGE)) {
result = SCAN_FAIL;
- goto tree_unlocked;
+ goto xa_unlocked;
}
- xa_lock_irq(&mapping->i_pages);
+ xas_lock_irq(&xas);
+ xas_set(&xas, index);
} else if (trylock_page(page)) {
get_page(page);
} else {
@@ -1400,7 +1399,7 @@ static void collapse_shmem(struct mm_struct *mm,
result = SCAN_TRUNCATED;
goto out_unlock;
}
- xa_unlock_irq(&mapping->i_pages);
+ xas_unlock_irq(&xas);
if (isolate_lru_page(page)) {
result = SCAN_DEL_PAGE_LRU;
@@ -1410,17 +1409,16 @@ static void collapse_shmem(struct mm_struct *mm,
if (page_mapped(page))
unmap_mapping_pages(mapping, index, 1, false);
- xa_lock_irq(&mapping->i_pages);
+ xas_lock(&xas);
+ xas_set(&xas, index);
- slot = radix_tree_lookup_slot(&mapping->i_pages, index);
- VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot,
- &mapping->i_pages.xa_lock), page);
+ VM_BUG_ON_PAGE(page != xas_load(&xas), page);
VM_BUG_ON_PAGE(page_mapped(page), page);
/*
* The page is expected to have page_count() == 3:
* - we hold a pin on it;
- * - one reference from radix tree;
+ * - one reference from page cache;
* - one from isolate_lru_page;
*/
if (!page_ref_freeze(page, 3)) {
@@ -1435,56 +1433,30 @@ static void collapse_shmem(struct mm_struct *mm,
list_add_tail(&page->lru, &pagelist);
/* Finally, replace with the new page. */
- radix_tree_replace_slot(&mapping->i_pages, slot,
- new_page + (index % HPAGE_PMD_NR));
-
- slot = radix_tree_iter_resume(slot, &iter);
- index++;
+ xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
continue;
out_lru:
- xa_unlock_irq(&mapping->i_pages);
+ xas_unlock_irq(&xas);
putback_lru_page(page);
out_isolate_failed:
unlock_page(page);
put_page(page);
- goto tree_unlocked;
+ goto xa_unlocked;
out_unlock:
unlock_page(page);
put_page(page);
break;
}
+ xas_unlock_irq(&xas);
- /*
- * Handle hole in radix tree at the end of the range.
- * This code only triggers if there's nothing in radix tree
- * beyond 'end'.
- */
- if (result == SCAN_SUCCEED && index < end) {
- int n = end - index;
-
- if (!shmem_charge(mapping->host, n)) {
- result = SCAN_FAIL;
- goto tree_locked;
- }
-
- for (; index < end; index++) {
- radix_tree_insert(&mapping->i_pages, index,
- new_page + (index % HPAGE_PMD_NR));
- }
- nr_none += n;
- }
-
-tree_locked:
- xa_unlock_irq(&mapping->i_pages);
-tree_unlocked:
-
+xa_unlocked:
if (result == SCAN_SUCCEED) {
- unsigned long flags;
+ struct page *page, *tmp;
struct zone *zone = page_zone(new_page);
/*
- * Replacing old pages with new one has succeed, now we need to
- * copy the content and free old pages.
+ * Replacing old pages with new one has succeeded, now we
+ * need to copy the content and free the old pages.
*/
list_for_each_entry_safe(page, tmp, &pagelist, lru) {
copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
@@ -1498,16 +1470,16 @@ static void collapse_shmem(struct mm_struct *mm,
put_page(page);
}
- local_irq_save(flags);
+ local_irq_disable();
__inc_node_page_state(new_page, NR_SHMEM_THPS);
if (nr_none) {
__mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
__mod_node_page_state(zone->zone_pgdat, NR_SHMEM, nr_none);
}
- local_irq_restore(flags);
+ local_irq_enable();
/*
- * Remove pte page tables, so we can re-faulti
+ * Remove pte page tables, so we can re-fault
* the page as huge.
*/
retract_page_tables(mapping, start);
@@ -1522,37 +1494,37 @@ static void collapse_shmem(struct mm_struct *mm,
*hpage = NULL;
} else {
- /* Something went wrong: rollback changes to the radix-tree */
+ struct page *page;
+ /* Something went wrong: roll back page cache changes */
shmem_uncharge(mapping->host, nr_none);
- xa_lock_irq(&mapping->i_pages);
- radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
- if (iter.index >= end)
- break;
+ xas_lock_irq(&xas);
+ xas_set(&xas, start);
+ xas_for_each(&xas, page, end - 1) {
page = list_first_entry_or_null(&pagelist,
struct page, lru);
- if (!page || iter.index < page->index) {
+ if (!page || xas.xa_index < page->index) {
if (!nr_none)
break;
nr_none--;
/* Put holes back where they were */
- radix_tree_delete(&mapping->i_pages, iter.index);
+ xas_store(&xas, NULL);
continue;
}
- VM_BUG_ON_PAGE(page->index != iter.index, page);
+ VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
/* Unfreeze the page. */
list_del(&page->lru);
page_ref_unfreeze(page, 2);
- radix_tree_replace_slot(&mapping->i_pages, slot, page);
- slot = radix_tree_iter_resume(slot, &iter);
- xa_unlock_irq(&mapping->i_pages);
+ xas_store(&xas, page);
+ xas_pause(&xas);
+ xas_unlock_irq(&xas);
putback_lru_page(page);
unlock_page(page);
- xa_lock_irq(&mapping->i_pages);
+ xas_lock_irq(&xas);
}
VM_BUG_ON(nr_none);
- xa_unlock_irq(&mapping->i_pages);
+ xas_unlock_irq(&xas);
/* Unfreeze new_page, caller would take care about freeing it */
page_ref_unfreeze(new_page, 1);
--
2.16.2
next prev parent reply other threads:[~2018-03-30 3:42 UTC|newest]
Thread overview: 74+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-30 3:41 [PATCH v10 00/62] Convert page cache to XArray Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 01/62] page cache: Use xa_lock Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 02/62] xarray: Replace exceptional entries Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 03/62] xarray: Change definition of sibling entries Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 04/62] xarray: Add definition of struct xarray Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 05/62] xarray: Define struct xa_node Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 06/62] xarray: Add documentation Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 07/62] xarray: Add xa_load Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 08/62] xarray: Add xa_get_tag, xa_set_tag and xa_clear_tag Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 09/62] xarray: Add xa_store Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 10/62] xarray: Add xa_cmpxchg and xa_insert Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 11/62] xarray: Add xa_for_each Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 12/62] xarray: Add xa_extract Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 13/62] xarray: Add xa_destroy Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 14/62] xarray: Add xas_next and xas_prev Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 15/62] xarray: Add xas_create_range Matthew Wilcox
2018-03-30 3:41 ` [PATCH v10 16/62] xarray: Add MAINTAINERS entry Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 17/62] page cache: Rearrange address_space Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 18/62] page cache: Convert hole search to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 19/62] page cache: Add and replace pages using the XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 20/62] page cache: Convert page deletion to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 21/62] page cache: Convert page cache lookups " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 22/62] page cache: Convert delete_batch " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 23/62] page cache: Remove stray radix comment Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 24/62] page cache: Convert filemap_range_has_page to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 25/62] mm: Convert page-writeback " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 26/62] mm: Convert workingset " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 27/62] mm: Convert truncate " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 28/62] mm: Convert add_to_swap_cache " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 29/62] mm: Convert delete_from_swap_cache " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 30/62] mm: Convert __do_page_cache_readahead " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 31/62] mm: Convert page migration " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 32/62] mm: Convert huge_memory " Matthew Wilcox
2018-03-30 3:42 ` Matthew Wilcox [this message]
2018-03-30 3:42 ` [PATCH v10 34/62] mm: Convert khugepaged_scan_shmem " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 35/62] pagevec: Use xa_tag_t Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 36/62] shmem: Convert replace to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 37/62] shmem: Convert shmem_confirm_swap " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 38/62] shmem: Convert find_swap_entry " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 39/62] shmem: Convert shmem_add_to_page_cache " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 40/62] shmem: Convert shmem_alloc_hugepage " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 41/62] shmem: Convert shmem_free_swap " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 42/62] shmem: Convert shmem_partial_swap_usage " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 43/62] memfd: Convert shmem_tag_pins " Matthew Wilcox
2018-03-31 0:05 ` Mike Kravetz
2018-03-31 2:11 ` Matthew Wilcox
2018-04-03 20:51 ` Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 44/62] memfd: Convert shmem_wait_for_pins " Matthew Wilcox
2018-03-31 0:07 ` Mike Kravetz
2018-03-30 3:42 ` [PATCH v10 45/62] shmem: Comment fixups Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 46/62] btrfs: Convert page cache to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 47/62] fs: Convert buffer " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 48/62] fs: Convert writeback " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 49/62] nilfs2: Convert " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 50/62] f2fs: " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 51/62] lustre: " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 52/62] dax: Fix use of zero page Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 53/62] dax: dax_insert_mapping_entry always succeeds Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 54/62] dax: Rename some functions Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 55/62] dax: Hash on XArray instead of mapping Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 56/62] dax: Convert dax_insert_pfn_mkwrite to XArray Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 57/62] dax: Convert dax_layout_busy_page " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 58/62] dax: Convert __dax_invalidate_entry " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 59/62] dax: Convert dax writeback " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 60/62] dax: Convert page fault handlers " Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 61/62] page cache: Finish XArray conversion Matthew Wilcox
2018-03-30 3:42 ` [PATCH v10 62/62] radix tree: Remove unused functions Matthew Wilcox
2018-04-04 16:35 ` [PATCH v10 00/62] Convert page cache to XArray Mike Kravetz
2018-04-05 3:52 ` Matthew Wilcox
2018-04-05 18:33 ` Mike Kravetz
2018-04-09 21:18 ` Goldwyn Rodrigues
2018-04-14 19:50 ` Matthew Wilcox
2018-04-14 19:58 ` Matthew Wilcox
2018-04-17 21:49 ` Goldwyn Rodrigues
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180330034245.10462-34-willy@infradead.org \
--to=willy@infradead.org \
--cc=andreas.dilger@intel.com \
--cc=hch@lst.de \
--cc=jack@suse.cz \
--cc=jaegeuk@kernel.org \
--cc=jlayton@redhat.com \
--cc=jsimmons@infradead.org \
--cc=konishi.ryusuke@lab.ntt.co.jp \
--cc=lczerner@redhat.com \
--cc=linux-f2fs-devel@lists.sourceforge.net \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-nilfs@vger.kernel.org \
--cc=mawilcox@microsoft.com \
--cc=mike.kravetz@oracle.com \
--cc=npiggin@gmail.com \
--cc=oleg.drokin@intel.com \
--cc=rgoldwyn@suse.com \
--cc=ross.zwisler@linux.intel.com \
--cc=yuchao0@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).