linux-fsdevel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Matthew Wilcox <willy@infradead.org>
To: linux-mm@kvack.org, linux-fsdevel@vger.kernel.org,
	linux-kernel@vger.kernel.org
Cc: Matthew Wilcox <willy@infradead.org>, Jan Kara <jack@suse.cz>,
	Jeff Layton <jlayton@redhat.com>,
	Lukas Czerner <lczerner@redhat.com>,
	Ross Zwisler <ross.zwisler@linux.intel.com>,
	Christoph Hellwig <hch@lst.de>,
	Goldwyn Rodrigues <rgoldwyn@suse.com>,
	Nicholas Piggin <npiggin@gmail.com>,
	Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>,
	linux-nilfs@vger.kernel.org, Jaegeuk Kim <jaegeuk@kernel.org>,
	Chao Yu <yuchao0@huawei.com>,
	linux-f2fs-devel@lists.sourceforge.net
Subject: [PATCH v14 64/74] dax: Convert dax_insert_pfn_mkwrite to XArray
Date: Sat, 16 Jun 2018 19:00:42 -0700	[thread overview]
Message-ID: <20180617020052.4759-65-willy@infradead.org> (raw)
In-Reply-To: <20180617020052.4759-1-willy@infradead.org>

Add some XArray-based helper functions to replace the radix tree based
metaphors currently in use.  The biggest change is that converted code
doesn't see its own lock bit; get_unlocked_entry() always returns an
entry with the lock bit clear, and locking the entry now returns void.
So we don't have to mess around loading the current entry and clearing
the lock bit; we can just store the entry that we were using.

Signed-off-by: Matthew Wilcox <willy@infradead.org>
---
 fs/dax.c | 145 +++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 113 insertions(+), 32 deletions(-)

diff --git a/fs/dax.c b/fs/dax.c
index b7f54e386da8..71181f4bb1d3 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -38,6 +38,17 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/fs_dax.h>
 
+static inline unsigned int pe_order(enum page_entry_size pe_size)
+{
+	if (pe_size == PE_SIZE_PTE)
+		return PAGE_SHIFT - PAGE_SHIFT;
+	if (pe_size == PE_SIZE_PMD)
+		return PMD_SHIFT - PAGE_SHIFT;
+	if (pe_size == PE_SIZE_PUD)
+		return PUD_SHIFT - PAGE_SHIFT;
+	return ~0;
+}
+
 /* We choose 4096 entries - same as per-zone page wait tables */
 #define DAX_WAIT_TABLE_BITS 12
 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
@@ -46,6 +57,9 @@
 #define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
 #define PG_PMD_NR	(PMD_SIZE >> PAGE_SHIFT)
 
+/* The order of a PMD entry */
+#define PMD_ORDER	(PMD_SHIFT - PAGE_SHIFT)
+
 static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
 
 static int __init init_dax_wait_table(void)
@@ -85,10 +99,15 @@ static void *dax_make_locked(unsigned long pfn, unsigned long flags)
 			DAX_LOCKED);
 }
 
+static bool dax_is_locked(void *entry)
+{
+	return xa_to_value(entry) & DAX_LOCKED;
+}
+
 static unsigned int dax_entry_order(void *entry)
 {
 	if (xa_to_value(entry) & DAX_PMD)
-		return PMD_SHIFT - PAGE_SHIFT;
+		return PMD_ORDER;
 	return 0;
 }
 
@@ -181,6 +200,77 @@ static void dax_wake_mapping_entry_waiter(struct xarray *xa,
 		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
 }
 
+static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
+{
+	return dax_wake_mapping_entry_waiter(xas->xa, xas->xa_index, entry,
+								wake_all);
+}
+
+/*
+ * Look up entry in page cache, wait for it to become unlocked if it
+ * is a DAX entry and return it.  The caller must subsequently call
+ * put_unlocked_entry() if it did not lock the entry or put_locked_entry()
+ * if it did.
+ *
+ * Must be called with the i_pages lock held.
+ */
+static void *get_unlocked_entry(struct xa_state *xas)
+{
+	void *entry;
+	struct wait_exceptional_entry_queue ewait;
+	wait_queue_head_t *wq;
+
+	init_wait(&ewait.wait);
+	ewait.wait.func = wake_exceptional_entry_func;
+
+	for (;;) {
+		entry = xas_load(xas);
+		if (!entry || WARN_ON_ONCE(!xa_is_value(entry)) ||
+				!dax_is_locked(entry))
+			return entry;
+
+		wq = dax_entry_waitqueue(xas->xa, xas->xa_index, entry,
+				&ewait.key);
+		prepare_to_wait_exclusive(wq, &ewait.wait,
+					  TASK_UNINTERRUPTIBLE);
+		xas_unlock_irq(xas);
+		xas_reset(xas);
+		schedule();
+		finish_wait(wq, &ewait.wait);
+		xas_lock_irq(xas);
+	}
+}
+
+static void put_unlocked_entry(struct xa_state *xas, void *entry)
+{
+	/* If we were the only waiter woken, wake the next one */
+	if (entry)
+		dax_wake_entry(xas, entry, false);
+}
+
+/*
+ * We used the xa_state to get the entry, but then we locked the entry and
+ * dropped the xa_lock, so we know the xa_state is stale and must be reset
+ * before use.
+ */
+static void put_locked_entry(struct xa_state *xas, void *entry)
+{
+	void *old;
+
+	xas_reset(xas);
+	xas_lock_irq(xas);
+	old = xas_store(xas, entry);
+	xas_unlock_irq(xas);
+	BUG_ON(!dax_is_locked(old));
+	dax_wake_entry(xas, entry, false);
+}
+
+static void dax_lock_entry(struct xa_state *xas, void *entry)
+{
+	unsigned long v = xa_to_value(entry);
+	xas_store(xas, xa_mk_value(v | DAX_LOCKED));
+}
+
 /*
  * Check whether the given slot is locked.  Must be called with the i_pages
  * lock held.
@@ -1705,50 +1795,46 @@ EXPORT_SYMBOL_GPL(dax_iomap_fault);
 /*
  * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
  * @vmf: The description of the fault
- * @pe_size: Size of entry to be inserted
  * @pfn: PFN to insert
+ * @order: Order of entry to insert.
  *
  * This function inserts a writeable PTE or PMD entry into the page tables
  * for an mmaped DAX file.  It also marks the page cache entry as dirty.
  */
-static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf,
-				  enum page_entry_size pe_size,
-				  pfn_t pfn)
+static vm_fault_t
+dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
 {
 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
-	void *entry, **slot;
-	pgoff_t index = vmf->pgoff;
+	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
+	void *entry;
 	vm_fault_t ret;
 
-	xa_lock_irq(&mapping->i_pages);
-	entry = get_unlocked_mapping_entry(mapping, index, &slot);
+	xas_lock_irq(&xas);
+	entry = get_unlocked_entry(&xas);
 	/* Did we race with someone splitting entry or so? */
 	if (!entry ||
-	    (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
-	    (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
-		put_unlocked_mapping_entry(mapping, index, entry);
-		xa_unlock_irq(&mapping->i_pages);
+	    (order == 0 && !dax_is_pte_entry(entry)) ||
+	    (order == PMD_ORDER && (xa_is_internal(entry) ||
+				    !dax_is_pmd_entry(entry)))) {
+		put_unlocked_entry(&xas, entry);
+		xas_unlock_irq(&xas);
 		trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
 						      VM_FAULT_NOPAGE);
 		return VM_FAULT_NOPAGE;
 	}
-	radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY);
-	entry = lock_slot(mapping, slot);
-	xa_unlock_irq(&mapping->i_pages);
-	switch (pe_size) {
-	case PE_SIZE_PTE:
+	xas_set_tag(&xas, PAGECACHE_TAG_DIRTY);
+	dax_lock_entry(&xas, entry);
+	xas_unlock_irq(&xas);
+	if (order == 0)
 		ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
-		break;
 #ifdef CONFIG_FS_DAX_PMD
-	case PE_SIZE_PMD:
+	else if (order == PMD_ORDER)
 		ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
 			pfn, true);
-		break;
 #endif
-	default:
+	else
 		ret = VM_FAULT_FALLBACK;
-	}
-	put_locked_mapping_entry(mapping, index);
+	put_locked_entry(&xas, entry);
 	trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
 	return ret;
 }
@@ -1768,17 +1854,12 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
 {
 	int err;
 	loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
-	size_t len = 0;
+	unsigned int order = pe_order(pe_size);
+	size_t len = PAGE_SIZE << order;
 
-	if (pe_size == PE_SIZE_PTE)
-		len = PAGE_SIZE;
-	else if (pe_size == PE_SIZE_PMD)
-		len = PMD_SIZE;
-	else
-		WARN_ON_ONCE(1);
 	err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
 	if (err)
 		return VM_FAULT_SIGBUS;
-	return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
+	return dax_insert_pfn_mkwrite(vmf, pfn, order);
 }
 EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
-- 
2.17.1

  parent reply	other threads:[~2018-06-17  2:00 UTC|newest]

Thread overview: 90+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-06-17  1:59 [PATCH v14 00/74] Convert page cache " Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 01/74] Update email address Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 02/74] radix tree test suite: Enable ubsan Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 03/74] dax: Fix use of zero page Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 04/74] xarray: Replace exceptional entries Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 05/74] xarray: Change definition of sibling entries Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 06/74] xarray: Add definition of struct xarray Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 07/74] xarray: Define struct xa_node Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 08/74] xarray: Add documentation Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 09/74] xarray: Add XArray load operation Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 10/74] xarray: Add XArray tags Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 11/74] xarray: Add XArray unconditional store operations Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 12/74] xarray: Add XArray conditional " Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 13/74] xarray: Add XArray iterators Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 14/74] xarray: Extract entries from an XArray Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 15/74] xarray: Destroy " Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 16/74] xarray: Step through " Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 17/74] xarray: Add xas_for_each_conflict Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 18/74] xarray: Add xas_create_range Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 19/74] xarray: Add MAINTAINERS entry Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 20/74] page cache: Rearrange address_space Matthew Wilcox
2018-06-17  1:59 ` [PATCH v14 21/74] page cache: Convert hole search to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 22/74] page cache: Add and replace pages using the XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 23/74] page cache: Convert page deletion to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 24/74] page cache: Convert find_get_entry " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 25/74] page cache: Convert find_get_entries " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 26/74] page cache: Convert find_get_pages_range " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 27/74] page cache: Convert find_get_pages_contig " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 28/74] page cache; Convert find_get_pages_range_tag " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 29/74] page cache: Convert find_get_entries_tag " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 30/74] page cache: Convert filemap_map_pages " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 31/74] radix tree test suite: Convert regression1 " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 32/74] page cache: Convert delete_batch " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 33/74] page cache: Remove stray radix comment Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 34/74] page cache: Convert filemap_range_has_page to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 35/74] mm: Convert page-writeback " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 36/74] mm: Convert workingset " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 37/74] mm: Convert truncate " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 38/74] mm: Convert add_to_swap_cache " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 39/74] mm: Convert delete_from_swap_cache " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 40/74] mm: Convert __do_page_cache_readahead " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 41/74] mm: Convert page migration " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 42/74] mm: Convert huge_memory " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 43/74] mm: Convert collapse_shmem " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 44/74] mm: Convert khugepaged_scan_shmem " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 45/74] mm: Convert is_page_cache_freeable " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 46/74] pagevec: Use xa_tag_t Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 47/74] shmem: Convert shmem_radix_tree_replace to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 48/74] shmem: Convert shmem_confirm_swap " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 49/74] shmem: Convert find_swap_entry " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 50/74] shmem: Convert shmem_add_to_page_cache " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 51/74] shmem: Convert shmem_alloc_hugepage " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 52/74] shmem: Convert shmem_free_swap " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 53/74] shmem: Convert shmem_partial_swap_usage " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 54/74] memfd: Convert memfd_wait_for_pins " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 55/74] memfd: Convert memfd_tag_pins " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 56/74] shmem: Comment fixups Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 57/74] btrfs: Convert page cache to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 58/74] fs: Convert buffer " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 59/74] fs: Convert writeback " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 60/74] nilfs2: Convert " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 61/74] f2fs: " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 62/74] dax: Rename some functions Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 63/74] dax: Hash on XArray instead of mapping Matthew Wilcox
2018-06-17  2:00 ` Matthew Wilcox [this message]
2018-06-17  2:00 ` [PATCH v14 65/74] dax: Convert dax_layout_busy_page to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 66/74] dax: Convert __dax_invalidate_entry " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 67/74] dax: Convert dax writeback " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 68/74] dax: Convert dax_lock_page " Matthew Wilcox
2018-06-29 17:30   ` Ross Zwisler
2018-07-06 11:54     ` [PATCH] fs/dax: remove unused function Anders Roxell
2018-07-06 13:04     ` [PATCH v14 68/74] dax: Convert dax_lock_page to XArray Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 69/74] dax: Convert page fault handlers " Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 70/74] page cache: Finish XArray conversion Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 71/74] radix tree: Remove radix_tree_update_node_t Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 72/74] radix tree: Remove split/join code Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 73/74] radix tree: Remove radix_tree_maybe_preload_order Matthew Wilcox
2018-06-17  2:00 ` [PATCH v14 74/74] radix tree: Remove radix_tree_clear_tags Matthew Wilcox
2018-06-19  3:12 ` [PATCH v14 00/74] Convert page cache to XArray Ross Zwisler
2018-06-19  9:22   ` Matthew Wilcox
2018-06-19 16:40     ` Ross Zwisler
2018-06-19 17:16       ` Matthew Wilcox
2018-06-27 11:05         ` Matthew Wilcox
2018-06-27 19:44           ` Ross Zwisler
2018-06-28  8:39             ` Matthew Wilcox
2018-06-28 16:30               ` Ross Zwisler
2018-07-25 21:03             ` Matthew Wilcox
2018-07-25 21:12               ` Ross Zwisler
2018-07-27 17:20               ` Ross Zwisler
2018-07-30 15:43                 ` Matthew Wilcox

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20180617020052.4759-65-willy@infradead.org \
    --to=willy@infradead.org \
    --cc=hch@lst.de \
    --cc=jack@suse.cz \
    --cc=jaegeuk@kernel.org \
    --cc=jlayton@redhat.com \
    --cc=konishi.ryusuke@lab.ntt.co.jp \
    --cc=lczerner@redhat.com \
    --cc=linux-f2fs-devel@lists.sourceforge.net \
    --cc=linux-fsdevel@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-mm@kvack.org \
    --cc=linux-nilfs@vger.kernel.org \
    --cc=npiggin@gmail.com \
    --cc=rgoldwyn@suse.com \
    --cc=ross.zwisler@linux.intel.com \
    --cc=yuchao0@huawei.com \
    --subject='Re: [PATCH v14 64/74] dax: Convert dax_insert_pfn_mkwrite to XArray' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
on how to clone and mirror all data and code used for this inbox