From: Dan Williams <dan.j.williams@intel.com>
To: linux-nvdimm@lists.01.org
Cc: jack@suse.cz, Peter Zijlstra <peterz@infradead.org>,
david@fromorbit.com, linux-kernel@vger.kernel.org,
linux-xfs@vger.kernel.org, Ingo Molnar <mingo@redhat.com>,
linux-fsdevel@vger.kernel.org, hch@lst.de
Subject: [PATCH v5 08/11] wait_bit: introduce {wait_on,wake_up}_atomic_one
Date: Fri, 09 Mar 2018 22:55:32 -0800 [thread overview]
Message-ID: <152066493247.40260.10849841915366086021.stgit@dwillia2-desk3.amr.corp.intel.com> (raw)
In-Reply-To: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com>
Add a generic facility for awaiting an atomic_t to reach a value of 1.
Page reference counts typically need to reach 0 to be considered a
free / inactive page. However, ZONE_DEVICE pages allocated via
devm_memremap_pages() are never 'onlined', i.e. the put_page() typically
done at init time to assign pages to the page allocator is skipped.
These pages will have their reference count elevated > 1 by
get_user_pages() when they are under DMA. In order to coordinate DMA to
these pages vs filesytem operations like hole-punch and truncate the
filesystem-dax implementation needs to capture the DMA-idle event i.e.
the 2 to 1 count transition).
For now, this implementation does not have functional behavior change,
follow-on patches will add waiters for these page-idle events.
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/dax/super.c | 2 +-
include/linux/wait_bit.h | 13 ++++++++++
kernel/sched/wait_bit.c | 59 +++++++++++++++++++++++++++++++++++++++-------
3 files changed, 64 insertions(+), 10 deletions(-)
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 619b1ed6434c..7e10fa3460e2 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -167,7 +167,7 @@ struct dax_device {
#if IS_ENABLED(CONFIG_FS_DAX)
static void generic_dax_pagefree(struct page *page, void *data)
{
- /* TODO: wakeup page-idle waiters */
+ wake_up_atomic_one(&page->_refcount);
}
struct dax_device *fs_dax_claim_bdev(struct block_device *bdev, void *owner)
diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h
index 61b39eaf7cad..564c9a0141cd 100644
--- a/include/linux/wait_bit.h
+++ b/include/linux/wait_bit.h
@@ -33,10 +33,15 @@ int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *
int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
void wake_up_bit(void *word, int bit);
void wake_up_atomic_t(atomic_t *p);
+static inline void wake_up_atomic_one(atomic_t *p)
+{
+ wake_up_atomic_t(p);
+}
int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode);
int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode);
int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode);
+int out_of_line_wait_on_atomic_one(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode);
struct wait_queue_head *bit_waitqueue(void *word, int bit);
extern void __init wait_bit_init(void);
@@ -262,4 +267,12 @@ int wait_on_atomic_t(atomic_t *val, wait_atomic_t_action_f action, unsigned mode
return out_of_line_wait_on_atomic_t(val, action, mode);
}
+static inline
+int wait_on_atomic_one(atomic_t *val, wait_atomic_t_action_f action, unsigned mode)
+{
+ might_sleep();
+ if (atomic_read(val) == 1)
+ return 0;
+ return out_of_line_wait_on_atomic_one(val, action, mode);
+}
#endif /* _LINUX_WAIT_BIT_H */
diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c
index 84cb3acd9260..8739b1e50df5 100644
--- a/kernel/sched/wait_bit.c
+++ b/kernel/sched/wait_bit.c
@@ -162,28 +162,47 @@ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
return bit_waitqueue(p, 0);
}
-static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
- void *arg)
+static struct wait_bit_queue_entry *to_wait_bit_q(
+ struct wait_queue_entry *wq_entry)
+{
+ return container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
+}
+
+static int __wake_atomic_t_function(struct wait_queue_entry *wq_entry,
+ unsigned mode, int sync, void *arg, int target)
{
struct wait_bit_key *key = arg;
- struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
+ struct wait_bit_queue_entry *wait_bit = to_wait_bit_q(wq_entry);
atomic_t *val = key->flags;
if (wait_bit->key.flags != key->flags ||
wait_bit->key.bit_nr != key->bit_nr ||
- atomic_read(val) != 0)
+ atomic_read(val) != target)
return 0;
return autoremove_wake_function(wq_entry, mode, sync, key);
}
+static int wake_atomic_t_function(struct wait_queue_entry *wq_entry,
+ unsigned mode, int sync, void *arg)
+{
+ return __wake_atomic_t_function(wq_entry, mode, sync, arg, 0);
+}
+
+static int wake_atomic_one_function(struct wait_queue_entry *wq_entry,
+ unsigned mode, int sync, void *arg)
+{
+ return __wake_atomic_t_function(wq_entry, mode, sync, arg, 1);
+}
+
/*
* To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
* the actions of __wait_on_atomic_t() are permitted return codes. Nonzero
* return codes halt waiting and return.
*/
static __sched
-int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
- wait_atomic_t_action_f action, unsigned int mode)
+int __wait_on_atomic_t(struct wait_queue_head *wq_head,
+ struct wait_bit_queue_entry *wbq_entry,
+ wait_atomic_t_action_f action, unsigned int mode, int target)
{
atomic_t *val;
int ret = 0;
@@ -191,10 +210,10 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en
do {
prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
val = wbq_entry->key.flags;
- if (atomic_read(val) == 0)
+ if (atomic_read(val) == target)
break;
ret = (*action)(val, mode);
- } while (!ret && atomic_read(val) != 0);
+ } while (!ret && atomic_read(val) != target);
finish_wait(wq_head, &wbq_entry->wq_entry);
return ret;
}
@@ -210,6 +229,17 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en
}, \
}
+#define DEFINE_WAIT_ATOMIC_ONE(name, p) \
+ struct wait_bit_queue_entry name = { \
+ .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \
+ .wq_entry = { \
+ .private = current, \
+ .func = wake_atomic_one_function, \
+ .entry = \
+ LIST_HEAD_INIT((name).wq_entry.entry), \
+ }, \
+ }
+
__sched int out_of_line_wait_on_atomic_t(atomic_t *p,
wait_atomic_t_action_f action,
unsigned int mode)
@@ -217,7 +247,7 @@ __sched int out_of_line_wait_on_atomic_t(atomic_t *p,
struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
DEFINE_WAIT_ATOMIC_T(wq_entry, p);
- return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
+ return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 0);
}
EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
@@ -230,6 +260,17 @@ __sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
}
EXPORT_SYMBOL(atomic_t_wait);
+__sched int out_of_line_wait_on_atomic_one(atomic_t *p,
+ wait_atomic_t_action_f action,
+ unsigned int mode)
+{
+ struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
+ DEFINE_WAIT_ATOMIC_ONE(wq_entry, p);
+
+ return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 1);
+}
+EXPORT_SYMBOL(out_of_line_wait_on_atomic_one);
+
/**
* wake_up_atomic_t - Wake up a waiter on a atomic_t
* @p: The atomic_t being waited on, a kernel virtual address
_______________________________________________
Linux-nvdimm mailing list
Linux-nvdimm@lists.01.org
https://lists.01.org/mailman/listinfo/linux-nvdimm
next prev parent reply other threads:[~2018-03-10 6:58 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-10 6:54 [PATCH v5 00/11] dax: fix dma vs truncate/hole-punch Dan Williams
2018-03-10 6:54 ` [PATCH v5 01/11] dax: store pfns in the radix Dan Williams
2018-03-10 6:54 ` [PATCH v5 02/11] xfs, dax: introduce xfs_dax_aops Dan Williams
2018-03-10 9:46 ` Christoph Hellwig
2018-03-10 17:40 ` Dan Williams
2018-03-11 19:16 ` Dan Williams
2018-03-12 7:51 ` Christoph Hellwig
2018-03-10 6:55 ` [PATCH v5 03/11] ext4, dax: introduce ext4_dax_aops Dan Williams
2018-03-10 6:55 ` [PATCH v5 04/11] ext2, dax: introduce ext2_dax_aops Dan Williams
2018-03-10 6:55 ` [PATCH v5 05/11] fs, dax: use page->mapping to warn if truncate collides with a busy page Dan Williams
2018-03-10 6:55 ` [PATCH v5 06/11] mm, dax: enable filesystems to trigger dev_pagemap ->page_free callbacks Dan Williams
2018-03-12 14:09 ` Jerome Glisse
2018-03-10 6:55 ` [PATCH v5 07/11] mm, dev_pagemap: introduce CONFIG_DEV_PAGEMAP_OPS Dan Williams
2018-03-12 14:17 ` Jerome Glisse
2018-03-12 18:17 ` Dan Williams
2018-03-10 6:55 ` Dan Williams [this message]
2018-03-11 11:27 ` [PATCH v5 08/11] wait_bit: introduce {wait_on, wake_up}_atomic_one Peter Zijlstra
2018-03-11 17:15 ` Dan Williams
2018-03-12 19:32 ` Dan Williams
2018-03-13 10:20 ` [RFC][PATCH] sched/wait_bit: Introduce wait_var_event()/wake_up_var() Peter Zijlstra
2018-03-14 4:12 ` Dan Williams
2018-03-15 5:46 ` Dan Williams
2018-03-15 9:58 ` David Howells
2018-03-15 11:19 ` Peter Zijlstra
2018-03-15 11:51 ` Peter Zijlstra
2018-03-15 14:45 ` David Howells
2018-03-15 14:53 ` Peter Zijlstra
2018-03-10 6:55 ` [PATCH v5 09/11] mm, fs, dax: handle layout changes to pinned dax mappings Dan Williams
2018-03-10 6:55 ` [PATCH v5 10/11] xfs: prepare xfs_break_layouts() for another layout type Dan Williams
2018-03-10 9:51 ` Christoph Hellwig
2018-03-10 6:55 ` [PATCH v5 11/11] xfs, dax: introduce xfs_break_dax_layouts() Dan Williams
2018-03-10 9:55 ` Christoph Hellwig
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=152066493247.40260.10849841915366086021.stgit@dwillia2-desk3.amr.corp.intel.com \
--to=dan.j.williams@intel.com \
--cc=david@fromorbit.com \
--cc=hch@lst.de \
--cc=jack@suse.cz \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nvdimm@lists.01.org \
--cc=linux-xfs@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).