From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ml01.01.org (Postfix) with ESMTPS id DDA8F220C2A68 for ; Fri, 9 Mar 2018 22:58:19 -0800 (PST) Subject: [PATCH v5 08/11] wait_bit: introduce {wait_on,wake_up}_atomic_one From: Dan Williams Date: Fri, 09 Mar 2018 22:55:32 -0800 Message-ID: <152066493247.40260.10849841915366086021.stgit@dwillia2-desk3.amr.corp.intel.com> In-Reply-To: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> References: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> MIME-Version: 1.0 List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: linux-nvdimm-bounces@lists.01.org Sender: "Linux-nvdimm" To: linux-nvdimm@lists.01.org Cc: jack@suse.cz, Peter Zijlstra , david@fromorbit.com, linux-kernel@vger.kernel.org, linux-xfs@vger.kernel.org, Ingo Molnar , linux-fsdevel@vger.kernel.org, hch@lst.de List-ID: Add a generic facility for awaiting an atomic_t to reach a value of 1. Page reference counts typically need to reach 0 to be considered a free / inactive page. However, ZONE_DEVICE pages allocated via devm_memremap_pages() are never 'onlined', i.e. the put_page() typically done at init time to assign pages to the page allocator is skipped. These pages will have their reference count elevated > 1 by get_user_pages() when they are under DMA. In order to coordinate DMA to these pages vs filesytem operations like hole-punch and truncate the filesystem-dax implementation needs to capture the DMA-idle event i.e. the 2 to 1 count transition). For now, this implementation does not have functional behavior change, follow-on patches will add waiters for these page-idle events. Cc: Ingo Molnar Cc: Peter Zijlstra Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- drivers/dax/super.c | 2 +- include/linux/wait_bit.h | 13 ++++++++++ kernel/sched/wait_bit.c | 59 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 619b1ed6434c..7e10fa3460e2 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -167,7 +167,7 @@ struct dax_device { #if IS_ENABLED(CONFIG_FS_DAX) static void generic_dax_pagefree(struct page *page, void *data) { - /* TODO: wakeup page-idle waiters */ + wake_up_atomic_one(&page->_refcount); } struct dax_device *fs_dax_claim_bdev(struct block_device *bdev, void *owner) diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 61b39eaf7cad..564c9a0141cd 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -33,10 +33,15 @@ int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry * int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode); void wake_up_bit(void *word, int bit); void wake_up_atomic_t(atomic_t *p); +static inline void wake_up_atomic_one(atomic_t *p) +{ + wake_up_atomic_t(p); +} int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout); int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); +int out_of_line_wait_on_atomic_one(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); struct wait_queue_head *bit_waitqueue(void *word, int bit); extern void __init wait_bit_init(void); @@ -262,4 +267,12 @@ int wait_on_atomic_t(atomic_t *val, wait_atomic_t_action_f action, unsigned mode return out_of_line_wait_on_atomic_t(val, action, mode); } +static inline +int wait_on_atomic_one(atomic_t *val, wait_atomic_t_action_f action, unsigned mode) +{ + might_sleep(); + if (atomic_read(val) == 1) + return 0; + return out_of_line_wait_on_atomic_one(val, action, mode); +} #endif /* _LINUX_WAIT_BIT_H */ diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 84cb3acd9260..8739b1e50df5 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -162,28 +162,47 @@ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) return bit_waitqueue(p, 0); } -static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, - void *arg) +static struct wait_bit_queue_entry *to_wait_bit_q( + struct wait_queue_entry *wq_entry) +{ + return container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); +} + +static int __wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg, int target) { struct wait_bit_key *key = arg; - struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); + struct wait_bit_queue_entry *wait_bit = to_wait_bit_q(wq_entry); atomic_t *val = key->flags; if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || - atomic_read(val) != 0) + atomic_read(val) != target) return 0; return autoremove_wake_function(wq_entry, mode, sync, key); } +static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 0); +} + +static int wake_atomic_one_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 1); +} + /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero * return codes halt waiting and return. */ static __sched -int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, - wait_atomic_t_action_f action, unsigned int mode) +int __wait_on_atomic_t(struct wait_queue_head *wq_head, + struct wait_bit_queue_entry *wbq_entry, + wait_atomic_t_action_f action, unsigned int mode, int target) { atomic_t *val; int ret = 0; @@ -191,10 +210,10 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en do { prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); val = wbq_entry->key.flags; - if (atomic_read(val) == 0) + if (atomic_read(val) == target) break; ret = (*action)(val, mode); - } while (!ret && atomic_read(val) != 0); + } while (!ret && atomic_read(val) != target); finish_wait(wq_head, &wbq_entry->wq_entry); return ret; } @@ -210,6 +229,17 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en }, \ } +#define DEFINE_WAIT_ATOMIC_ONE(name, p) \ + struct wait_bit_queue_entry name = { \ + .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ + .wq_entry = { \ + .private = current, \ + .func = wake_atomic_one_function, \ + .entry = \ + LIST_HEAD_INIT((name).wq_entry.entry), \ + }, \ + } + __sched int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode) @@ -217,7 +247,7 @@ __sched int out_of_line_wait_on_atomic_t(atomic_t *p, struct wait_queue_head *wq_head = atomic_t_waitqueue(p); DEFINE_WAIT_ATOMIC_T(wq_entry, p); - return __wait_on_atomic_t(wq_head, &wq_entry, action, mode); + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 0); } EXPORT_SYMBOL(out_of_line_wait_on_atomic_t); @@ -230,6 +260,17 @@ __sched int atomic_t_wait(atomic_t *counter, unsigned int mode) } EXPORT_SYMBOL(atomic_t_wait); +__sched int out_of_line_wait_on_atomic_one(atomic_t *p, + wait_atomic_t_action_f action, + unsigned int mode) +{ + struct wait_queue_head *wq_head = atomic_t_waitqueue(p); + DEFINE_WAIT_ATOMIC_ONE(wq_entry, p); + + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 1); +} +EXPORT_SYMBOL(out_of_line_wait_on_atomic_one); + /** * wake_up_atomic_t - Wake up a waiter on a atomic_t * @p: The atomic_t being waited on, a kernel virtual address _______________________________________________ Linux-nvdimm mailing list Linux-nvdimm@lists.01.org https://lists.01.org/mailman/listinfo/linux-nvdimm From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752004AbeCJHEn (ORCPT ); Sat, 10 Mar 2018 02:04:43 -0500 Received: from mga07.intel.com ([134.134.136.100]:34007 "EHLO mga07.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751348AbeCJHEi (ORCPT ); Sat, 10 Mar 2018 02:04:38 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,449,1515484800"; d="scan'208";a="24453590" Subject: [PATCH v5 08/11] wait_bit: introduce {wait_on,wake_up}_atomic_one From: Dan Williams To: linux-nvdimm@lists.01.org Cc: Ingo Molnar , Peter Zijlstra , Christoph Hellwig , david@fromorbit.com, linux-xfs@vger.kernel.org, linux-fsdevel@vger.kernel.org, jack@suse.cz, ross.zwisler@linux.intel.com, hch@lst.de, linux-kernel@vger.kernel.org Date: Fri, 09 Mar 2018 22:55:32 -0800 Message-ID: <152066493247.40260.10849841915366086021.stgit@dwillia2-desk3.amr.corp.intel.com> In-Reply-To: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> References: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> User-Agent: StGit/0.18-2-gc94f MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Add a generic facility for awaiting an atomic_t to reach a value of 1. Page reference counts typically need to reach 0 to be considered a free / inactive page. However, ZONE_DEVICE pages allocated via devm_memremap_pages() are never 'onlined', i.e. the put_page() typically done at init time to assign pages to the page allocator is skipped. These pages will have their reference count elevated > 1 by get_user_pages() when they are under DMA. In order to coordinate DMA to these pages vs filesytem operations like hole-punch and truncate the filesystem-dax implementation needs to capture the DMA-idle event i.e. the 2 to 1 count transition). For now, this implementation does not have functional behavior change, follow-on patches will add waiters for these page-idle events. Cc: Ingo Molnar Cc: Peter Zijlstra Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- drivers/dax/super.c | 2 +- include/linux/wait_bit.h | 13 ++++++++++ kernel/sched/wait_bit.c | 59 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 619b1ed6434c..7e10fa3460e2 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -167,7 +167,7 @@ struct dax_device { #if IS_ENABLED(CONFIG_FS_DAX) static void generic_dax_pagefree(struct page *page, void *data) { - /* TODO: wakeup page-idle waiters */ + wake_up_atomic_one(&page->_refcount); } struct dax_device *fs_dax_claim_bdev(struct block_device *bdev, void *owner) diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 61b39eaf7cad..564c9a0141cd 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -33,10 +33,15 @@ int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry * int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode); void wake_up_bit(void *word, int bit); void wake_up_atomic_t(atomic_t *p); +static inline void wake_up_atomic_one(atomic_t *p) +{ + wake_up_atomic_t(p); +} int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout); int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); +int out_of_line_wait_on_atomic_one(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); struct wait_queue_head *bit_waitqueue(void *word, int bit); extern void __init wait_bit_init(void); @@ -262,4 +267,12 @@ int wait_on_atomic_t(atomic_t *val, wait_atomic_t_action_f action, unsigned mode return out_of_line_wait_on_atomic_t(val, action, mode); } +static inline +int wait_on_atomic_one(atomic_t *val, wait_atomic_t_action_f action, unsigned mode) +{ + might_sleep(); + if (atomic_read(val) == 1) + return 0; + return out_of_line_wait_on_atomic_one(val, action, mode); +} #endif /* _LINUX_WAIT_BIT_H */ diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 84cb3acd9260..8739b1e50df5 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -162,28 +162,47 @@ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) return bit_waitqueue(p, 0); } -static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, - void *arg) +static struct wait_bit_queue_entry *to_wait_bit_q( + struct wait_queue_entry *wq_entry) +{ + return container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); +} + +static int __wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg, int target) { struct wait_bit_key *key = arg; - struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); + struct wait_bit_queue_entry *wait_bit = to_wait_bit_q(wq_entry); atomic_t *val = key->flags; if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || - atomic_read(val) != 0) + atomic_read(val) != target) return 0; return autoremove_wake_function(wq_entry, mode, sync, key); } +static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 0); +} + +static int wake_atomic_one_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 1); +} + /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero * return codes halt waiting and return. */ static __sched -int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, - wait_atomic_t_action_f action, unsigned int mode) +int __wait_on_atomic_t(struct wait_queue_head *wq_head, + struct wait_bit_queue_entry *wbq_entry, + wait_atomic_t_action_f action, unsigned int mode, int target) { atomic_t *val; int ret = 0; @@ -191,10 +210,10 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en do { prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); val = wbq_entry->key.flags; - if (atomic_read(val) == 0) + if (atomic_read(val) == target) break; ret = (*action)(val, mode); - } while (!ret && atomic_read(val) != 0); + } while (!ret && atomic_read(val) != target); finish_wait(wq_head, &wbq_entry->wq_entry); return ret; } @@ -210,6 +229,17 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en }, \ } +#define DEFINE_WAIT_ATOMIC_ONE(name, p) \ + struct wait_bit_queue_entry name = { \ + .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ + .wq_entry = { \ + .private = current, \ + .func = wake_atomic_one_function, \ + .entry = \ + LIST_HEAD_INIT((name).wq_entry.entry), \ + }, \ + } + __sched int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode) @@ -217,7 +247,7 @@ __sched int out_of_line_wait_on_atomic_t(atomic_t *p, struct wait_queue_head *wq_head = atomic_t_waitqueue(p); DEFINE_WAIT_ATOMIC_T(wq_entry, p); - return __wait_on_atomic_t(wq_head, &wq_entry, action, mode); + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 0); } EXPORT_SYMBOL(out_of_line_wait_on_atomic_t); @@ -230,6 +260,17 @@ __sched int atomic_t_wait(atomic_t *counter, unsigned int mode) } EXPORT_SYMBOL(atomic_t_wait); +__sched int out_of_line_wait_on_atomic_one(atomic_t *p, + wait_atomic_t_action_f action, + unsigned int mode) +{ + struct wait_queue_head *wq_head = atomic_t_waitqueue(p); + DEFINE_WAIT_ATOMIC_ONE(wq_entry, p); + + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 1); +} +EXPORT_SYMBOL(out_of_line_wait_on_atomic_one); + /** * wake_up_atomic_t - Wake up a waiter on a atomic_t * @p: The atomic_t being waited on, a kernel virtual address From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga07.intel.com ([134.134.136.100]:34007 "EHLO mga07.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1751348AbeCJHEi (ORCPT ); Sat, 10 Mar 2018 02:04:38 -0500 Subject: [PATCH v5 08/11] wait_bit: introduce {wait_on,wake_up}_atomic_one From: Dan Williams Date: Fri, 09 Mar 2018 22:55:32 -0800 Message-ID: <152066493247.40260.10849841915366086021.stgit@dwillia2-desk3.amr.corp.intel.com> In-Reply-To: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> References: <152066488891.40260.14605734226832760468.stgit@dwillia2-desk3.amr.corp.intel.com> MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Sender: linux-xfs-owner@vger.kernel.org List-ID: List-Id: xfs To: linux-nvdimm@lists.01.org Cc: Ingo Molnar , Peter Zijlstra , Christoph Hellwig , david@fromorbit.com, linux-xfs@vger.kernel.org, linux-fsdevel@vger.kernel.org, jack@suse.cz, ross.zwisler@linux.intel.comhch@lst.de, linux-kernel@vger.kernel.org Add a generic facility for awaiting an atomic_t to reach a value of 1. Page reference counts typically need to reach 0 to be considered a free / inactive page. However, ZONE_DEVICE pages allocated via devm_memremap_pages() are never 'onlined', i.e. the put_page() typically done at init time to assign pages to the page allocator is skipped. These pages will have their reference count elevated > 1 by get_user_pages() when they are under DMA. In order to coordinate DMA to these pages vs filesytem operations like hole-punch and truncate the filesystem-dax implementation needs to capture the DMA-idle event i.e. the 2 to 1 count transition). For now, this implementation does not have functional behavior change, follow-on patches will add waiters for these page-idle events. Cc: Ingo Molnar Cc: Peter Zijlstra Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- drivers/dax/super.c | 2 +- include/linux/wait_bit.h | 13 ++++++++++ kernel/sched/wait_bit.c | 59 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 64 insertions(+), 10 deletions(-) diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 619b1ed6434c..7e10fa3460e2 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -167,7 +167,7 @@ struct dax_device { #if IS_ENABLED(CONFIG_FS_DAX) static void generic_dax_pagefree(struct page *page, void *data) { - /* TODO: wakeup page-idle waiters */ + wake_up_atomic_one(&page->_refcount); } struct dax_device *fs_dax_claim_bdev(struct block_device *bdev, void *owner) diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 61b39eaf7cad..564c9a0141cd 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -33,10 +33,15 @@ int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry * int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode); void wake_up_bit(void *word, int bit); void wake_up_atomic_t(atomic_t *p); +static inline void wake_up_atomic_one(atomic_t *p) +{ + wake_up_atomic_t(p); +} int out_of_line_wait_on_bit(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_bit_timeout(void *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout); int out_of_line_wait_on_bit_lock(void *word, int, wait_bit_action_f *action, unsigned int mode); int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); +int out_of_line_wait_on_atomic_one(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode); struct wait_queue_head *bit_waitqueue(void *word, int bit); extern void __init wait_bit_init(void); @@ -262,4 +267,12 @@ int wait_on_atomic_t(atomic_t *val, wait_atomic_t_action_f action, unsigned mode return out_of_line_wait_on_atomic_t(val, action, mode); } +static inline +int wait_on_atomic_one(atomic_t *val, wait_atomic_t_action_f action, unsigned mode) +{ + might_sleep(); + if (atomic_read(val) == 1) + return 0; + return out_of_line_wait_on_atomic_one(val, action, mode); +} #endif /* _LINUX_WAIT_BIT_H */ diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 84cb3acd9260..8739b1e50df5 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -162,28 +162,47 @@ static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p) return bit_waitqueue(p, 0); } -static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, - void *arg) +static struct wait_bit_queue_entry *to_wait_bit_q( + struct wait_queue_entry *wq_entry) +{ + return container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); +} + +static int __wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg, int target) { struct wait_bit_key *key = arg; - struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); + struct wait_bit_queue_entry *wait_bit = to_wait_bit_q(wq_entry); atomic_t *val = key->flags; if (wait_bit->key.flags != key->flags || wait_bit->key.bit_nr != key->bit_nr || - atomic_read(val) != 0) + atomic_read(val) != target) return 0; return autoremove_wake_function(wq_entry, mode, sync, key); } +static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 0); +} + +static int wake_atomic_one_function(struct wait_queue_entry *wq_entry, + unsigned mode, int sync, void *arg) +{ + return __wake_atomic_t_function(wq_entry, mode, sync, arg, 1); +} + /* * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting, * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero * return codes halt waiting and return. */ static __sched -int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, - wait_atomic_t_action_f action, unsigned int mode) +int __wait_on_atomic_t(struct wait_queue_head *wq_head, + struct wait_bit_queue_entry *wbq_entry, + wait_atomic_t_action_f action, unsigned int mode, int target) { atomic_t *val; int ret = 0; @@ -191,10 +210,10 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en do { prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); val = wbq_entry->key.flags; - if (atomic_read(val) == 0) + if (atomic_read(val) == target) break; ret = (*action)(val, mode); - } while (!ret && atomic_read(val) != 0); + } while (!ret && atomic_read(val) != target); finish_wait(wq_head, &wbq_entry->wq_entry); return ret; } @@ -210,6 +229,17 @@ int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_en }, \ } +#define DEFINE_WAIT_ATOMIC_ONE(name, p) \ + struct wait_bit_queue_entry name = { \ + .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \ + .wq_entry = { \ + .private = current, \ + .func = wake_atomic_one_function, \ + .entry = \ + LIST_HEAD_INIT((name).wq_entry.entry), \ + }, \ + } + __sched int out_of_line_wait_on_atomic_t(atomic_t *p, wait_atomic_t_action_f action, unsigned int mode) @@ -217,7 +247,7 @@ __sched int out_of_line_wait_on_atomic_t(atomic_t *p, struct wait_queue_head *wq_head = atomic_t_waitqueue(p); DEFINE_WAIT_ATOMIC_T(wq_entry, p); - return __wait_on_atomic_t(wq_head, &wq_entry, action, mode); + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 0); } EXPORT_SYMBOL(out_of_line_wait_on_atomic_t); @@ -230,6 +260,17 @@ __sched int atomic_t_wait(atomic_t *counter, unsigned int mode) } EXPORT_SYMBOL(atomic_t_wait); +__sched int out_of_line_wait_on_atomic_one(atomic_t *p, + wait_atomic_t_action_f action, + unsigned int mode) +{ + struct wait_queue_head *wq_head = atomic_t_waitqueue(p); + DEFINE_WAIT_ATOMIC_ONE(wq_entry, p); + + return __wait_on_atomic_t(wq_head, &wq_entry, action, mode, 1); +} +EXPORT_SYMBOL(out_of_line_wait_on_atomic_one); + /** * wake_up_atomic_t - Wake up a waiter on a atomic_t * @p: The atomic_t being waited on, a kernel virtual address