From: Ming Lei <ming.lei@redhat.com>
To: Sagi Grimberg <sagi@grimberg.me>
Cc: Jens Axboe <axboe@kernel.dk>,
"Paul E. McKenney" <paulmck@kernel.org>,
linux-nvme@lists.infradead.org, linux-block@vger.kernel.org,
Chao Leng <lengchao@huawei.com>, Keith Busch <kbusch@kernel.org>,
Ming Lin <mlin@kernel.org>, Christoph Hellwig <hch@lst.de>
Subject: Re: [PATCH v5 1/2] blk-mq: add tagset quiesce interface
Date: Tue, 28 Jul 2020 18:10:42 +0800 [thread overview]
Message-ID: <20200728101042.GA1336890@T590> (raw)
In-Reply-To: <6a678d5d-22b2-5238-92c5-d68e2aafeb9e@grimberg.me>
On Tue, Jul 28, 2020 at 02:43:06AM -0700, Sagi Grimberg wrote:
>
> > > > > > I like the tagset based interface. But the idea of doing a per-hctx
> > > > > > allocation and wait doesn't seem very scalable.
> > > > > >
> > > > > > Paul, do you have any good idea for an interface that waits on
> > > > > > multiple srcu heads? As far as I can tell we could just have a single
> > > > > > global completion and counter, and each call_srcu would just just
> > > > > > decrement it and then the final one would do the
> > > > > > wakeup. It would just
> > > > > > be great to figure out a way to keep the struct rcu_synchronize and
> > > > > > counter on stack to avoid an allocation.
> > > > > >
> > > > > > But if we can't do with an on-stack object I'd much rather just embedd
> > > > > > the rcu_head in the hw_ctx.
> > > > >
> > > > > I think we can do that, please see the following patch which
> > > > > is against Sagi's V5:
> > > >
> > > > I don't think you can send a single rcu_head to multiple
> > > > call_srcu calls.
> > >
> > > OK, then one variant is to put the rcu_head into blk_mq_hw_ctx, and put
> > > rcu_synchronize into blk_mq_tag_set.
> >
> > I can cook up a spin,
>
> Nope.. spoke too soon, the rcu_head needs to be in a context that has
> access to the counter (which is what you called blk_mq_srcu_sync).
> you want to add also a pointer to hctx? that is almost as big as
> rcu_synchronize...
We can just put rcu_head into hctx, and put the count & completion into
tag_set, and the tagset can be retrieved via hctx, something like the
following patch:
diff --git a/block/blk-mq.c b/block/blk-mq.c
index c3856377b961..129665da4dbd 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -27,6 +27,7 @@
#include <linux/crash_dump.h>
#include <linux/prefetch.h>
#include <linux/blk-crypto.h>
+#include <linux/rcupdate_wait.h>
#include <trace/events/block.h>
@@ -209,6 +210,34 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q)
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
+static void blk_mq_wakeme_after_rcu(struct rcu_head *head)
+{
+
+ struct blk_mq_srcu_struct *srcu = container_of(head,
+ struct blk_mq_srcu_struct, head);
+ struct blk_mq_hw_ctx *hctx = (void *)srcu -
+ sizeof(struct blk_mq_hw_ctx);
+ struct blk_mq_tag_set *set = hctx->queue->tag_set;
+
+ if (atomic_dec_and_test(&set->quiesce_count))
+ complete(&set->quiesce_completion);
+}
+
+static void blk_mq_quiesce_blocking_queue_async(struct request_queue *q)
+{
+ struct blk_mq_hw_ctx *hctx;
+ unsigned int i;
+
+ blk_mq_quiesce_queue_nowait(q);
+
+ queue_for_each_hw_ctx(q, hctx, i) {
+ WARN_ON_ONCE(!(hctx->flags & BLK_MQ_F_BLOCKING));
+ init_rcu_head(&hctx->srcu->head);
+ call_srcu(&hctx->srcu->srcu, &hctx->srcu->head,
+ blk_mq_wakeme_after_rcu);
+ }
+}
+
/**
* blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
* @q: request queue.
@@ -228,7 +257,7 @@ void blk_mq_quiesce_queue(struct request_queue *q)
queue_for_each_hw_ctx(q, hctx, i) {
if (hctx->flags & BLK_MQ_F_BLOCKING)
- synchronize_srcu(hctx->srcu);
+ synchronize_srcu(&hctx->srcu->srcu);
else
rcu = true;
}
@@ -700,23 +729,23 @@ void blk_mq_complete_request(struct request *rq)
EXPORT_SYMBOL(blk_mq_complete_request);
static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
- __releases(hctx->srcu)
+ __releases(&hctx->srcu->srcu)
{
if (!(hctx->flags & BLK_MQ_F_BLOCKING))
rcu_read_unlock();
else
- srcu_read_unlock(hctx->srcu, srcu_idx);
+ srcu_read_unlock(&hctx->srcu->srcu, srcu_idx);
}
static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
- __acquires(hctx->srcu)
+ __acquires(&hctx->srcu->srcu)
{
if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
/* shut up gcc false positive */
*srcu_idx = 0;
rcu_read_lock();
} else
- *srcu_idx = srcu_read_lock(hctx->srcu);
+ *srcu_idx = srcu_read_lock(&hctx->srcu->srcu);
}
/**
@@ -2599,7 +2628,7 @@ static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
sizeof(struct blk_mq_hw_ctx));
if (tag_set->flags & BLK_MQ_F_BLOCKING)
- hw_ctx_size += sizeof(struct srcu_struct);
+ hw_ctx_size += sizeof(struct blk_mq_srcu_struct);
return hw_ctx_size;
}
@@ -2684,7 +2713,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
goto free_bitmap;
if (hctx->flags & BLK_MQ_F_BLOCKING)
- init_srcu_struct(hctx->srcu);
+ init_srcu_struct(&hctx->srcu->srcu);
blk_mq_hctx_kobj_init(hctx);
return hctx;
@@ -2880,6 +2909,43 @@ static void queue_set_hctx_shared(struct request_queue *q, bool shared)
}
}
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
+{
+ struct request_queue *q;
+
+ mutex_lock(&set->tag_list_lock);
+ if (set->flags & BLK_MQ_F_BLOCKING) {
+ int count = 0;
+
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ count++;
+
+ atomic_set(&set->quiesce_count, count);
+ init_completion(&set->quiesce_completion);
+
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ blk_mq_quiesce_blocking_queue_async(q);
+ wait_for_completion(&set->quiesce_completion);
+ } else {
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ blk_mq_quiesce_queue_nowait(q);
+ synchronize_rcu();
+ }
+ mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
+
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
+{
+ struct request_queue *q;
+
+ mutex_lock(&set->tag_list_lock);
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ blk_mq_unquiesce_queue(q);
+ mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
+
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set,
bool shared)
{
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 23230c1d031e..9ef7fdb809a7 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -9,6 +9,11 @@
struct blk_mq_tags;
struct blk_flush_queue;
+struct blk_mq_srcu_struct {
+ struct srcu_struct srcu;
+ struct rcu_head head;
+};
+
/**
* struct blk_mq_hw_ctx - State for a hardware queue facing the hardware
* block device
@@ -175,7 +180,7 @@ struct blk_mq_hw_ctx {
* blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
* blk_mq_hw_ctx_size().
*/
- struct srcu_struct srcu[];
+ struct blk_mq_srcu_struct srcu[];
};
/**
@@ -254,6 +259,9 @@ struct blk_mq_tag_set {
struct mutex tag_list_lock;
struct list_head tag_list;
+
+ struct completion quiesce_completion;
+ atomic_t quiesce_count;
};
/**
@@ -532,6 +540,8 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap);
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
void blk_mq_quiesce_queue_nowait(struct request_queue *q);
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set);
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set);
unsigned int blk_mq_rq_cpu(struct request *rq);
--
Ming
next prev parent reply other threads:[~2020-07-28 10:11 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-07-27 23:10 [PATCH v5 0/2] improve nvme quiesce time for large amount of namespaces Sagi Grimberg
2020-07-27 23:10 ` [PATCH v5 1/2] blk-mq: add tagset quiesce interface Sagi Grimberg
2020-07-27 23:32 ` Keith Busch
2020-07-28 0:12 ` Sagi Grimberg
2020-07-28 1:40 ` Ming Lei
2020-07-28 1:51 ` Jens Axboe
2020-07-28 2:17 ` Ming Lei
2020-07-28 2:23 ` Jens Axboe
2020-07-28 2:28 ` Ming Lei
2020-07-28 2:32 ` Jens Axboe
2020-07-28 3:29 ` Sagi Grimberg
2020-07-28 3:25 ` Sagi Grimberg
2020-07-28 7:18 ` Christoph Hellwig
2020-07-28 7:48 ` Sagi Grimberg
2020-07-28 9:16 ` Ming Lei
2020-07-28 9:24 ` Sagi Grimberg
2020-07-28 9:33 ` Ming Lei
2020-07-28 9:37 ` Sagi Grimberg
2020-07-28 9:43 ` Sagi Grimberg
2020-07-28 10:10 ` Ming Lei [this message]
2020-07-28 10:57 ` Christoph Hellwig
2020-07-28 14:13 ` Paul E. McKenney
2020-07-28 10:58 ` Christoph Hellwig
2020-07-28 16:25 ` Sagi Grimberg
2020-07-28 13:54 ` Paul E. McKenney
2020-07-28 23:46 ` Sagi Grimberg
2020-07-29 0:31 ` Paul E. McKenney
2020-07-29 0:43 ` Sagi Grimberg
2020-07-29 0:59 ` Keith Busch
2020-07-29 4:39 ` Sagi Grimberg
2020-08-07 9:04 ` Chao Leng
2020-08-07 9:24 ` Ming Lei
2020-08-07 9:35 ` Chao Leng
2020-07-29 4:10 ` Paul E. McKenney
2020-07-29 4:37 ` Sagi Grimberg
2020-07-27 23:10 ` [PATCH v5 2/2] nvme: use blk_mq_[un]quiesce_tagset Sagi Grimberg
2020-07-28 0:54 ` Sagi Grimberg
2020-07-28 3:21 ` Chao Leng
2020-07-28 3:34 ` Sagi Grimberg
2020-07-28 3:51 ` Chao Leng
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200728101042.GA1336890@T590 \
--to=ming.lei@redhat.com \
--cc=axboe@kernel.dk \
--cc=hch@lst.de \
--cc=kbusch@kernel.org \
--cc=lengchao@huawei.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=mlin@kernel.org \
--cc=paulmck@kernel.org \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).