From: Ming Lei <ming.lei@redhat.com> To: Jens Axboe <axboe@kernel.dk> Cc: linux-block@vger.kernel.org, Jeffle Xu <jefflexu@linux.alibaba.com>, Mike Snitzer <snitzer@redhat.com>, dm-devel@redhat.com, Hannes Reinecke <hare@suse.de>, Ming Lei <ming.lei@redhat.com> Subject: [PATCH V5 09/12] blk-mq: limit hw queues to be polled in each blk_poll() Date: Thu, 1 Apr 2021 10:19:24 +0800 [thread overview] Message-ID: <20210401021927.343727-10-ming.lei@redhat.com> (raw) In-Reply-To: <20210401021927.343727-1-ming.lei@redhat.com> Limit at most 8 queues are polled in each blk_pull(), avoid to add extra latency when queue depth is high. Reviewed-by: Jeffle Xu <jefflexu@linux.alibaba.com> Signed-off-by: Ming Lei <ming.lei@redhat.com> --- block/blk-mq.c | 79 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 20bfc8c2d02e..e2701d502e51 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3865,36 +3865,31 @@ static inline int blk_mq_poll_hctx(struct request_queue *q, return ret; } -static int blk_mq_poll_io(struct bio *bio) -{ - struct request_queue *q = bio->bi_bdev->bd_disk->queue; - blk_qc_t cookie = bio_get_private_data(bio); - int ret = 0; +#define POLL_HCTX_MAX_CNT 8 - /* wait until the bio is submitted really */ - if (!blk_qc_t_ready(cookie)) - return 0; +static bool blk_add_unique_hctx(struct blk_mq_hw_ctx **data, int *cnt, + struct blk_mq_hw_ctx *hctx) +{ + int i; - if (!bio_flagged(bio, BIO_DONE) && blk_qc_t_valid(cookie)) { - struct blk_mq_hw_ctx *hctx = - q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; + for (i = 0; i < *cnt; i++) { + if (data[i] == hctx) + goto exit; + } - ret += blk_mq_poll_hctx(q, hctx); + if (i < POLL_HCTX_MAX_CNT) { + data[i] = hctx; + (*cnt)++; } - return ret; + exit: + return *cnt == POLL_HCTX_MAX_CNT; } -static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) +static void blk_build_poll_queues(struct bio_grp_list *grps, + struct blk_mq_hw_ctx **data, int *cnt) { - int ret = 0; int i; - /* - * Poll hw queue first. - * - * TODO: limit max poll times and make sure to not poll same - * hw queue one more time. - */ for (i = 0; i < grps->nr_grps; i++) { struct bio_grp_list_data *grp = &grps->head[i]; struct bio *bio; @@ -3902,11 +3897,31 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) if (bio_grp_list_grp_empty(grp)) continue; - for (bio = grp->list.head; bio; bio = bio->bi_poll) - ret += blk_mq_poll_io(bio); + for (bio = grp->list.head; bio; bio = bio->bi_poll) { + blk_qc_t cookie; + struct blk_mq_hw_ctx *hctx; + struct request_queue *q; + + if (bio_flagged(bio, BIO_DONE)) + continue; + + /* wait until the bio is submitted really */ + cookie = bio_get_private_data(bio); + if (!blk_qc_t_ready(cookie) || !blk_qc_t_valid(cookie)) + continue; + + q = bio->bi_bdev->bd_disk->queue; + hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; + if (blk_add_unique_hctx(data, cnt, hctx)) + return; + } } +} + +static void blk_bio_poll_reap_ios(struct bio_grp_list *grps) +{ + int i; - /* reap bios */ for (i = 0; i < grps->nr_grps; i++) { struct bio_grp_list_data *grp = &grps->head[i]; struct bio *bio; @@ -3931,6 +3946,22 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) } __bio_grp_list_merge(&grp->list, &bl); } +} + +static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) +{ + int ret = 0; + int i; + struct blk_mq_hw_ctx *hctx[POLL_HCTX_MAX_CNT]; + int cnt = 0; + + blk_build_poll_queues(grps, hctx, &cnt); + + for (i = 0; i < cnt; i++) + ret += blk_mq_poll_hctx(hctx[i]->queue, hctx[i]); + + blk_bio_poll_reap_ios(grps); + return ret; } -- 2.29.2
WARNING: multiple messages have this Message-ID (diff)
From: Ming Lei <ming.lei@redhat.com> To: Jens Axboe <axboe@kernel.dk> Cc: Mike Snitzer <snitzer@redhat.com>, Ming Lei <ming.lei@redhat.com>, linux-block@vger.kernel.org, dm-devel@redhat.com, Jeffle Xu <jefflexu@linux.alibaba.com> Subject: [dm-devel] [PATCH V5 09/12] blk-mq: limit hw queues to be polled in each blk_poll() Date: Thu, 1 Apr 2021 10:19:24 +0800 [thread overview] Message-ID: <20210401021927.343727-10-ming.lei@redhat.com> (raw) In-Reply-To: <20210401021927.343727-1-ming.lei@redhat.com> Limit at most 8 queues are polled in each blk_pull(), avoid to add extra latency when queue depth is high. Reviewed-by: Jeffle Xu <jefflexu@linux.alibaba.com> Signed-off-by: Ming Lei <ming.lei@redhat.com> --- block/blk-mq.c | 79 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 20bfc8c2d02e..e2701d502e51 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3865,36 +3865,31 @@ static inline int blk_mq_poll_hctx(struct request_queue *q, return ret; } -static int blk_mq_poll_io(struct bio *bio) -{ - struct request_queue *q = bio->bi_bdev->bd_disk->queue; - blk_qc_t cookie = bio_get_private_data(bio); - int ret = 0; +#define POLL_HCTX_MAX_CNT 8 - /* wait until the bio is submitted really */ - if (!blk_qc_t_ready(cookie)) - return 0; +static bool blk_add_unique_hctx(struct blk_mq_hw_ctx **data, int *cnt, + struct blk_mq_hw_ctx *hctx) +{ + int i; - if (!bio_flagged(bio, BIO_DONE) && blk_qc_t_valid(cookie)) { - struct blk_mq_hw_ctx *hctx = - q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; + for (i = 0; i < *cnt; i++) { + if (data[i] == hctx) + goto exit; + } - ret += blk_mq_poll_hctx(q, hctx); + if (i < POLL_HCTX_MAX_CNT) { + data[i] = hctx; + (*cnt)++; } - return ret; + exit: + return *cnt == POLL_HCTX_MAX_CNT; } -static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) +static void blk_build_poll_queues(struct bio_grp_list *grps, + struct blk_mq_hw_ctx **data, int *cnt) { - int ret = 0; int i; - /* - * Poll hw queue first. - * - * TODO: limit max poll times and make sure to not poll same - * hw queue one more time. - */ for (i = 0; i < grps->nr_grps; i++) { struct bio_grp_list_data *grp = &grps->head[i]; struct bio *bio; @@ -3902,11 +3897,31 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) if (bio_grp_list_grp_empty(grp)) continue; - for (bio = grp->list.head; bio; bio = bio->bi_poll) - ret += blk_mq_poll_io(bio); + for (bio = grp->list.head; bio; bio = bio->bi_poll) { + blk_qc_t cookie; + struct blk_mq_hw_ctx *hctx; + struct request_queue *q; + + if (bio_flagged(bio, BIO_DONE)) + continue; + + /* wait until the bio is submitted really */ + cookie = bio_get_private_data(bio); + if (!blk_qc_t_ready(cookie) || !blk_qc_t_valid(cookie)) + continue; + + q = bio->bi_bdev->bd_disk->queue; + hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; + if (blk_add_unique_hctx(data, cnt, hctx)) + return; + } } +} + +static void blk_bio_poll_reap_ios(struct bio_grp_list *grps) +{ + int i; - /* reap bios */ for (i = 0; i < grps->nr_grps; i++) { struct bio_grp_list_data *grp = &grps->head[i]; struct bio *bio; @@ -3931,6 +3946,22 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) } __bio_grp_list_merge(&grp->list, &bl); } +} + +static int blk_bio_poll_and_end_io(struct bio_grp_list *grps) +{ + int ret = 0; + int i; + struct blk_mq_hw_ctx *hctx[POLL_HCTX_MAX_CNT]; + int cnt = 0; + + blk_build_poll_queues(grps, hctx, &cnt); + + for (i = 0; i < cnt; i++) + ret += blk_mq_poll_hctx(hctx[i]->queue, hctx[i]); + + blk_bio_poll_reap_ios(grps); + return ret; } -- 2.29.2 -- dm-devel mailing list dm-devel@redhat.com https://listman.redhat.com/mailman/listinfo/dm-devel
next prev parent reply other threads:[~2021-04-01 2:22 UTC|newest] Thread overview: 92+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-04-01 2:19 [PATCH V5 00/12] block: support bio based io polling Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 01/12] block: add helper of blk_queue_poll Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 02/12] block: add one helper to free io_context Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 03/12] block: create io poll context for submission and poll task Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 10:19 ` Christoph Hellwig 2021-04-12 10:19 ` [dm-devel] " Christoph Hellwig 2021-04-01 2:19 ` [PATCH V5 04/12] block: add req flag of REQ_POLL_CTX Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 05/12] block: add new field into 'struct bvec_iter' Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 9:26 ` Christoph Hellwig 2021-04-12 9:26 ` [dm-devel] " Christoph Hellwig 2021-04-13 9:36 ` Ming Lei 2021-04-13 9:36 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 06/12] block/mq: extract one helper function polling hw queue Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 9:29 ` Christoph Hellwig 2021-04-12 9:29 ` [dm-devel] " Christoph Hellwig 2021-04-01 2:19 ` [PATCH V5 07/12] block: prepare for supporting bio_list via other link Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 10:18 ` Christoph Hellwig 2021-04-12 10:18 ` [dm-devel] " Christoph Hellwig 2021-04-12 11:37 ` Ming Lei 2021-04-12 11:37 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` [PATCH V5 08/12] block: use per-task poll context to implement bio based io polling Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 9:54 ` Christoph Hellwig 2021-04-12 9:54 ` [dm-devel] " Christoph Hellwig 2021-04-12 10:20 ` Ming Lei 2021-04-12 10:20 ` [dm-devel] " Ming Lei 2021-04-12 10:29 ` Christoph Hellwig 2021-04-12 10:29 ` [dm-devel] " Christoph Hellwig 2021-04-12 11:42 ` Ming Lei 2021-04-12 11:42 ` [dm-devel] " Ming Lei 2021-04-12 10:16 ` Christoph Hellwig 2021-04-12 10:16 ` [dm-devel] " Christoph Hellwig 2021-04-12 10:37 ` Ming Lei 2021-04-12 10:37 ` [dm-devel] " Ming Lei 2021-04-01 2:19 ` Ming Lei [this message] 2021-04-01 2:19 ` [dm-devel] [PATCH V5 09/12] blk-mq: limit hw queues to be polled in each blk_poll() Ming Lei 2021-04-01 2:19 ` [PATCH V5 10/12] block: add queue_to_disk() to get gendisk from request_queue Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 12:52 ` Jens Axboe 2021-04-12 12:52 ` [dm-devel] " Jens Axboe 2021-04-01 2:19 ` [PATCH V5 11/12] block: add poll_capable method to support bio-based IO polling Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-12 9:38 ` Christoph Hellwig 2021-04-12 9:38 ` [dm-devel] " Christoph Hellwig 2021-04-14 8:38 ` JeffleXu 2021-04-14 8:38 ` [dm-devel] " JeffleXu 2021-04-14 11:24 ` Ming Lei 2021-04-14 11:24 ` [dm-devel] " Ming Lei 2021-04-15 1:34 ` JeffleXu 2021-04-15 1:34 ` [dm-devel] " JeffleXu 2021-04-15 7:43 ` Ming Lei 2021-04-15 7:43 ` [dm-devel] " Ming Lei 2021-04-15 9:21 ` JeffleXu 2021-04-15 9:21 ` [dm-devel] " JeffleXu 2021-04-15 10:06 ` Ming Lei 2021-04-15 10:06 ` [dm-devel] " Ming Lei 2021-04-15 11:21 ` JeffleXu 2021-04-15 11:21 ` [dm-devel] " JeffleXu 2021-04-15 13:08 ` Ming Lei 2021-04-15 13:08 ` [dm-devel] " Ming Lei 2021-04-16 8:00 ` [PATCH] block: introduce QUEUE_FLAG_POLL_CAP flag Jeffle Xu 2021-04-16 8:00 ` [dm-devel] " Jeffle Xu 2021-04-16 8:42 ` JeffleXu 2021-04-16 8:42 ` [dm-devel] " JeffleXu 2021-04-16 9:07 ` Ming Lei 2021-04-16 9:07 ` [dm-devel] " Ming Lei 2021-04-16 10:20 ` JeffleXu 2021-04-16 10:20 ` [dm-devel] " JeffleXu 2021-04-17 14:06 ` JeffleXu 2021-04-17 14:06 ` [dm-devel] " JeffleXu 2021-04-19 2:21 ` Ming Lei 2021-04-19 2:21 ` [dm-devel] " Ming Lei 2021-04-19 5:40 ` JeffleXu 2021-04-19 5:40 ` [dm-devel] " JeffleXu 2021-04-19 13:36 ` Ming Lei 2021-04-19 13:36 ` [dm-devel] " Ming Lei 2021-04-20 7:25 ` JeffleXu 2021-04-20 7:25 ` [dm-devel] " JeffleXu 2021-04-01 2:19 ` [PATCH V5 12/12] dm: support IO polling for bio-based dm device Ming Lei 2021-04-01 2:19 ` [dm-devel] " Ming Lei 2021-04-09 15:39 ` [PATCH V5 00/12] block: support bio based io polling Ming Lei 2021-04-09 15:39 ` [dm-devel] " Ming Lei 2021-04-12 9:46 ` Christoph Hellwig 2021-04-12 9:46 ` [dm-devel] " Christoph Hellwig
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210401021927.343727-10-ming.lei@redhat.com \ --to=ming.lei@redhat.com \ --cc=axboe@kernel.dk \ --cc=dm-devel@redhat.com \ --cc=hare@suse.de \ --cc=jefflexu@linux.alibaba.com \ --cc=linux-block@vger.kernel.org \ --cc=snitzer@redhat.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.