From: Pavel Begunkov <asml.silence@gmail.com>
To: linux-block@vger.kernel.org
Cc: Jens Axboe <axboe@kernel.dk>, Pavel Begunkov <asml.silence@gmail.com>
Subject: [PATCH 07/16] blocK: move plug flush functions to blk-mq.c
Date: Tue, 19 Oct 2021 22:24:16 +0100 [thread overview]
Message-ID: <bb6fe6de2fe6bd69ccf9bc8af049ffedcf52bda0.1634676157.git.asml.silence@gmail.com> (raw)
In-Reply-To: <cover.1634676157.git.asml.silence@gmail.com>
Flushing is tightly coupled with blk-mq and almost all
blk_flush_plug_list() callees are in blk-mq.c. So move the whole thing
there, so the compiler is able to apply more optimisations and inline.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
block/blk-core.c | 27 ---------------------------
block/blk-mq.c | 33 +++++++++++++++++++++++++++++----
block/blk-mq.h | 1 -
include/linux/blk-mq.h | 2 --
4 files changed, 29 insertions(+), 34 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index 88752e51d2b6..52019b8a1487 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1595,23 +1595,6 @@ void blk_start_plug(struct blk_plug *plug)
}
EXPORT_SYMBOL(blk_start_plug);
-static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
-{
- LIST_HEAD(callbacks);
-
- while (!list_empty(&plug->cb_list)) {
- list_splice_init(&plug->cb_list, &callbacks);
-
- while (!list_empty(&callbacks)) {
- struct blk_plug_cb *cb = list_first_entry(&callbacks,
- struct blk_plug_cb,
- list);
- list_del(&cb->list);
- cb->callback(cb, from_schedule);
- }
- }
-}
-
struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
int size)
{
@@ -1637,16 +1620,6 @@ struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
}
EXPORT_SYMBOL(blk_check_plugged);
-void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
-{
- flush_plug_callbacks(plug, from_schedule);
-
- if (!rq_list_empty(plug->mq_list))
- blk_mq_flush_plug_list(plug, from_schedule);
- if (unlikely(!from_schedule && plug->cached_rq))
- blk_mq_free_plug_rqs(plug);
-}
-
/**
* blk_finish_plug - mark the end of a batch of submitted I/O
* @plug: The &struct blk_plug passed to blk_start_plug()
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 218bfaa98591..6bdbaa838030 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -604,7 +604,7 @@ void blk_mq_free_request(struct request *rq)
}
EXPORT_SYMBOL_GPL(blk_mq_free_request);
-void blk_mq_free_plug_rqs(struct blk_plug *plug)
+static void blk_mq_free_plug_rqs(struct blk_plug *plug)
{
struct request *rq;
@@ -2199,15 +2199,13 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule)
blk_mq_commit_rqs(hctx, &queued, from_schedule);
}
-void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
+static void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
struct blk_mq_hw_ctx *this_hctx;
struct blk_mq_ctx *this_ctx;
unsigned int depth;
LIST_HEAD(list);
- if (rq_list_empty(plug->mq_list))
- return;
plug->rq_count = 0;
if (!plug->multiple_queues && !plug->has_elevator) {
@@ -2249,6 +2247,33 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
}
}
+static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
+{
+ LIST_HEAD(callbacks);
+
+ while (!list_empty(&plug->cb_list)) {
+ list_splice_init(&plug->cb_list, &callbacks);
+
+ while (!list_empty(&callbacks)) {
+ struct blk_plug_cb *cb = list_first_entry(&callbacks,
+ struct blk_plug_cb,
+ list);
+ list_del(&cb->list);
+ cb->callback(cb, from_schedule);
+ }
+ }
+}
+
+void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
+{
+ flush_plug_callbacks(plug, from_schedule);
+
+ if (!rq_list_empty(plug->mq_list))
+ blk_mq_flush_plug_list(plug, from_schedule);
+ if (unlikely(!from_schedule && plug->cached_rq))
+ blk_mq_free_plug_rqs(plug);
+}
+
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
diff --git a/block/blk-mq.h b/block/blk-mq.h
index ebf67f4d4f2e..bab40688e59b 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -121,7 +121,6 @@ extern int __blk_mq_register_dev(struct device *dev, struct request_queue *q);
extern int blk_mq_sysfs_register(struct request_queue *q);
extern void blk_mq_sysfs_unregister(struct request_queue *q);
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
-void blk_mq_free_plug_rqs(struct blk_plug *plug);
void blk_mq_release(struct request_queue *q);
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 6cf35de151a9..e13780236550 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -656,8 +656,6 @@ int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
unsigned int set_flags);
void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
-void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
-
void blk_mq_free_request(struct request *rq);
bool blk_mq_queue_inflight(struct request_queue *q);
--
2.33.1
next prev parent reply other threads:[~2021-10-19 21:24 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-19 21:24 [PATCH 00/16] block optimisation round Pavel Begunkov
2021-10-19 21:24 ` [PATCH 01/16] block: turn macro helpers into inline functions Pavel Begunkov
2021-10-20 6:09 ` Christoph Hellwig
2021-10-19 21:24 ` [PATCH 02/16] block: convert leftovers to bdev_get_queue Pavel Begunkov
2021-10-19 22:34 ` Chaitanya Kulkarni
2021-10-19 21:24 ` [PATCH 03/16] block: optimise req_bio_endio() Pavel Begunkov
2021-10-20 6:11 ` Christoph Hellwig
2021-10-22 9:58 ` Shinichiro Kawasaki
2021-10-22 10:58 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 04/16] block: don't bloat enter_queue with percpu_ref Pavel Begunkov
2021-10-19 22:32 ` Chaitanya Kulkarni
2021-10-20 6:12 ` Christoph Hellwig
2021-10-20 12:08 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 05/16] block: inline a part of bio_release_pages() Pavel Begunkov
2021-10-20 6:13 ` Christoph Hellwig
2021-10-20 12:19 ` Pavel Begunkov
2021-10-20 14:15 ` Jens Axboe
2021-10-20 17:29 ` Christoph Hellwig
2021-10-19 21:24 ` [PATCH 06/16] block: clean up blk_mq_submit_bio() merging Pavel Begunkov
2021-10-20 6:16 ` Christoph Hellwig
2021-10-20 12:20 ` Pavel Begunkov
2021-10-19 21:24 ` Pavel Begunkov [this message]
2021-10-19 22:34 ` [PATCH 07/16] blocK: move plug flush functions to blk-mq.c Chaitanya Kulkarni
2021-10-20 6:17 ` Christoph Hellwig
2021-10-20 12:23 ` Pavel Begunkov
2021-10-20 12:37 ` Christoph Hellwig
2021-10-20 13:18 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 08/16] block: optimise blk_flush_plug_list Pavel Begunkov
2021-10-20 6:29 ` Christoph Hellwig
2021-10-20 12:26 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 09/16] block: optimise boundary blkdev_read_iter's checks Pavel Begunkov
2021-10-20 6:29 ` Christoph Hellwig
2021-10-19 21:24 ` [PATCH 10/16] block: optimise blkdev_bio_end_io() Pavel Begunkov
2021-10-20 6:30 ` Christoph Hellwig
2021-10-20 12:29 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 11/16] block: add optimised version bio_set_dev() Pavel Begunkov
2021-10-19 22:36 ` Chaitanya Kulkarni
2021-10-20 6:20 ` Christoph Hellwig
2021-10-20 12:29 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 12/16] block: add single bio async direct IO helper Pavel Begunkov
2021-10-20 6:36 ` Christoph Hellwig
2021-10-20 12:35 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 13/16] block: add async version of bio_set_polled Pavel Begunkov
2021-10-20 6:37 ` Christoph Hellwig
2021-10-20 12:58 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 14/16] block: skip advance when async and not needed Pavel Begunkov
2021-10-20 6:41 ` Christoph Hellwig
2021-10-19 21:24 ` [PATCH 15/16] block: optimise blk_may_split for normal rw Pavel Begunkov
2021-10-20 6:25 ` Christoph Hellwig
2021-10-20 13:38 ` Pavel Begunkov
2021-10-19 21:24 ` [PATCH 16/16] block: optimise submit_bio_checks " Pavel Begunkov
2021-10-20 6:26 ` Christoph Hellwig
2021-10-19 23:31 ` [PATCH 00/16] block optimisation round Jens Axboe
2021-10-20 0:21 ` Jens Axboe
2021-10-20 0:22 ` Jens Axboe
2021-10-20 14:12 ` (subset) " Jens Axboe
2021-10-20 14:54 ` Pavel Begunkov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=bb6fe6de2fe6bd69ccf9bc8af049ffedcf52bda0.1634676157.git.asml.silence@gmail.com \
--to=asml.silence@gmail.com \
--cc=axboe@kernel.dk \
--cc=linux-block@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).