From: Christoph Hellwig <hch@lst.de>
To: Jens Axboe <axboe@kernel.dk>
Cc: Jeffle Xu <jefflexu@linux.alibaba.com>,
Ming Lei <ming.lei@redhat.com>,
Damien Le Moal <Damien.LeMoal@wdc.com>,
Keith Busch <kbusch@kernel.org>, Sagi Grimberg <sagi@grimberg.me>,
"Wunderlich, Mark" <mark.wunderlich@intel.com>,
"Vasudevan, Anil" <anil.vasudevan@intel.com>,
linux-block@vger.kernel.org, linux-fsdevel@vger.kernel.org,
linux-nvme@lists.infradead.org
Subject: [PATCH 08/16] block: replace the spin argument to blk_iopoll with a flags argument
Date: Tue, 15 Jun 2021 15:10:26 +0200 [thread overview]
Message-ID: <20210615131034.752623-9-hch@lst.de> (raw)
In-Reply-To: <20210615131034.752623-1-hch@lst.de>
Switch the boolean spin argument to blk_poll to passing a set of flags
instead. This will allow to control polling behavior in a more fine
grained way.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Tested-by: Mark Wunderlich <mark.wunderlich@intel.com>
---
block/blk-mq.c | 17 +++++++----------
drivers/nvme/host/core.c | 2 +-
fs/block_dev.c | 8 ++++----
fs/io_uring.c | 14 +++++++-------
fs/iomap/direct-io.c | 6 +++---
include/linux/blkdev.h | 4 +++-
include/linux/fs.h | 2 +-
include/linux/iomap.h | 2 +-
mm/page_io.c | 2 +-
9 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index cb94cf9bcf99..c2b2c7abd712 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -3951,7 +3951,7 @@ static bool blk_mq_poll_hybrid(struct request_queue *q, blk_qc_t qc)
}
static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
- bool spin)
+ unsigned int flags)
{
struct blk_mq_hw_ctx *hctx = blk_qc_to_hctx(q, cookie);
long state = current->state;
@@ -3974,7 +3974,7 @@ static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
if (current->state == TASK_RUNNING)
return 1;
- if (ret < 0 || !spin)
+ if (ret < 0 || (flags & BLK_POLL_ONESHOT))
break;
cpu_relax();
} while (!need_resched());
@@ -3987,15 +3987,13 @@ static int blk_mq_poll_classic(struct request_queue *q, blk_qc_t cookie,
* blk_poll - poll for IO completions
* @q: the queue
* @cookie: cookie passed back at IO submission time
- * @spin: whether to spin for completions
+ * @flags: BLK_POLL_* flags that control the behavior
*
* Description:
* Poll for completions on the passed in queue. Returns number of
- * completed entries found. If @spin is true, then blk_poll will continue
- * looping until at least one completion is found, unless the task is
- * otherwise marked running (or we need to reschedule).
+ * completed entries found.
*/
-int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
+int blk_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags)
{
if (cookie == BLK_QC_T_NONE ||
!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
@@ -4004,12 +4002,11 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
if (current->plug)
blk_flush_plug_list(current->plug, false);
- /* If specified not to spin, we also should not sleep. */
- if (spin && q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
+ if (q->poll_nsec != BLK_MQ_POLL_CLASSIC) {
if (blk_mq_poll_hybrid(q, cookie))
return 1;
}
- return blk_mq_poll_classic(q, cookie, spin);
+ return blk_mq_poll_classic(q, cookie, flags);
}
EXPORT_SYMBOL_GPL(blk_poll);
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 24bcae88587a..115c8d90530a 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -1032,7 +1032,7 @@ static void nvme_execute_rq_polled(struct request_queue *q,
blk_execute_rq_nowait(bd_disk, rq, at_head, nvme_end_sync_rq);
while (!completion_done(&wait)) {
- blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
+ blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), 0);
cond_resched();
}
}
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 8600c651b0b0..c9bdaa138909 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -286,7 +286,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
if (!READ_ONCE(bio.bi_private))
break;
if (!(iocb->ki_flags & IOCB_HIPRI) ||
- !blk_poll(bdev_get_queue(bdev), qc, true))
+ !blk_poll(bdev_get_queue(bdev), qc, 0))
blk_io_schedule();
}
__set_current_state(TASK_RUNNING);
@@ -319,12 +319,12 @@ struct blkdev_dio {
static struct bio_set blkdev_dio_pool;
-static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
+static int blkdev_iopoll(struct kiocb *kiocb, unsigned int flags)
{
struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
struct request_queue *q = bdev_get_queue(bdev);
- return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
+ return blk_poll(q, READ_ONCE(kiocb->ki_cookie), flags);
}
static void blkdev_bio_end_io(struct bio *bio)
@@ -475,7 +475,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
if (!READ_ONCE(dio->waiter))
break;
- if (!do_poll || !blk_poll(bdev_get_queue(bdev), qc, true))
+ if (!do_poll || !blk_poll(bdev_get_queue(bdev), qc, 0))
blk_io_schedule();
}
__set_current_state(TASK_RUNNING);
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 5f82954004f6..de8d39e9a154 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2290,18 +2290,18 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
long min)
{
+ unsigned int poll_flags = 0;
struct io_kiocb *req, *tmp;
LIST_HEAD(done);
- bool spin;
- int ret;
+ int ret = 0;
/*
* Only spin for completions if we don't have multiple devices hanging
* off our complete list, and we're under the requested amount.
*/
- spin = !ctx->poll_multi_file && *nr_events < min;
+ if (ctx->poll_multi_file || *nr_events >= min)
+ poll_flags |= BLK_POLL_ONESHOT;
- ret = 0;
list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
struct kiocb *kiocb = &req->rw.kiocb;
@@ -2317,7 +2317,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
if (!list_empty(&done))
break;
- ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
+ ret = kiocb->ki_filp->f_op->iopoll(kiocb, poll_flags);
if (ret < 0)
break;
@@ -2325,8 +2325,8 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
if (READ_ONCE(req->iopoll_completed))
list_move_tail(&req->inflight_entry, &done);
- if (ret && spin)
- spin = false;
+ if (ret)
+ poll_flags |= BLK_POLL_ONESHOT;
ret = 0;
}
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index d5637f467109..9b6c26da3a2d 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -49,13 +49,13 @@ struct iomap_dio {
};
};
-int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
+int iomap_dio_iopoll(struct kiocb *kiocb, unsigned int flags)
{
struct request_queue *q = READ_ONCE(kiocb->private);
if (!q)
return 0;
- return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
+ return blk_poll(q, READ_ONCE(kiocb->ki_cookie), flags);
}
EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
@@ -640,7 +640,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
if (!(iocb->ki_flags & IOCB_HIPRI) ||
!dio->submit.last_queue ||
!blk_poll(dio->submit.last_queue,
- dio->submit.cookie, true))
+ dio->submit.cookie, 0))
blk_io_schedule();
}
__set_current_state(TASK_RUNNING);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d66d0da72529..c7061f25ee5f 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -946,7 +946,9 @@ extern const char *blk_op_str(unsigned int op);
int blk_status_to_errno(blk_status_t status);
blk_status_t errno_to_blk_status(int errno);
-int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
+/* only poll the hardware once, don't continue until a completion was found */
+#define BLK_POLL_ONESHOT (1 << 0)
+int blk_poll(struct request_queue *q, blk_qc_t cookie, unsigned int flags);
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
{
diff --git a/include/linux/fs.h b/include/linux/fs.h
index c3c88fdb9b2a..483fb557d92f 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2026,7 +2026,7 @@ struct file_operations {
ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
- int (*iopoll)(struct kiocb *kiocb, bool spin);
+ int (*iopoll)(struct kiocb *kiocb, unsigned int flags);
int (*iterate) (struct file *, struct dir_context *);
int (*iterate_shared) (struct file *, struct dir_context *);
__poll_t (*poll) (struct file *, struct poll_table_struct *);
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index c87d0cb0de6d..56e5949ccb60 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -275,7 +275,7 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
unsigned int dio_flags);
ssize_t iomap_dio_complete(struct iomap_dio *dio);
-int iomap_dio_iopoll(struct kiocb *kiocb, bool spin);
+int iomap_dio_iopoll(struct kiocb *kiocb, unsigned int flags);
#ifdef CONFIG_SWAP
struct file;
diff --git a/mm/page_io.c b/mm/page_io.c
index c493ce9ebcf5..5d5543fcefa4 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -428,7 +428,7 @@ int swap_readpage(struct page *page, bool synchronous)
if (!READ_ONCE(bio->bi_private))
break;
- if (!blk_poll(disk->queue, qc, true))
+ if (!blk_poll(disk->queue, qc, 0))
blk_io_schedule();
}
__set_current_state(TASK_RUNNING);
--
2.30.2
next prev parent reply other threads:[~2021-06-15 13:13 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-15 13:10 switch block layer polling to a bio based model v4 Christoph Hellwig
2021-06-15 13:10 ` [PATCH 01/16] direct-io: remove blk_poll support Christoph Hellwig
2021-06-15 13:10 ` [PATCH 02/16] block: don't try to poll multi-bio I/Os in __blkdev_direct_IO Christoph Hellwig
2021-06-15 13:10 ` [PATCH 03/16] iomap: don't try to poll multi-bio I/Os in __iomap_dio_rw Christoph Hellwig
2021-06-15 13:10 ` [PATCH 04/16] blk-mq: factor out a blk_qc_to_hctx helper Christoph Hellwig
2021-06-15 13:10 ` [PATCH 05/16] blk-mq: factor out a "classic" poll helper Christoph Hellwig
2021-06-15 13:10 ` [PATCH 06/16] blk-mq: remove blk_qc_t_to_tag and blk_qc_t_is_internal Christoph Hellwig
2021-06-15 13:10 ` [PATCH 07/16] blk-mq: remove blk_qc_t_valid Christoph Hellwig
2021-06-15 13:10 ` Christoph Hellwig [this message]
2021-06-15 13:10 ` [PATCH 09/16] io_uring: don't sleep when polling for I/O Christoph Hellwig
2021-06-15 13:10 ` [PATCH 10/16] block: rename REQ_HIPRI to REQ_POLLED Christoph Hellwig
2021-06-15 13:10 ` [PATCH 11/16] block: use SLAB_TYPESAFE_BY_RCU for the bio slab Christoph Hellwig
2021-06-15 13:10 ` [PATCH 12/16] block: define 'struct bvec_iter' as packed Christoph Hellwig
2021-06-15 13:10 ` [PATCH 13/16] block: switch polling to be bio based Christoph Hellwig
2021-06-16 2:30 ` Ming Lei
2021-06-18 14:01 ` Christoph Hellwig
2021-06-18 14:27 ` Ming Lei
2021-06-21 7:20 ` Christoph Hellwig
2021-06-21 8:35 ` Ming Lei
2021-06-15 13:10 ` [PATCH 14/16] block: don't allow writing to the poll queue attribute Christoph Hellwig
2021-06-15 13:10 ` [PATCH 15/16] nvme-multipath: set QUEUE_FLAG_NOWAIT Christoph Hellwig
2021-06-15 13:10 ` [PATCH 16/16] nvme-multipath: enable polled I/O Christoph Hellwig
2021-06-15 14:37 ` switch block layer polling to a bio based model v4 Keith Busch
2021-06-15 16:06 ` Christoph Hellwig
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210615131034.752623-9-hch@lst.de \
--to=hch@lst.de \
--cc=Damien.LeMoal@wdc.com \
--cc=anil.vasudevan@intel.com \
--cc=axboe@kernel.dk \
--cc=jefflexu@linux.alibaba.com \
--cc=kbusch@kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-nvme@lists.infradead.org \
--cc=mark.wunderlich@intel.com \
--cc=ming.lei@redhat.com \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).