* bio splitting cleanups v4
@ 2022-07-27 16:22 Christoph Hellwig
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
` (5 more replies)
0 siblings, 6 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block
Hi Jens,
this series has two parts: the first part moves the ->bio_split
bio_set to the gendisk as it only is used for file system style I/O.
The other patches reshuffle the bio splitting code so that in the future
blk_bio_segment_split can be used to split REQ_OP_ZONE_APPEND bios
under file system / remapping driver control. I plan to use that in
btrfs in the next merge window.
Changes since v3:
- fix a merge error and rebase to the for-5.20/drivers-post branch
Changes since v2:
- trivial rebase to the for-next tree
Changes since v1:
- drop a bogus patch
- fix a comment typo
- fix a commit log typo
- clean up the blk_queue_split calling convention and name
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 1/6] block: change the blk_queue_split calling convention
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
@ 2022-07-27 16:22 ` Christoph Hellwig
2022-07-27 16:29 ` Jens Axboe
` (2 more replies)
2022-07-27 16:22 ` [PATCH 2/6] block: change the blk_queue_bounce " Christoph Hellwig
` (4 subsequent siblings)
5 siblings, 3 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block
The double indirect bio leads to somewhat suboptimal code generation.
Instead return the (original or split) bio, and make sure the
request_queue arguments to the lower level helpers is passed after the
bio to avoid constant reshuffling of the argument passing registers.
Also give it and the helpers used to implement it more descriptive names.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/blk-merge.c | 98 +++++++++++++++++------------------
block/blk-mq.c | 4 +-
block/blk.h | 6 +--
drivers/block/drbd/drbd_req.c | 2 +-
drivers/block/pktcdvd.c | 2 +-
drivers/block/ps3vram.c | 2 +-
drivers/md/dm.c | 6 +--
drivers/md/md.c | 2 +-
drivers/nvme/host/multipath.c | 2 +-
drivers/s390/block/dcssblk.c | 2 +-
include/linux/blkdev.h | 2 +-
11 files changed, 63 insertions(+), 65 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 4c8a699754c97..6e29fb28584ef 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -95,10 +95,8 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
return bio_will_gap(req->q, NULL, bio, req->bio);
}
-static struct bio *blk_bio_discard_split(struct request_queue *q,
- struct bio *bio,
- struct bio_set *bs,
- unsigned *nsegs)
+static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
+ unsigned *nsegs, struct bio_set *bs)
{
unsigned int max_discard_sectors, granularity;
int alignment;
@@ -139,8 +137,8 @@ static struct bio *blk_bio_discard_split(struct request_queue *q,
return bio_split(bio, split_sectors, GFP_NOIO, bs);
}
-static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
- struct bio *bio, struct bio_set *bs, unsigned *nsegs)
+static struct bio *bio_split_write_zeroes(struct bio *bio,
+ struct request_queue *q, unsigned *nsegs, struct bio_set *bs)
{
*nsegs = 0;
@@ -161,8 +159,8 @@ static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
* requests that are submitted to a block device if the start of a bio is not
* aligned to a physical block boundary.
*/
-static inline unsigned get_max_io_size(struct request_queue *q,
- struct bio *bio)
+static inline unsigned get_max_io_size(struct bio *bio,
+ struct request_queue *q)
{
unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT;
unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT;
@@ -247,16 +245,16 @@ static bool bvec_split_segs(const struct request_queue *q,
}
/**
- * blk_bio_segment_split - split a bio in two bios
- * @q: [in] request queue pointer
+ * bio_split_rw - split a bio in two bios
* @bio: [in] bio to be split
- * @bs: [in] bio set to allocate the clone from
+ * @q: [in] request queue pointer
* @segs: [out] number of segments in the bio with the first half of the sectors
+ * @bs: [in] bio set to allocate the clone from
*
* Clone @bio, update the bi_iter of the clone to represent the first sectors
* of @bio and update @bio->bi_iter to represent the remaining sectors. The
* following is guaranteed for the cloned bio:
- * - That it has at most get_max_io_size(@q, @bio) sectors.
+ * - That it has at most get_max_io_size(@bio, @q) sectors.
* - That it has at most queue_max_segments(@q) segments.
*
* Except for discard requests the cloned bio will point at the bi_io_vec of
@@ -265,15 +263,13 @@ static bool bvec_split_segs(const struct request_queue *q,
* responsible for ensuring that @bs is only destroyed after processing of the
* split bio has finished.
*/
-static struct bio *blk_bio_segment_split(struct request_queue *q,
- struct bio *bio,
- struct bio_set *bs,
- unsigned *segs)
+static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
+ unsigned *segs, struct bio_set *bs)
{
struct bio_vec bv, bvprv, *bvprvp = NULL;
struct bvec_iter iter;
unsigned nsegs = 0, bytes = 0;
- const unsigned max_bytes = get_max_io_size(q, bio) << 9;
+ const unsigned max_bytes = get_max_io_size(bio, q) << 9;
const unsigned max_segs = queue_max_segments(q);
bio_for_each_bvec(bv, bio, iter) {
@@ -320,34 +316,33 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
}
/**
- * __blk_queue_split - split a bio and submit the second half
- * @q: [in] request_queue new bio is being queued at
- * @bio: [in, out] bio to be split
- * @nr_segs: [out] number of segments in the first bio
+ * __bio_split_to_limits - split a bio to fit the queue limits
+ * @bio: bio to be split
+ * @q: request_queue new bio is being queued at
+ * @nr_segs: returns the number of segments in the returned bio
+ *
+ * Check if @bio needs splitting based on the queue limits, and if so split off
+ * a bio fitting the limits from the beginning of @bio and return it. @bio is
+ * shortened to the remainder and re-submitted.
*
- * Split a bio into two bios, chain the two bios, submit the second half and
- * store a pointer to the first half in *@bio. If the second bio is still too
- * big it will be split by a recursive call to this function. Since this
- * function may allocate a new bio from q->bio_split, it is the responsibility
- * of the caller to ensure that q->bio_split is only released after processing
- * of the split bio has finished.
+ * The split bio is allocated from @q->bio_split, which is provided by the
+ * block layer.
*/
-void __blk_queue_split(struct request_queue *q, struct bio **bio,
+struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
unsigned int *nr_segs)
{
- struct bio *split = NULL;
+ struct bio *split;
- switch (bio_op(*bio)) {
+ switch (bio_op(bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
- split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs);
+ split = bio_split_discard(bio, q, nr_segs, &q->bio_split);
break;
case REQ_OP_WRITE_ZEROES:
- split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split,
- nr_segs);
+ split = bio_split_write_zeroes(bio, q, nr_segs, &q->bio_split);
break;
default:
- split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
+ split = bio_split_rw(bio, q, nr_segs, &q->bio_split);
break;
}
@@ -356,32 +351,35 @@ void __blk_queue_split(struct request_queue *q, struct bio **bio,
split->bi_opf |= REQ_NOMERGE;
blkcg_bio_issue_init(split);
- bio_chain(split, *bio);
- trace_block_split(split, (*bio)->bi_iter.bi_sector);
- submit_bio_noacct(*bio);
- *bio = split;
+ bio_chain(split, bio);
+ trace_block_split(split, bio->bi_iter.bi_sector);
+ submit_bio_noacct(bio);
+ return split;
}
+ return bio;
}
/**
- * blk_queue_split - split a bio and submit the second half
- * @bio: [in, out] bio to be split
+ * bio_split_to_limits - split a bio to fit the queue limits
+ * @bio: bio to be split
+ *
+ * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
+ * if so split off a bio fitting the limits from the beginning of @bio and
+ * return it. @bio is shortened to the remainder and re-submitted.
*
- * Split a bio into two bios, chains the two bios, submit the second half and
- * store a pointer to the first half in *@bio. Since this function may allocate
- * a new bio from q->bio_split, it is the responsibility of the caller to ensure
- * that q->bio_split is only released after processing of the split bio has
- * finished.
+ * The split bio is allocated from @q->bio_split, which is provided by the
+ * block layer.
*/
-void blk_queue_split(struct bio **bio)
+struct bio *bio_split_to_limits(struct bio *bio)
{
- struct request_queue *q = bdev_get_queue((*bio)->bi_bdev);
+ struct request_queue *q = bdev_get_queue(bio->bi_bdev);
unsigned int nr_segs;
- if (blk_may_split(q, *bio))
- __blk_queue_split(q, bio, &nr_segs);
+ if (bio_may_exceed_limits(bio, q))
+ return __bio_split_to_limits(bio, q, &nr_segs);
+ return bio;
}
-EXPORT_SYMBOL(blk_queue_split);
+EXPORT_SYMBOL(bio_split_to_limits);
unsigned int blk_recalc_rq_segments(struct request *rq)
{
diff --git a/block/blk-mq.c b/block/blk-mq.c
index d716b7f3763f3..7a756ffb15679 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2816,8 +2816,8 @@ void blk_mq_submit_bio(struct bio *bio)
blk_status_t ret;
blk_queue_bounce(q, &bio);
- if (blk_may_split(q, bio))
- __blk_queue_split(q, &bio, &nr_segs);
+ if (bio_may_exceed_limits(bio, q))
+ bio = __bio_split_to_limits(bio, q, &nr_segs);
if (!bio_integrity_prep(bio))
return;
diff --git a/block/blk.h b/block/blk.h
index c4b084bfe87c9..42d00d032bd2a 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -293,7 +293,7 @@ ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *,
const char *, size_t);
-static inline bool blk_may_split(struct request_queue *q, struct bio *bio)
+static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *q)
{
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
@@ -316,8 +316,8 @@ static inline bool blk_may_split(struct request_queue *q, struct bio *bio)
bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
}
-void __blk_queue_split(struct request_queue *q, struct bio **bio,
- unsigned int *nr_segs);
+struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
+ unsigned int *nr_segs);
int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs);
bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 6d8dd14458c69..8f7f144e54f3a 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -1608,7 +1608,7 @@ void drbd_submit_bio(struct bio *bio)
{
struct drbd_device *device = bio->bi_bdev->bd_disk->private_data;
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
/*
* what we "blindly" assume:
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 01a15dbd9cde2..4cea3b08087ed 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -2399,7 +2399,7 @@ static void pkt_submit_bio(struct bio *bio)
struct pktcdvd_device *pd = bio->bi_bdev->bd_disk->queue->queuedata;
struct bio *split;
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
pkt_dbg(2, pd, "start = %6llx stop = %6llx\n",
(unsigned long long)bio->bi_iter.bi_sector,
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
index d1e0fefec90ba..e1d080f680edf 100644
--- a/drivers/block/ps3vram.c
+++ b/drivers/block/ps3vram.c
@@ -586,7 +586,7 @@ static void ps3vram_submit_bio(struct bio *bio)
dev_dbg(&dev->core, "%s\n", __func__);
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
spin_lock_irq(&priv->lock);
busy = !bio_list_empty(&priv->list);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 54c2a23f4e55c..a014a002298bd 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1091,7 +1091,7 @@ static sector_t max_io_len(struct dm_target *ti, sector_t sector)
* Does the target need to split IO even further?
* - varied (per target) IO splitting is a tenet of DM; this
* explains why stacked chunk_sectors based splitting via
- * blk_queue_split() isn't possible here.
+ * bio_split_to_limits() isn't possible here.
*/
if (!ti->max_io_len)
return len;
@@ -1669,10 +1669,10 @@ static void dm_split_and_process_bio(struct mapped_device *md,
is_abnormal = is_abnormal_io(bio);
if (unlikely(is_abnormal)) {
/*
- * Use blk_queue_split() for abnormal IO (e.g. discard, etc)
+ * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc)
* otherwise associated queue_limits won't be imposed.
*/
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
}
init_clone_info(&ci, md, map, bio, is_abnormal);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 35b895813c88b..afaf36b2f6ab8 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -442,7 +442,7 @@ static void md_submit_bio(struct bio *bio)
return;
}
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
if (mddev->ro == 1 && unlikely(rw == WRITE)) {
if (bio_sectors(bio) != 0)
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index a22383844159e..f889b2271ddc3 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -346,7 +346,7 @@ static void nvme_ns_head_submit_bio(struct bio *bio)
* different queue via blk_steal_bios(), so we need to use the bio_split
* pool from the original queue to allocate the bvecs from.
*/
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
srcu_idx = srcu_read_lock(&head->srcu);
ns = nvme_find_path(head);
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 4d8d1759775ae..5187705bd0f39 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -863,7 +863,7 @@ dcssblk_submit_bio(struct bio *bio)
unsigned long source_addr;
unsigned long bytes_done;
- blk_queue_split(&bio);
+ bio = bio_split_to_limits(bio);
bytes_done = 0;
dev_info = bio->bi_bdev->bd_disk->private_data;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d04bdf549efa9..5eef8d2eddc1c 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -864,9 +864,9 @@ void blk_request_module(dev_t devt);
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
void submit_bio_noacct(struct bio *bio);
+struct bio *bio_split_to_limits(struct bio *bio);
extern int blk_lld_busy(struct request_queue *q);
-extern void blk_queue_split(struct bio **);
extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
extern void blk_queue_exit(struct request_queue *q);
extern void blk_sync_queue(struct request_queue *q);
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 2/6] block: change the blk_queue_bounce calling convention
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
@ 2022-07-27 16:22 ` Christoph Hellwig
2022-07-28 7:56 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
` (3 subsequent siblings)
5 siblings, 1 reply; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
The double indirect bio leads to somewhat suboptimal code generation.
Instead return the (original or split) bio, and make sure the
request_queue arguments to the lower level helpers is passed after the
bio to avoid constant reshuffling of the argument passing registers.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-mq.c | 2 +-
block/blk.h | 10 ++++++----
block/bounce.c | 26 +++++++++++++-------------
3 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 7a756ffb15679..f469808579525 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2815,7 +2815,7 @@ void blk_mq_submit_bio(struct bio *bio)
unsigned int nr_segs = 1;
blk_status_t ret;
- blk_queue_bounce(q, &bio);
+ bio = blk_queue_bounce(bio, q);
if (bio_may_exceed_limits(bio, q))
bio = __bio_split_to_limits(bio, q, &nr_segs);
diff --git a/block/blk.h b/block/blk.h
index 42d00d032bd2a..1cc813241cd4f 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -383,7 +383,7 @@ static inline void blk_throtl_bio_endio(struct bio *bio) { }
static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
#endif
-void __blk_queue_bounce(struct request_queue *q, struct bio **bio);
+struct bio *__blk_queue_bounce(struct bio *bio, struct request_queue *q);
static inline bool blk_queue_may_bounce(struct request_queue *q)
{
@@ -392,10 +392,12 @@ static inline bool blk_queue_may_bounce(struct request_queue *q)
max_low_pfn >= max_pfn;
}
-static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
+static inline struct bio *blk_queue_bounce(struct bio *bio,
+ struct request_queue *q)
{
- if (unlikely(blk_queue_may_bounce(q) && bio_has_data(*bio)))
- __blk_queue_bounce(q, bio);
+ if (unlikely(blk_queue_may_bounce(q) && bio_has_data(bio)))
+ return __blk_queue_bounce(bio, q);
+ return bio;
}
#ifdef CONFIG_BLK_CGROUP_IOLATENCY
diff --git a/block/bounce.c b/block/bounce.c
index c8f487af7be37..7cfcb242f9a11 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -199,24 +199,24 @@ static struct bio *bounce_clone_bio(struct bio *bio_src)
return NULL;
}
-void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
+struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q)
{
struct bio *bio;
- int rw = bio_data_dir(*bio_orig);
+ int rw = bio_data_dir(bio_orig);
struct bio_vec *to, from;
struct bvec_iter iter;
unsigned i = 0, bytes = 0;
bool bounce = false;
int sectors;
- bio_for_each_segment(from, *bio_orig, iter) {
+ bio_for_each_segment(from, bio_orig, iter) {
if (i++ < BIO_MAX_VECS)
bytes += from.bv_len;
if (PageHighMem(from.bv_page))
bounce = true;
}
if (!bounce)
- return;
+ return bio_orig;
/*
* Individual bvecs might not be logical block aligned. Round down
@@ -225,13 +225,13 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
*/
sectors = ALIGN_DOWN(bytes, queue_logical_block_size(q)) >>
SECTOR_SHIFT;
- if (sectors < bio_sectors(*bio_orig)) {
- bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
- bio_chain(bio, *bio_orig);
- submit_bio_noacct(*bio_orig);
- *bio_orig = bio;
+ if (sectors < bio_sectors(bio_orig)) {
+ bio = bio_split(bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
+ bio_chain(bio, bio_orig);
+ submit_bio_noacct(bio_orig);
+ bio_orig = bio;
}
- bio = bounce_clone_bio(*bio_orig);
+ bio = bounce_clone_bio(bio_orig);
/*
* Bvec table can't be updated by bio_for_each_segment_all(),
@@ -254,7 +254,7 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
to->bv_page = bounce_page;
}
- trace_block_bio_bounce(*bio_orig);
+ trace_block_bio_bounce(bio_orig);
bio->bi_flags |= (1 << BIO_BOUNCED);
@@ -263,6 +263,6 @@ void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
else
bio->bi_end_io = bounce_end_io_write;
- bio->bi_private = *bio_orig;
- *bio_orig = bio;
+ bio->bi_private = bio_orig;
+ return bio;
}
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
2022-07-27 16:22 ` [PATCH 2/6] block: change the blk_queue_bounce " Christoph Hellwig
@ 2022-07-27 16:22 ` Christoph Hellwig
2022-07-28 8:07 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split Christoph Hellwig
` (2 subsequent siblings)
5 siblings, 1 reply; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Only non-passthrough requests are split by the block layer and use the
->bio_split bio_set. Move it from the request_queue to the gendisk.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-core.c | 9 +--------
block/blk-merge.c | 7 ++++---
block/blk-sysfs.c | 2 --
block/genhd.c | 8 +++++++-
drivers/md/dm.c | 2 +-
include/linux/blkdev.h | 3 ++-
6 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index 123468b9d2e43..59f13d011949d 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -377,7 +377,6 @@ static void blk_timeout_work(struct work_struct *work)
struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
{
struct request_queue *q;
- int ret;
q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
GFP_KERNEL | __GFP_ZERO, node_id);
@@ -396,13 +395,9 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
if (q->id < 0)
goto fail_srcu;
- ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
- if (ret)
- goto fail_id;
-
q->stats = blk_alloc_queue_stats();
if (!q->stats)
- goto fail_split;
+ goto fail_id;
q->node = node_id;
@@ -439,8 +434,6 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
fail_stats:
blk_free_queue_stats(q->stats);
-fail_split:
- bioset_exit(&q->bio_split);
fail_id:
ida_free(&blk_queue_ida, q->id);
fail_srcu:
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6e29fb28584ef..30872a3537648 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -331,18 +331,19 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
unsigned int *nr_segs)
{
+ struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
struct bio *split;
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
- split = bio_split_discard(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_discard(bio, q, nr_segs, bs);
break;
case REQ_OP_WRITE_ZEROES:
- split = bio_split_write_zeroes(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_write_zeroes(bio, q, nr_segs, bs);
break;
default:
- split = bio_split_rw(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_rw(bio, q, nr_segs, bs);
break;
}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index c0303026752d5..e1f009aba6fd2 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -779,8 +779,6 @@ static void blk_release_queue(struct kobject *kobj)
if (queue_is_mq(q))
blk_mq_release(q);
- bioset_exit(&q->bio_split);
-
if (blk_queue_has_srcu(q))
cleanup_srcu_struct(q->srcu);
diff --git a/block/genhd.c b/block/genhd.c
index 44dfcf67ed96a..e3daebeaf9ae9 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1139,6 +1139,7 @@ static void disk_release(struct device *dev)
WARN_ON_ONCE(disk_live(disk));
blkcg_exit_queue(disk->queue);
+ bioset_exit(&disk->bio_split);
disk_release_events(disk);
kfree(disk->random);
@@ -1330,9 +1331,12 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
if (!disk)
goto out_put_queue;
+ if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))
+ goto out_free_disk;
+
disk->bdi = bdi_alloc(node_id);
if (!disk->bdi)
- goto out_free_disk;
+ goto out_free_bioset;
/* bdev_alloc() might need the queue, set before the first call */
disk->queue = q;
@@ -1370,6 +1374,8 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
iput(disk->part0->bd_inode);
out_free_bdi:
bdi_put(disk->bdi);
+out_free_bioset:
+ bioset_exit(&disk->bio_split);
out_free_disk:
kfree(disk);
out_put_queue:
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index a014a002298bd..b7458f2dd3e45 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1693,7 +1693,7 @@ static void dm_split_and_process_bio(struct mapped_device *md,
*/
WARN_ON_ONCE(!dm_io_flagged(io, DM_IO_WAS_SPLIT));
io->split_bio = bio_split(bio, io->sectors, GFP_NOIO,
- &md->queue->bio_split);
+ &md->disk->bio_split);
bio_chain(io->split_bio, bio);
trace_block_split(io->split_bio, bio->bi_iter.bi_sector);
submit_bio_noacct(bio);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 5eef8d2eddc1c..49dcd31e283e8 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -140,6 +140,8 @@ struct gendisk {
struct request_queue *queue;
void *private_data;
+ struct bio_set bio_split;
+
int flags;
unsigned long state;
#define GD_NEED_PART_SCAN 0
@@ -531,7 +533,6 @@ struct request_queue {
struct blk_mq_tag_set *tag_set;
struct list_head tag_set_list;
- struct bio_set bio_split;
struct dentry *debugfs_dir;
struct dentry *sched_debugfs_dir;
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
` (2 preceding siblings ...)
2022-07-27 16:22 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
@ 2022-07-27 16:22 ` Christoph Hellwig
2022-07-28 8:08 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 5/6] block: move bio_allowed_max_sectors to blk-merge.c Christoph Hellwig
2022-07-27 16:23 ` [PATCH 6/6] block: pass struct queue_limits to the bio splitting helpers Christoph Hellwig
5 siblings, 1 reply; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Prepare for reusing blk_bio_segment_split for (file system controlled)
splits of REQ_OP_ZONE_APPEND bios by letting the caller control the
maximum size of the bio.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-merge.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 30872a3537648..ce73b3b6c2a71 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -250,11 +250,12 @@ static bool bvec_split_segs(const struct request_queue *q,
* @q: [in] request queue pointer
* @segs: [out] number of segments in the bio with the first half of the sectors
* @bs: [in] bio set to allocate the clone from
+ * @max_bytes: [in] maximum number of bytes per bio
*
* Clone @bio, update the bi_iter of the clone to represent the first sectors
* of @bio and update @bio->bi_iter to represent the remaining sectors. The
* following is guaranteed for the cloned bio:
- * - That it has at most get_max_io_size(@bio, @q) sectors.
+ * - That it has at most @max_bytes worth of data
* - That it has at most queue_max_segments(@q) segments.
*
* Except for discard requests the cloned bio will point at the bi_io_vec of
@@ -264,12 +265,11 @@ static bool bvec_split_segs(const struct request_queue *q,
* split bio has finished.
*/
static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
- unsigned *segs, struct bio_set *bs)
+ unsigned *segs, struct bio_set *bs, unsigned max_bytes)
{
struct bio_vec bv, bvprv, *bvprvp = NULL;
struct bvec_iter iter;
unsigned nsegs = 0, bytes = 0;
- const unsigned max_bytes = get_max_io_size(bio, q) << 9;
const unsigned max_segs = queue_max_segments(q);
bio_for_each_bvec(bv, bio, iter) {
@@ -343,7 +343,8 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
split = bio_split_write_zeroes(bio, q, nr_segs, bs);
break;
default:
- split = bio_split_rw(bio, q, nr_segs, bs);
+ split = bio_split_rw(bio, q, nr_segs, bs,
+ get_max_io_size(bio, q) << SECTOR_SHIFT);
break;
}
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 5/6] block: move bio_allowed_max_sectors to blk-merge.c
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
` (3 preceding siblings ...)
2022-07-27 16:22 ` [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split Christoph Hellwig
@ 2022-07-27 16:22 ` Christoph Hellwig
2022-07-27 16:23 ` [PATCH 6/6] block: pass struct queue_limits to the bio splitting helpers Christoph Hellwig
5 siblings, 0 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Move this helper into the only file where it is used.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-merge.c | 10 ++++++++++
block/blk.h | 10 ----------
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index ce73b3b6c2a71..c2bf8723f30c6 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -95,6 +95,16 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
return bio_will_gap(req->q, NULL, bio, req->bio);
}
+/*
+ * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
+ * is defined as 'unsigned int', meantime it has to be aligned to with the
+ * logical block size, which is the minimum accepted unit by hardware.
+ */
+static unsigned int bio_allowed_max_sectors(struct request_queue *q)
+{
+ return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9;
+}
+
static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
unsigned *nsegs, struct bio_set *bs)
{
diff --git a/block/blk.h b/block/blk.h
index 1cc813241cd4f..19dd12ade8087 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -349,16 +349,6 @@ static inline void req_set_nomerge(struct request_queue *q, struct request *req)
q->last_merge = NULL;
}
-/*
- * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
- * is defined as 'unsigned int', meantime it has to aligned to with logical
- * block size which is the minimum accepted unit by hardware.
- */
-static inline unsigned int bio_allowed_max_sectors(struct request_queue *q)
-{
- return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9;
-}
-
/*
* Internal io_context interface
*/
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 6/6] block: pass struct queue_limits to the bio splitting helpers
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
` (4 preceding siblings ...)
2022-07-27 16:22 ` [PATCH 5/6] block: move bio_allowed_max_sectors to blk-merge.c Christoph Hellwig
@ 2022-07-27 16:23 ` Christoph Hellwig
5 siblings, 0 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-27 16:23 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Allow using the splitting helpers on just a queue_limits instead of
a full request_queue structure. This will eventually allow file systems
or remapping drivers to split REQ_OP_ZONE_APPEND bios based on limits
calculated as the minimum common capabilities over multiple devices.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/bio-integrity.c | 2 +-
block/bio.c | 2 +-
block/blk-merge.c | 107 ++++++++++++++++++++----------------------
block/blk-mq.c | 4 +-
block/blk.h | 25 +++++-----
5 files changed, 68 insertions(+), 72 deletions(-)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 32929c89ba8a6..3f5685c00e360 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -134,7 +134,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
iv = bip->bip_vec + bip->bip_vcnt;
if (bip->bip_vcnt &&
- bvec_gap_to_prev(bdev_get_queue(bio->bi_bdev),
+ bvec_gap_to_prev(&bdev_get_queue(bio->bi_bdev)->limits,
&bip->bip_vec[bip->bip_vcnt - 1], offset))
return 0;
diff --git a/block/bio.c b/block/bio.c
index 6f9f883f9a65a..0d866d44ce533 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -965,7 +965,7 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio,
* would create a gap, disallow it.
*/
bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
- if (bvec_gap_to_prev(q, bvec, offset))
+ if (bvec_gap_to_prev(&q->limits, bvec, offset))
return 0;
}
diff --git a/block/blk-merge.c b/block/blk-merge.c
index c2bf8723f30c6..ff04e9290715a 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -82,7 +82,7 @@ static inline bool bio_will_gap(struct request_queue *q,
bio_get_first_bvec(next, &nb);
if (biovec_phys_mergeable(q, &pb, &nb))
return false;
- return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
+ return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
}
static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
@@ -100,26 +100,25 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
* is defined as 'unsigned int', meantime it has to be aligned to with the
* logical block size, which is the minimum accepted unit by hardware.
*/
-static unsigned int bio_allowed_max_sectors(struct request_queue *q)
+static unsigned int bio_allowed_max_sectors(struct queue_limits *lim)
{
- return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9;
+ return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
}
-static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
+static struct bio *bio_split_discard(struct bio *bio, struct queue_limits *lim,
unsigned *nsegs, struct bio_set *bs)
{
unsigned int max_discard_sectors, granularity;
- int alignment;
sector_t tmp;
unsigned split_sectors;
*nsegs = 1;
/* Zero-sector (unknown) and one-sector granularities are the same. */
- granularity = max(q->limits.discard_granularity >> 9, 1U);
+ granularity = max(lim->discard_granularity >> 9, 1U);
- max_discard_sectors = min(q->limits.max_discard_sectors,
- bio_allowed_max_sectors(q));
+ max_discard_sectors =
+ min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
max_discard_sectors -= max_discard_sectors % granularity;
if (unlikely(!max_discard_sectors)) {
@@ -136,9 +135,8 @@ static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
* If the next starting sector would be misaligned, stop the discard at
* the previous aligned sector.
*/
- alignment = (q->limits.discard_alignment >> 9) % granularity;
-
- tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
+ tmp = bio->bi_iter.bi_sector + split_sectors -
+ ((lim->discard_alignment >> 9) % granularity);
tmp = sector_div(tmp, granularity);
if (split_sectors > tmp)
@@ -148,17 +146,14 @@ static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
}
static struct bio *bio_split_write_zeroes(struct bio *bio,
- struct request_queue *q, unsigned *nsegs, struct bio_set *bs)
+ struct queue_limits *lim, unsigned *nsegs, struct bio_set *bs)
{
*nsegs = 0;
-
- if (!q->limits.max_write_zeroes_sectors)
+ if (!lim->max_write_zeroes_sectors)
return NULL;
-
- if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)
+ if (bio_sectors(bio) <= lim->max_write_zeroes_sectors)
return NULL;
-
- return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);
+ return bio_split(bio, lim->max_write_zeroes_sectors, GFP_NOIO, bs);
}
/*
@@ -170,16 +165,16 @@ static struct bio *bio_split_write_zeroes(struct bio *bio,
* aligned to a physical block boundary.
*/
static inline unsigned get_max_io_size(struct bio *bio,
- struct request_queue *q)
+ struct queue_limits *lim)
{
- unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT;
- unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT;
- unsigned max_sectors = queue_max_sectors(q), start, end;
+ unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
+ unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
+ unsigned max_sectors = lim->max_sectors, start, end;
- if (q->limits.chunk_sectors) {
+ if (lim->chunk_sectors) {
max_sectors = min(max_sectors,
blk_chunk_sectors_left(bio->bi_iter.bi_sector,
- q->limits.chunk_sectors));
+ lim->chunk_sectors));
}
start = bio->bi_iter.bi_sector & (pbs - 1);
@@ -189,11 +184,10 @@ static inline unsigned get_max_io_size(struct bio *bio,
return max_sectors & ~(lbs - 1);
}
-static inline unsigned get_max_segment_size(const struct request_queue *q,
- struct page *start_page,
- unsigned long offset)
+static inline unsigned get_max_segment_size(struct queue_limits *lim,
+ struct page *start_page, unsigned long offset)
{
- unsigned long mask = queue_segment_boundary(q);
+ unsigned long mask = lim->seg_boundary_mask;
offset = mask & (page_to_phys(start_page) + offset);
@@ -202,12 +196,12 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,
* on 32bit arch, use queue's max segment size when that happens.
*/
return min_not_zero(mask - offset + 1,
- (unsigned long)queue_max_segment_size(q));
+ (unsigned long)lim->max_segment_size);
}
/**
* bvec_split_segs - verify whether or not a bvec should be split in the middle
- * @q: [in] request queue associated with the bio associated with @bv
+ * @lim: [in] queue limits to split based on
* @bv: [in] bvec to examine
* @nsegs: [in,out] Number of segments in the bio being built. Incremented
* by the number of segments from @bv that may be appended to that
@@ -225,10 +219,9 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,
* *@nsegs segments and *@sectors sectors would make that bio unacceptable for
* the block driver.
*/
-static bool bvec_split_segs(const struct request_queue *q,
- const struct bio_vec *bv, unsigned *nsegs,
- unsigned *bytes, unsigned max_segs,
- unsigned max_bytes)
+static bool bvec_split_segs(struct queue_limits *lim, const struct bio_vec *bv,
+ unsigned *nsegs, unsigned *bytes, unsigned max_segs,
+ unsigned max_bytes)
{
unsigned max_len = min(max_bytes, UINT_MAX) - *bytes;
unsigned len = min(bv->bv_len, max_len);
@@ -236,7 +229,7 @@ static bool bvec_split_segs(const struct request_queue *q,
unsigned seg_size = 0;
while (len && *nsegs < max_segs) {
- seg_size = get_max_segment_size(q, bv->bv_page,
+ seg_size = get_max_segment_size(lim, bv->bv_page,
bv->bv_offset + total_len);
seg_size = min(seg_size, len);
@@ -244,7 +237,7 @@ static bool bvec_split_segs(const struct request_queue *q,
total_len += seg_size;
len -= seg_size;
- if ((bv->bv_offset + total_len) & queue_virt_boundary(q))
+ if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
break;
}
@@ -257,7 +250,7 @@ static bool bvec_split_segs(const struct request_queue *q,
/**
* bio_split_rw - split a bio in two bios
* @bio: [in] bio to be split
- * @q: [in] request queue pointer
+ * @lim: [in] queue limits to split based on
* @segs: [out] number of segments in the bio with the first half of the sectors
* @bs: [in] bio set to allocate the clone from
* @max_bytes: [in] maximum number of bytes per bio
@@ -274,30 +267,30 @@ static bool bvec_split_segs(const struct request_queue *q,
* responsible for ensuring that @bs is only destroyed after processing of the
* split bio has finished.
*/
-static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
+static struct bio *bio_split_rw(struct bio *bio, struct queue_limits *lim,
unsigned *segs, struct bio_set *bs, unsigned max_bytes)
{
struct bio_vec bv, bvprv, *bvprvp = NULL;
struct bvec_iter iter;
unsigned nsegs = 0, bytes = 0;
- const unsigned max_segs = queue_max_segments(q);
bio_for_each_bvec(bv, bio, iter) {
/*
* If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it.
*/
- if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
+ if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
goto split;
- if (nsegs < max_segs &&
+ if (nsegs < lim->max_segments &&
bytes + bv.bv_len <= max_bytes &&
bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
nsegs++;
bytes += bv.bv_len;
- } else if (bvec_split_segs(q, &bv, &nsegs, &bytes, max_segs,
- max_bytes)) {
- goto split;
+ } else {
+ if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
+ lim->max_segments, max_bytes))
+ goto split;
}
bvprv = bv;
@@ -314,7 +307,7 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
* split size so that each bio is properly block size aligned, even if
* we do not use the full hardware limits.
*/
- bytes = ALIGN_DOWN(bytes, queue_logical_block_size(q));
+ bytes = ALIGN_DOWN(bytes, lim->logical_block_size);
/*
* Bio splitting may cause subtle trouble such as hang when doing sync
@@ -328,7 +321,7 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
/**
* __bio_split_to_limits - split a bio to fit the queue limits
* @bio: bio to be split
- * @q: request_queue new bio is being queued at
+ * @lim: queue limits to split based on
* @nr_segs: returns the number of segments in the returned bio
*
* Check if @bio needs splitting based on the queue limits, and if so split off
@@ -338,7 +331,7 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
* The split bio is allocated from @q->bio_split, which is provided by the
* block layer.
*/
-struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
+struct bio *__bio_split_to_limits(struct bio *bio, struct queue_limits *lim,
unsigned int *nr_segs)
{
struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
@@ -347,14 +340,14 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
- split = bio_split_discard(bio, q, nr_segs, bs);
+ split = bio_split_discard(bio, lim, nr_segs, bs);
break;
case REQ_OP_WRITE_ZEROES:
- split = bio_split_write_zeroes(bio, q, nr_segs, bs);
+ split = bio_split_write_zeroes(bio, lim, nr_segs, bs);
break;
default:
- split = bio_split_rw(bio, q, nr_segs, bs,
- get_max_io_size(bio, q) << SECTOR_SHIFT);
+ split = bio_split_rw(bio, lim, nr_segs, bs,
+ get_max_io_size(bio, lim) << SECTOR_SHIFT);
break;
}
@@ -384,11 +377,11 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
*/
struct bio *bio_split_to_limits(struct bio *bio)
{
- struct request_queue *q = bdev_get_queue(bio->bi_bdev);
+ struct queue_limits *lim = &bdev_get_queue(bio->bi_bdev)->limits;
unsigned int nr_segs;
- if (bio_may_exceed_limits(bio, q))
- return __bio_split_to_limits(bio, q, &nr_segs);
+ if (bio_may_exceed_limits(bio, lim))
+ return __bio_split_to_limits(bio, lim, &nr_segs);
return bio;
}
EXPORT_SYMBOL(bio_split_to_limits);
@@ -421,7 +414,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq)
}
rq_for_each_bvec(bv, rq, iter)
- bvec_split_segs(rq->q, &bv, &nr_phys_segs, &bytes,
+ bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
UINT_MAX, UINT_MAX);
return nr_phys_segs;
}
@@ -452,8 +445,8 @@ static unsigned blk_bvec_map_sg(struct request_queue *q,
while (nbytes > 0) {
unsigned offset = bvec->bv_offset + total;
- unsigned len = min(get_max_segment_size(q, bvec->bv_page,
- offset), nbytes);
+ unsigned len = min(get_max_segment_size(&q->limits,
+ bvec->bv_page, offset), nbytes);
struct page *page = bvec->bv_page;
/*
diff --git a/block/blk-mq.c b/block/blk-mq.c
index f469808579525..8c2476c577129 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2816,8 +2816,8 @@ void blk_mq_submit_bio(struct bio *bio)
blk_status_t ret;
bio = blk_queue_bounce(bio, q);
- if (bio_may_exceed_limits(bio, q))
- bio = __bio_split_to_limits(bio, q, &nr_segs);
+ if (bio_may_exceed_limits(bio, &q->limits))
+ bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
if (!bio_integrity_prep(bio))
return;
diff --git a/block/blk.h b/block/blk.h
index 19dd12ade8087..d70c95986e116 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -102,23 +102,23 @@ static inline bool biovec_phys_mergeable(struct request_queue *q,
return true;
}
-static inline bool __bvec_gap_to_prev(struct request_queue *q,
+static inline bool __bvec_gap_to_prev(struct queue_limits *lim,
struct bio_vec *bprv, unsigned int offset)
{
- return (offset & queue_virt_boundary(q)) ||
- ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
+ return (offset & lim->virt_boundary_mask) ||
+ ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
}
/*
* Check if adding a bio_vec after bprv with offset would create a gap in
* the SG list. Most drivers don't care about this, but some do.
*/
-static inline bool bvec_gap_to_prev(struct request_queue *q,
+static inline bool bvec_gap_to_prev(struct queue_limits *lim,
struct bio_vec *bprv, unsigned int offset)
{
- if (!queue_virt_boundary(q))
+ if (!lim->virt_boundary_mask)
return false;
- return __bvec_gap_to_prev(q, bprv, offset);
+ return __bvec_gap_to_prev(lim, bprv, offset);
}
static inline bool rq_mergeable(struct request *rq)
@@ -194,7 +194,8 @@ static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio_integrity_payload *bip = bio_integrity(req->bio);
struct bio_integrity_payload *bip_next = bio_integrity(next);
- return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+ return bvec_gap_to_prev(&req->q->limits,
+ &bip->bip_vec[bip->bip_vcnt - 1],
bip_next->bip_vec[0].bv_offset);
}
@@ -204,7 +205,8 @@ static inline bool integrity_req_gap_front_merge(struct request *req,
struct bio_integrity_payload *bip = bio_integrity(bio);
struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
- return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
+ return bvec_gap_to_prev(&req->q->limits,
+ &bip->bip_vec[bip->bip_vcnt - 1],
bip_next->bip_vec[0].bv_offset);
}
@@ -293,7 +295,8 @@ ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *,
const char *, size_t);
-static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *q)
+static inline bool bio_may_exceed_limits(struct bio *bio,
+ struct queue_limits *lim)
{
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
@@ -312,11 +315,11 @@ static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *
* to the performance impact of cloned bios themselves the loop below
* doesn't matter anyway.
*/
- return q->limits.chunk_sectors || bio->bi_vcnt != 1 ||
+ return lim->chunk_sectors || bio->bi_vcnt != 1 ||
bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
}
-struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
+struct bio *__bio_split_to_limits(struct bio *bio, struct queue_limits *lim,
unsigned int *nr_segs);
int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs);
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* Re: [PATCH 1/6] block: change the blk_queue_split calling convention
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
@ 2022-07-27 16:29 ` Jens Axboe
2022-07-28 1:53 ` Damien Le Moal
2022-07-28 7:55 ` Hannes Reinecke
2 siblings, 0 replies; 17+ messages in thread
From: Jens Axboe @ 2022-07-27 16:29 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: linux-block
On Wed, 27 Jul 2022 12:22:55 -0400, Christoph Hellwig wrote:
> The double indirect bio leads to somewhat suboptimal code generation.
> Instead return the (original or split) bio, and make sure the
> request_queue arguments to the lower level helpers is passed after the
> bio to avoid constant reshuffling of the argument passing registers.
>
> Also give it and the helpers used to implement it more descriptive names.
>
> [...]
Applied, thanks!
[1/6] block: change the blk_queue_split calling convention
commit: b1b9c6f4d078f50b62e8cc0f4d3bd4c87d411377
[2/6] block: change the blk_queue_bounce calling convention
commit: 4d70e071de1fe363e81fdfd7b0b1686355120fdb
[3/6] block: move ->bio_split to the gendisk
commit: 1be3479b85330141b54a102903e5f07948362695
[4/6] block: move the call to get_max_io_size out of blk_bio_segment_split
commit: 0ef1e5aa4337e291e9fe590e01e1da3021be6743
[5/6] block: move bio_allowed_max_sectors to blk-merge.c
commit: fa785c340621a4da8f6ee70b6e5ad263c4c4bbb5
[6/6] block: pass struct queue_limits to the bio splitting helpers
commit: c9ca8dcc66a99d1123f0fdc2dc161436b93d194b
Best regards,
--
Jens Axboe
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 1/6] block: change the blk_queue_split calling convention
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
2022-07-27 16:29 ` Jens Axboe
@ 2022-07-28 1:53 ` Damien Le Moal
2022-07-28 7:55 ` Hannes Reinecke
2 siblings, 0 replies; 17+ messages in thread
From: Damien Le Moal @ 2022-07-28 1:53 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe; +Cc: linux-block
On 7/28/22 01:22, Christoph Hellwig wrote:
> The double indirect bio leads to somewhat suboptimal code generation.
> Instead return the (original or split) bio, and make sure the
> request_queue arguments to the lower level helpers is passed after the
> bio to avoid constant reshuffling of the argument passing registers.
>
> Also give it and the helpers used to implement it more descriptive names.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
Looks good to me.
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
> ---
> block/blk-merge.c | 98 +++++++++++++++++------------------
> block/blk-mq.c | 4 +-
> block/blk.h | 6 +--
> drivers/block/drbd/drbd_req.c | 2 +-
> drivers/block/pktcdvd.c | 2 +-
> drivers/block/ps3vram.c | 2 +-
> drivers/md/dm.c | 6 +--
> drivers/md/md.c | 2 +-
> drivers/nvme/host/multipath.c | 2 +-
> drivers/s390/block/dcssblk.c | 2 +-
> include/linux/blkdev.h | 2 +-
> 11 files changed, 63 insertions(+), 65 deletions(-)
>
> diff --git a/block/blk-merge.c b/block/blk-merge.c
> index 4c8a699754c97..6e29fb28584ef 100644
> --- a/block/blk-merge.c
> +++ b/block/blk-merge.c
> @@ -95,10 +95,8 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
> return bio_will_gap(req->q, NULL, bio, req->bio);
> }
>
> -static struct bio *blk_bio_discard_split(struct request_queue *q,
> - struct bio *bio,
> - struct bio_set *bs,
> - unsigned *nsegs)
> +static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
> + unsigned *nsegs, struct bio_set *bs)
> {
> unsigned int max_discard_sectors, granularity;
> int alignment;
> @@ -139,8 +137,8 @@ static struct bio *blk_bio_discard_split(struct request_queue *q,
> return bio_split(bio, split_sectors, GFP_NOIO, bs);
> }
>
> -static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
> - struct bio *bio, struct bio_set *bs, unsigned *nsegs)
> +static struct bio *bio_split_write_zeroes(struct bio *bio,
> + struct request_queue *q, unsigned *nsegs, struct bio_set *bs)
> {
> *nsegs = 0;
>
> @@ -161,8 +159,8 @@ static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
> * requests that are submitted to a block device if the start of a bio is not
> * aligned to a physical block boundary.
> */
> -static inline unsigned get_max_io_size(struct request_queue *q,
> - struct bio *bio)
> +static inline unsigned get_max_io_size(struct bio *bio,
> + struct request_queue *q)
> {
> unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT;
> unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT;
> @@ -247,16 +245,16 @@ static bool bvec_split_segs(const struct request_queue *q,
> }
>
> /**
> - * blk_bio_segment_split - split a bio in two bios
> - * @q: [in] request queue pointer
> + * bio_split_rw - split a bio in two bios
> * @bio: [in] bio to be split
> - * @bs: [in] bio set to allocate the clone from
> + * @q: [in] request queue pointer
> * @segs: [out] number of segments in the bio with the first half of the sectors
> + * @bs: [in] bio set to allocate the clone from
> *
> * Clone @bio, update the bi_iter of the clone to represent the first sectors
> * of @bio and update @bio->bi_iter to represent the remaining sectors. The
> * following is guaranteed for the cloned bio:
> - * - That it has at most get_max_io_size(@q, @bio) sectors.
> + * - That it has at most get_max_io_size(@bio, @q) sectors.
> * - That it has at most queue_max_segments(@q) segments.
> *
> * Except for discard requests the cloned bio will point at the bi_io_vec of
> @@ -265,15 +263,13 @@ static bool bvec_split_segs(const struct request_queue *q,
> * responsible for ensuring that @bs is only destroyed after processing of the
> * split bio has finished.
> */
> -static struct bio *blk_bio_segment_split(struct request_queue *q,
> - struct bio *bio,
> - struct bio_set *bs,
> - unsigned *segs)
> +static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
> + unsigned *segs, struct bio_set *bs)
> {
> struct bio_vec bv, bvprv, *bvprvp = NULL;
> struct bvec_iter iter;
> unsigned nsegs = 0, bytes = 0;
> - const unsigned max_bytes = get_max_io_size(q, bio) << 9;
> + const unsigned max_bytes = get_max_io_size(bio, q) << 9;
> const unsigned max_segs = queue_max_segments(q);
>
> bio_for_each_bvec(bv, bio, iter) {
> @@ -320,34 +316,33 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
> }
>
> /**
> - * __blk_queue_split - split a bio and submit the second half
> - * @q: [in] request_queue new bio is being queued at
> - * @bio: [in, out] bio to be split
> - * @nr_segs: [out] number of segments in the first bio
> + * __bio_split_to_limits - split a bio to fit the queue limits
> + * @bio: bio to be split
> + * @q: request_queue new bio is being queued at
> + * @nr_segs: returns the number of segments in the returned bio
> + *
> + * Check if @bio needs splitting based on the queue limits, and if so split off
> + * a bio fitting the limits from the beginning of @bio and return it. @bio is
> + * shortened to the remainder and re-submitted.
> *
> - * Split a bio into two bios, chain the two bios, submit the second half and
> - * store a pointer to the first half in *@bio. If the second bio is still too
> - * big it will be split by a recursive call to this function. Since this
> - * function may allocate a new bio from q->bio_split, it is the responsibility
> - * of the caller to ensure that q->bio_split is only released after processing
> - * of the split bio has finished.
> + * The split bio is allocated from @q->bio_split, which is provided by the
> + * block layer.
> */
> -void __blk_queue_split(struct request_queue *q, struct bio **bio,
> +struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
> unsigned int *nr_segs)
> {
> - struct bio *split = NULL;
> + struct bio *split;
>
> - switch (bio_op(*bio)) {
> + switch (bio_op(bio)) {
> case REQ_OP_DISCARD:
> case REQ_OP_SECURE_ERASE:
> - split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs);
> + split = bio_split_discard(bio, q, nr_segs, &q->bio_split);
> break;
> case REQ_OP_WRITE_ZEROES:
> - split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split,
> - nr_segs);
> + split = bio_split_write_zeroes(bio, q, nr_segs, &q->bio_split);
> break;
> default:
> - split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
> + split = bio_split_rw(bio, q, nr_segs, &q->bio_split);
> break;
> }
>
> @@ -356,32 +351,35 @@ void __blk_queue_split(struct request_queue *q, struct bio **bio,
> split->bi_opf |= REQ_NOMERGE;
>
> blkcg_bio_issue_init(split);
> - bio_chain(split, *bio);
> - trace_block_split(split, (*bio)->bi_iter.bi_sector);
> - submit_bio_noacct(*bio);
> - *bio = split;
> + bio_chain(split, bio);
> + trace_block_split(split, bio->bi_iter.bi_sector);
> + submit_bio_noacct(bio);
> + return split;
> }
> + return bio;
> }
>
> /**
> - * blk_queue_split - split a bio and submit the second half
> - * @bio: [in, out] bio to be split
> + * bio_split_to_limits - split a bio to fit the queue limits
> + * @bio: bio to be split
> + *
> + * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
> + * if so split off a bio fitting the limits from the beginning of @bio and
> + * return it. @bio is shortened to the remainder and re-submitted.
> *
> - * Split a bio into two bios, chains the two bios, submit the second half and
> - * store a pointer to the first half in *@bio. Since this function may allocate
> - * a new bio from q->bio_split, it is the responsibility of the caller to ensure
> - * that q->bio_split is only released after processing of the split bio has
> - * finished.
> + * The split bio is allocated from @q->bio_split, which is provided by the
> + * block layer.
> */
> -void blk_queue_split(struct bio **bio)
> +struct bio *bio_split_to_limits(struct bio *bio)
> {
> - struct request_queue *q = bdev_get_queue((*bio)->bi_bdev);
> + struct request_queue *q = bdev_get_queue(bio->bi_bdev);
> unsigned int nr_segs;
>
> - if (blk_may_split(q, *bio))
> - __blk_queue_split(q, bio, &nr_segs);
> + if (bio_may_exceed_limits(bio, q))
> + return __bio_split_to_limits(bio, q, &nr_segs);
> + return bio;
> }
> -EXPORT_SYMBOL(blk_queue_split);
> +EXPORT_SYMBOL(bio_split_to_limits);
>
> unsigned int blk_recalc_rq_segments(struct request *rq)
> {
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index d716b7f3763f3..7a756ffb15679 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -2816,8 +2816,8 @@ void blk_mq_submit_bio(struct bio *bio)
> blk_status_t ret;
>
> blk_queue_bounce(q, &bio);
> - if (blk_may_split(q, bio))
> - __blk_queue_split(q, &bio, &nr_segs);
> + if (bio_may_exceed_limits(bio, q))
> + bio = __bio_split_to_limits(bio, q, &nr_segs);
>
> if (!bio_integrity_prep(bio))
> return;
> diff --git a/block/blk.h b/block/blk.h
> index c4b084bfe87c9..42d00d032bd2a 100644
> --- a/block/blk.h
> +++ b/block/blk.h
> @@ -293,7 +293,7 @@ ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
> ssize_t part_timeout_store(struct device *, struct device_attribute *,
> const char *, size_t);
>
> -static inline bool blk_may_split(struct request_queue *q, struct bio *bio)
> +static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *q)
> {
> switch (bio_op(bio)) {
> case REQ_OP_DISCARD:
> @@ -316,8 +316,8 @@ static inline bool blk_may_split(struct request_queue *q, struct bio *bio)
> bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
> }
>
> -void __blk_queue_split(struct request_queue *q, struct bio **bio,
> - unsigned int *nr_segs);
> +struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
> + unsigned int *nr_segs);
> int ll_back_merge_fn(struct request *req, struct bio *bio,
> unsigned int nr_segs);
> bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
> diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
> index 6d8dd14458c69..8f7f144e54f3a 100644
> --- a/drivers/block/drbd/drbd_req.c
> +++ b/drivers/block/drbd/drbd_req.c
> @@ -1608,7 +1608,7 @@ void drbd_submit_bio(struct bio *bio)
> {
> struct drbd_device *device = bio->bi_bdev->bd_disk->private_data;
>
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> /*
> * what we "blindly" assume:
> diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
> index 01a15dbd9cde2..4cea3b08087ed 100644
> --- a/drivers/block/pktcdvd.c
> +++ b/drivers/block/pktcdvd.c
> @@ -2399,7 +2399,7 @@ static void pkt_submit_bio(struct bio *bio)
> struct pktcdvd_device *pd = bio->bi_bdev->bd_disk->queue->queuedata;
> struct bio *split;
>
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> pkt_dbg(2, pd, "start = %6llx stop = %6llx\n",
> (unsigned long long)bio->bi_iter.bi_sector,
> diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
> index d1e0fefec90ba..e1d080f680edf 100644
> --- a/drivers/block/ps3vram.c
> +++ b/drivers/block/ps3vram.c
> @@ -586,7 +586,7 @@ static void ps3vram_submit_bio(struct bio *bio)
>
> dev_dbg(&dev->core, "%s\n", __func__);
>
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> spin_lock_irq(&priv->lock);
> busy = !bio_list_empty(&priv->list);
> diff --git a/drivers/md/dm.c b/drivers/md/dm.c
> index 54c2a23f4e55c..a014a002298bd 100644
> --- a/drivers/md/dm.c
> +++ b/drivers/md/dm.c
> @@ -1091,7 +1091,7 @@ static sector_t max_io_len(struct dm_target *ti, sector_t sector)
> * Does the target need to split IO even further?
> * - varied (per target) IO splitting is a tenet of DM; this
> * explains why stacked chunk_sectors based splitting via
> - * blk_queue_split() isn't possible here.
> + * bio_split_to_limits() isn't possible here.
> */
> if (!ti->max_io_len)
> return len;
> @@ -1669,10 +1669,10 @@ static void dm_split_and_process_bio(struct mapped_device *md,
> is_abnormal = is_abnormal_io(bio);
> if (unlikely(is_abnormal)) {
> /*
> - * Use blk_queue_split() for abnormal IO (e.g. discard, etc)
> + * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc)
> * otherwise associated queue_limits won't be imposed.
> */
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
> }
>
> init_clone_info(&ci, md, map, bio, is_abnormal);
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 35b895813c88b..afaf36b2f6ab8 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -442,7 +442,7 @@ static void md_submit_bio(struct bio *bio)
> return;
> }
>
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> if (mddev->ro == 1 && unlikely(rw == WRITE)) {
> if (bio_sectors(bio) != 0)
> diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
> index a22383844159e..f889b2271ddc3 100644
> --- a/drivers/nvme/host/multipath.c
> +++ b/drivers/nvme/host/multipath.c
> @@ -346,7 +346,7 @@ static void nvme_ns_head_submit_bio(struct bio *bio)
> * different queue via blk_steal_bios(), so we need to use the bio_split
> * pool from the original queue to allocate the bvecs from.
> */
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> srcu_idx = srcu_read_lock(&head->srcu);
> ns = nvme_find_path(head);
> diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
> index 4d8d1759775ae..5187705bd0f39 100644
> --- a/drivers/s390/block/dcssblk.c
> +++ b/drivers/s390/block/dcssblk.c
> @@ -863,7 +863,7 @@ dcssblk_submit_bio(struct bio *bio)
> unsigned long source_addr;
> unsigned long bytes_done;
>
> - blk_queue_split(&bio);
> + bio = bio_split_to_limits(bio);
>
> bytes_done = 0;
> dev_info = bio->bi_bdev->bd_disk->private_data;
> diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
> index d04bdf549efa9..5eef8d2eddc1c 100644
> --- a/include/linux/blkdev.h
> +++ b/include/linux/blkdev.h
> @@ -864,9 +864,9 @@ void blk_request_module(dev_t devt);
> extern int blk_register_queue(struct gendisk *disk);
> extern void blk_unregister_queue(struct gendisk *disk);
> void submit_bio_noacct(struct bio *bio);
> +struct bio *bio_split_to_limits(struct bio *bio);
>
> extern int blk_lld_busy(struct request_queue *q);
> -extern void blk_queue_split(struct bio **);
> extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
> extern void blk_queue_exit(struct request_queue *q);
> extern void blk_sync_queue(struct request_queue *q);
--
Damien Le Moal
Western Digital Research
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 1/6] block: change the blk_queue_split calling convention
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
2022-07-27 16:29 ` Jens Axboe
2022-07-28 1:53 ` Damien Le Moal
@ 2022-07-28 7:55 ` Hannes Reinecke
2 siblings, 0 replies; 17+ messages in thread
From: Hannes Reinecke @ 2022-07-28 7:55 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe; +Cc: linux-block
On 7/27/22 18:22, Christoph Hellwig wrote:
> The double indirect bio leads to somewhat suboptimal code generation.
> Instead return the (original or split) bio, and make sure the
> request_queue arguments to the lower level helpers is passed after the
> bio to avoid constant reshuffling of the argument passing registers.
>
> Also give it and the helpers used to implement it more descriptive names.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
> block/blk-merge.c | 98 +++++++++++++++++------------------
> block/blk-mq.c | 4 +-
> block/blk.h | 6 +--
> drivers/block/drbd/drbd_req.c | 2 +-
> drivers/block/pktcdvd.c | 2 +-
> drivers/block/ps3vram.c | 2 +-
> drivers/md/dm.c | 6 +--
> drivers/md/md.c | 2 +-
> drivers/nvme/host/multipath.c | 2 +-
> drivers/s390/block/dcssblk.c | 2 +-
> include/linux/blkdev.h | 2 +-
> 11 files changed, 63 insertions(+), 65 deletions(-)
>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions Germany GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Felix Imendörffer
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 2/6] block: change the blk_queue_bounce calling convention
2022-07-27 16:22 ` [PATCH 2/6] block: change the blk_queue_bounce " Christoph Hellwig
@ 2022-07-28 7:56 ` Hannes Reinecke
0 siblings, 0 replies; 17+ messages in thread
From: Hannes Reinecke @ 2022-07-28 7:56 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe
Cc: linux-block, Damien Le Moal, Johannes Thumshirn
On 7/27/22 18:22, Christoph Hellwig wrote:
> The double indirect bio leads to somewhat suboptimal code generation.
> Instead return the (original or split) bio, and make sure the
> request_queue arguments to the lower level helpers is passed after the
> bio to avoid constant reshuffling of the argument passing registers.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
> Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> ---
> block/blk-mq.c | 2 +-
> block/blk.h | 10 ++++++----
> block/bounce.c | 26 +++++++++++++-------------
> 3 files changed, 20 insertions(+), 18 deletions(-)
>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions Germany GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Felix Imendörffer
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-27 16:22 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
@ 2022-07-28 8:07 ` Hannes Reinecke
2022-07-28 14:32 ` Christoph Hellwig
2022-07-28 15:35 ` Ming Lei
0 siblings, 2 replies; 17+ messages in thread
From: Hannes Reinecke @ 2022-07-28 8:07 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe
Cc: linux-block, Damien Le Moal, Johannes Thumshirn
On 7/27/22 18:22, Christoph Hellwig wrote:
> Only non-passthrough requests are split by the block layer and use the
> ->bio_split bio_set. Move it from the request_queue to the gendisk.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
> Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> ---
> block/blk-core.c | 9 +--------
> block/blk-merge.c | 7 ++++---
> block/blk-sysfs.c | 2 --
> block/genhd.c | 8 +++++++-
> drivers/md/dm.c | 2 +-
> include/linux/blkdev.h | 3 ++-
> 6 files changed, 15 insertions(+), 16 deletions(-)
>
> diff --git a/block/blk-core.c b/block/blk-core.c
> index 123468b9d2e43..59f13d011949d 100644
> --- a/block/blk-core.c
> +++ b/block/blk-core.c
> @@ -377,7 +377,6 @@ static void blk_timeout_work(struct work_struct *work)
> struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
> {
> struct request_queue *q;
> - int ret;
>
> q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
> GFP_KERNEL | __GFP_ZERO, node_id);
> @@ -396,13 +395,9 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
> if (q->id < 0)
> goto fail_srcu;
>
> - ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
> - if (ret)
> - goto fail_id;
> -
> q->stats = blk_alloc_queue_stats();
> if (!q->stats)
> - goto fail_split;
> + goto fail_id;
>
> q->node = node_id;
>
> @@ -439,8 +434,6 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
>
> fail_stats:
> blk_free_queue_stats(q->stats);
> -fail_split:
> - bioset_exit(&q->bio_split);
> fail_id:
> ida_free(&blk_queue_ida, q->id);
> fail_srcu:
> diff --git a/block/blk-merge.c b/block/blk-merge.c
> index 6e29fb28584ef..30872a3537648 100644
> --- a/block/blk-merge.c
> +++ b/block/blk-merge.c
> @@ -331,18 +331,19 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
> struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
> unsigned int *nr_segs)
> {
> + struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
> struct bio *split;
>
What happens for nvme-multipath?
While I know that we shouldn't split on a path, experience shows that we
_will_ do it eventually.
Hence, shouldn't we take precaution for hidden disks with no gendisk
attached here?
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions Germany GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Felix Imendörffer
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split
2022-07-27 16:22 ` [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split Christoph Hellwig
@ 2022-07-28 8:08 ` Hannes Reinecke
0 siblings, 0 replies; 17+ messages in thread
From: Hannes Reinecke @ 2022-07-28 8:08 UTC (permalink / raw)
To: Christoph Hellwig, Jens Axboe
Cc: linux-block, Damien Le Moal, Johannes Thumshirn
On 7/27/22 18:22, Christoph Hellwig wrote:
> Prepare for reusing blk_bio_segment_split for (file system controlled)
> splits of REQ_OP_ZONE_APPEND bios by letting the caller control the
> maximum size of the bio.
>
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
> Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> ---
> block/blk-merge.c | 9 +++++----
> 1 file changed, 5 insertions(+), 4 deletions(-)
>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Cheers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions Germany GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), GF: Felix Imendörffer
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-28 8:07 ` Hannes Reinecke
@ 2022-07-28 14:32 ` Christoph Hellwig
2022-07-28 15:35 ` Ming Lei
1 sibling, 0 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-28 14:32 UTC (permalink / raw)
To: Hannes Reinecke
Cc: Christoph Hellwig, Jens Axboe, linux-block, Damien Le Moal,
Johannes Thumshirn
On Thu, Jul 28, 2022 at 10:07:14AM +0200, Hannes Reinecke wrote:
>> + struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
>> struct bio *split;
>>
>
> What happens for nvme-multipath?
> While I know that we shouldn't split on a path, experience shows that we
> _will_ do it eventually.
> Hence, shouldn't we take precaution for hidden disks with no gendisk
> attached here?
Every block device including nvme-multioath has a valid gendisk.
^ permalink raw reply [flat|nested] 17+ messages in thread
* Re: [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-28 8:07 ` Hannes Reinecke
2022-07-28 14:32 ` Christoph Hellwig
@ 2022-07-28 15:35 ` Ming Lei
1 sibling, 0 replies; 17+ messages in thread
From: Ming Lei @ 2022-07-28 15:35 UTC (permalink / raw)
To: Hannes Reinecke
Cc: Christoph Hellwig, Jens Axboe, linux-block, Damien Le Moal,
Johannes Thumshirn
On Thu, Jul 28, 2022 at 10:07:14AM +0200, Hannes Reinecke wrote:
> On 7/27/22 18:22, Christoph Hellwig wrote:
> > Only non-passthrough requests are split by the block layer and use the
> > ->bio_split bio_set. Move it from the request_queue to the gendisk.
> >
> > Signed-off-by: Christoph Hellwig <hch@lst.de>
> > Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
> > Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
> > ---
> > block/blk-core.c | 9 +--------
> > block/blk-merge.c | 7 ++++---
> > block/blk-sysfs.c | 2 --
> > block/genhd.c | 8 +++++++-
> > drivers/md/dm.c | 2 +-
> > include/linux/blkdev.h | 3 ++-
> > 6 files changed, 15 insertions(+), 16 deletions(-)
> >
> > diff --git a/block/blk-core.c b/block/blk-core.c
> > index 123468b9d2e43..59f13d011949d 100644
> > --- a/block/blk-core.c
> > +++ b/block/blk-core.c
> > @@ -377,7 +377,6 @@ static void blk_timeout_work(struct work_struct *work)
> > struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
> > {
> > struct request_queue *q;
> > - int ret;
> > q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
> > GFP_KERNEL | __GFP_ZERO, node_id);
> > @@ -396,13 +395,9 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
> > if (q->id < 0)
> > goto fail_srcu;
> > - ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
> > - if (ret)
> > - goto fail_id;
> > -
> > q->stats = blk_alloc_queue_stats();
> > if (!q->stats)
> > - goto fail_split;
> > + goto fail_id;
> > q->node = node_id;
> > @@ -439,8 +434,6 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
> > fail_stats:
> > blk_free_queue_stats(q->stats);
> > -fail_split:
> > - bioset_exit(&q->bio_split);
> > fail_id:
> > ida_free(&blk_queue_ida, q->id);
> > fail_srcu:
> > diff --git a/block/blk-merge.c b/block/blk-merge.c
> > index 6e29fb28584ef..30872a3537648 100644
> > --- a/block/blk-merge.c
> > +++ b/block/blk-merge.c
> > @@ -331,18 +331,19 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
> > struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
> > unsigned int *nr_segs)
> > {
> > + struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
> > struct bio *split;
>
> What happens for nvme-multipath?
> While I know that we shouldn't split on a path, experience shows that we
> _will_ do it eventually.
You mean contiguous bios should be mapped to one same path? If yes, at
least the current block layer can't do that, here dm-mapth is same with
nvme-mpath, since bio is always split first, and the split small bio
can be mapped to other path.
Thanks,
Ming
^ permalink raw reply [flat|nested] 17+ messages in thread
* [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-26 18:30 bio splitting cleanups v3 Christoph Hellwig
@ 2022-07-26 18:30 ` Christoph Hellwig
0 siblings, 0 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-26 18:30 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Only non-passthrough requests are split by the block layer and use the
->bio_split bio_set. Move it from the request_queue to the gendisk.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-core.c | 9 +--------
block/blk-merge.c | 7 ++++---
block/blk-sysfs.c | 2 --
block/genhd.c | 8 +++++++-
drivers/md/dm.c | 2 +-
include/linux/blkdev.h | 3 ++-
6 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index 3d286a256d3d3..a0d1104c5590c 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -377,7 +377,6 @@ static void blk_timeout_work(struct work_struct *work)
struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
{
struct request_queue *q;
- int ret;
q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
GFP_KERNEL | __GFP_ZERO, node_id);
@@ -396,13 +395,9 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
if (q->id < 0)
goto fail_srcu;
- ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
- if (ret)
- goto fail_id;
-
q->stats = blk_alloc_queue_stats();
if (!q->stats)
- goto fail_split;
+ goto fail_id;
q->node = node_id;
@@ -439,8 +434,6 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
fail_stats:
blk_free_queue_stats(q->stats);
-fail_split:
- bioset_exit(&q->bio_split);
fail_id:
ida_free(&blk_queue_ida, q->id);
fail_srcu:
diff --git a/block/blk-merge.c b/block/blk-merge.c
index e8e92948ac981..9454c0d927d8b 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -331,18 +331,19 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
unsigned int *nr_segs)
{
+ struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
struct bio *split;
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
- split = bio_split_discard(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_discard(bio, q, nr_segs, bs);
break;
case REQ_OP_WRITE_ZEROES:
- split = bio_split_write_zeroes(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_write_zeroes(bio, q, nr_segs, bs);
break;
default:
- split = bio_split_rw(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_rw(bio, q, nr_segs, bs);
break;
}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index c0303026752d5..e1f009aba6fd2 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -779,8 +779,6 @@ static void blk_release_queue(struct kobject *kobj)
if (queue_is_mq(q))
blk_mq_release(q);
- bioset_exit(&q->bio_split);
-
if (blk_queue_has_srcu(q))
cleanup_srcu_struct(q->srcu);
diff --git a/block/genhd.c b/block/genhd.c
index e1d5b10ac1931..b901fea1d55a4 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1151,6 +1151,7 @@ static void disk_release(struct device *dev)
blk_mq_exit_queue(disk->queue);
blkcg_exit_queue(disk->queue);
+ bioset_exit(&disk->bio_split);
disk_release_events(disk);
kfree(disk->random);
@@ -1342,9 +1343,12 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
if (!disk)
goto out_put_queue;
+ if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))
+ goto out_free_disk;
+
disk->bdi = bdi_alloc(node_id);
if (!disk->bdi)
- goto out_free_disk;
+ goto out_free_bioset;
/* bdev_alloc() might need the queue, set before the first call */
disk->queue = q;
@@ -1382,6 +1386,8 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
iput(disk->part0->bd_inode);
out_free_bdi:
bdi_put(disk->bdi);
+out_free_bioset:
+ bioset_exit(&disk->bio_split);
out_free_disk:
kfree(disk);
out_put_queue:
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index a014a002298bd..b7458f2dd3e45 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1693,7 +1693,7 @@ static void dm_split_and_process_bio(struct mapped_device *md,
*/
WARN_ON_ONCE(!dm_io_flagged(io, DM_IO_WAS_SPLIT));
io->split_bio = bio_split(bio, io->sectors, GFP_NOIO,
- &md->queue->bio_split);
+ &md->disk->bio_split);
bio_chain(io->split_bio, bio);
trace_block_split(io->split_bio, bio->bi_iter.bi_sector);
submit_bio_noacct(bio);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 5eef8d2eddc1c..49dcd31e283e8 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -140,6 +140,8 @@ struct gendisk {
struct request_queue *queue;
void *private_data;
+ struct bio_set bio_split;
+
int flags;
unsigned long state;
#define GD_NEED_PART_SCAN 0
@@ -531,7 +533,6 @@ struct request_queue {
struct blk_mq_tag_set *tag_set;
struct list_head tag_set_list;
- struct bio_set bio_split;
struct dentry *debugfs_dir;
struct dentry *sched_debugfs_dir;
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
* [PATCH 3/6] block: move ->bio_split to the gendisk
2022-07-23 6:28 bio splitting cleanups v2 Christoph Hellwig
@ 2022-07-23 6:28 ` Christoph Hellwig
0 siblings, 0 replies; 17+ messages in thread
From: Christoph Hellwig @ 2022-07-23 6:28 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-block, Damien Le Moal, Johannes Thumshirn
Only non-passthrough requests are split by the block layer and use the
->bio_split bio_set. Move it from the request_queue to the gendisk.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
---
block/blk-core.c | 9 +--------
block/blk-merge.c | 7 ++++---
block/blk-sysfs.c | 2 --
block/genhd.c | 8 +++++++-
drivers/md/dm.c | 2 +-
include/linux/blkdev.h | 3 ++-
6 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index 3d286a256d3d3..a0d1104c5590c 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -377,7 +377,6 @@ static void blk_timeout_work(struct work_struct *work)
struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
{
struct request_queue *q;
- int ret;
q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
GFP_KERNEL | __GFP_ZERO, node_id);
@@ -396,13 +395,9 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
if (q->id < 0)
goto fail_srcu;
- ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
- if (ret)
- goto fail_id;
-
q->stats = blk_alloc_queue_stats();
if (!q->stats)
- goto fail_split;
+ goto fail_id;
q->node = node_id;
@@ -439,8 +434,6 @@ struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
fail_stats:
blk_free_queue_stats(q->stats);
-fail_split:
- bioset_exit(&q->bio_split);
fail_id:
ida_free(&blk_queue_ida, q->id);
fail_srcu:
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 95c0f2d72c741..83a9d23f2a333 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -331,18 +331,19 @@ static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q,
struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
unsigned int *nr_segs)
{
+ struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
struct bio *split;
switch (bio_op(bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
- split = bio_split_discard(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_discard(bio, q, nr_segs, bs);
break;
case REQ_OP_WRITE_ZEROES:
- split = bio_split_write_zeroes(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_write_zeroes(bio, q, nr_segs, bs);
break;
default:
- split = bio_split_rw(bio, q, nr_segs, &q->bio_split);
+ split = bio_split_rw(bio, q, nr_segs, bs);
break;
}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index c0303026752d5..e1f009aba6fd2 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -779,8 +779,6 @@ static void blk_release_queue(struct kobject *kobj)
if (queue_is_mq(q))
blk_mq_release(q);
- bioset_exit(&q->bio_split);
-
if (blk_queue_has_srcu(q))
cleanup_srcu_struct(q->srcu);
diff --git a/block/genhd.c b/block/genhd.c
index e1d5b10ac1931..b901fea1d55a4 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1151,6 +1151,7 @@ static void disk_release(struct device *dev)
blk_mq_exit_queue(disk->queue);
blkcg_exit_queue(disk->queue);
+ bioset_exit(&disk->bio_split);
disk_release_events(disk);
kfree(disk->random);
@@ -1342,9 +1343,12 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
if (!disk)
goto out_put_queue;
+ if (bioset_init(&disk->bio_split, BIO_POOL_SIZE, 0, 0))
+ goto out_free_disk;
+
disk->bdi = bdi_alloc(node_id);
if (!disk->bdi)
- goto out_free_disk;
+ goto out_free_bioset;
/* bdev_alloc() might need the queue, set before the first call */
disk->queue = q;
@@ -1382,6 +1386,8 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
iput(disk->part0->bd_inode);
out_free_bdi:
bdi_put(disk->bdi);
+out_free_bioset:
+ bioset_exit(&disk->bio_split);
out_free_disk:
kfree(disk);
out_put_queue:
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index a014a002298bd..b7458f2dd3e45 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1693,7 +1693,7 @@ static void dm_split_and_process_bio(struct mapped_device *md,
*/
WARN_ON_ONCE(!dm_io_flagged(io, DM_IO_WAS_SPLIT));
io->split_bio = bio_split(bio, io->sectors, GFP_NOIO,
- &md->queue->bio_split);
+ &md->disk->bio_split);
bio_chain(io->split_bio, bio);
trace_block_split(io->split_bio, bio->bi_iter.bi_sector);
submit_bio_noacct(bio);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 5eef8d2eddc1c..49dcd31e283e8 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -140,6 +140,8 @@ struct gendisk {
struct request_queue *queue;
void *private_data;
+ struct bio_set bio_split;
+
int flags;
unsigned long state;
#define GD_NEED_PART_SCAN 0
@@ -531,7 +533,6 @@ struct request_queue {
struct blk_mq_tag_set *tag_set;
struct list_head tag_set_list;
- struct bio_set bio_split;
struct dentry *debugfs_dir;
struct dentry *sched_debugfs_dir;
--
2.30.2
^ permalink raw reply related [flat|nested] 17+ messages in thread
end of thread, other threads:[~2022-07-28 15:35 UTC | newest]
Thread overview: 17+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-07-27 16:22 bio splitting cleanups v4 Christoph Hellwig
2022-07-27 16:22 ` [PATCH 1/6] block: change the blk_queue_split calling convention Christoph Hellwig
2022-07-27 16:29 ` Jens Axboe
2022-07-28 1:53 ` Damien Le Moal
2022-07-28 7:55 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 2/6] block: change the blk_queue_bounce " Christoph Hellwig
2022-07-28 7:56 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
2022-07-28 8:07 ` Hannes Reinecke
2022-07-28 14:32 ` Christoph Hellwig
2022-07-28 15:35 ` Ming Lei
2022-07-27 16:22 ` [PATCH 4/6] block: move the call to get_max_io_size out of blk_bio_segment_split Christoph Hellwig
2022-07-28 8:08 ` Hannes Reinecke
2022-07-27 16:22 ` [PATCH 5/6] block: move bio_allowed_max_sectors to blk-merge.c Christoph Hellwig
2022-07-27 16:23 ` [PATCH 6/6] block: pass struct queue_limits to the bio splitting helpers Christoph Hellwig
-- strict thread matches above, loose matches on Subject: below --
2022-07-26 18:30 bio splitting cleanups v3 Christoph Hellwig
2022-07-26 18:30 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
2022-07-23 6:28 bio splitting cleanups v2 Christoph Hellwig
2022-07-23 6:28 ` [PATCH 3/6] block: move ->bio_split to the gendisk Christoph Hellwig
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.