* [PATCH 01/11] block: reuse BIO_INLINE_VECS for integrity bvecs
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 02/11] block: move struct biovec_slab to bio.c Christoph Hellwig
` (11 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
bvec_alloc always uses biovec_slabs, and thus always needs to use the
same number of inline vecs. Share a single definition for the data
and integrity bvecs.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio-integrity.c | 6 ++----
block/bio.c | 6 ------
block/blk.h | 1 +
3 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index c3e5abcfdc98c3..19617fa326c35f 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -14,8 +14,6 @@
#include <linux/slab.h>
#include "blk.h"
-#define BIP_INLINE_VECS 4
-
static struct kmem_cache *bip_slab;
static struct workqueue_struct *kintegrityd_wq;
@@ -63,7 +61,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
inline_vecs = nr_vecs;
} else {
bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
- inline_vecs = BIP_INLINE_VECS;
+ inline_vecs = BIO_INLINE_VECS;
}
if (unlikely(!bip))
@@ -470,6 +468,6 @@ void __init bio_integrity_init(void)
bip_slab = kmem_cache_create("bio_integrity_payload",
sizeof(struct bio_integrity_payload) +
- sizeof(struct bio_vec) * BIP_INLINE_VECS,
+ sizeof(struct bio_vec) * BIO_INLINE_VECS,
0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
}
diff --git a/block/bio.c b/block/bio.c
index 757fee46cefc79..cee2d310f02e78 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -25,12 +25,6 @@
#include "blk.h"
#include "blk-rq-qos.h"
-/*
- * Test patch to inline a certain number of bi_io_vec's inside the bio
- * itself, to shrink a bio data allocation from two mempool calls to one
- */
-#define BIO_INLINE_VECS 4
-
/*
* if you change this list, also change bvec_alloc or things will
* break badly! cannot be bigger than what you can fit into an
diff --git a/block/blk.h b/block/blk.h
index 0198335c583881..e022a0d0f2ce45 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -55,6 +55,7 @@ void blk_free_flush_queue(struct blk_flush_queue *q);
void blk_freeze_queue(struct request_queue *q);
+#define BIO_INLINE_VECS 4
struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
unsigned int bvec_nr_vecs(unsigned short idx);
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 02/11] block: move struct biovec_slab to bio.c
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
2021-02-02 17:19 ` [PATCH 01/11] block: reuse BIO_INLINE_VECS for integrity bvecs Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 03/11] block: factor out a bvec_alloc_gfp helper Christoph Hellwig
` (10 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
struct biovec_slab is only used inside of bio.c, so move it there.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 6 ++++++
include/linux/bio.h | 6 ------
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index cee2d310f02e78..2c359dadfdf6dc 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -25,6 +25,12 @@
#include "blk.h"
#include "blk-rq-qos.h"
+struct biovec_slab {
+ int nr_vecs;
+ char *name;
+ struct kmem_cache *slab;
+};
+
/*
* if you change this list, also change bvec_alloc or things will
* break badly! cannot be bigger than what you can fit into an
diff --git a/include/linux/bio.h b/include/linux/bio.h
index c74857cf12528c..4a84207dd99663 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -720,12 +720,6 @@ struct bio_set {
struct workqueue_struct *rescue_workqueue;
};
-struct biovec_slab {
- int nr_vecs;
- char *name;
- struct kmem_cache *slab;
-};
-
static inline bool bioset_initialized(struct bio_set *bs)
{
return bs->bio_slab != NULL;
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 03/11] block: factor out a bvec_alloc_gfp helper
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
2021-02-02 17:19 ` [PATCH 01/11] block: reuse BIO_INLINE_VECS for integrity bvecs Christoph Hellwig
2021-02-02 17:19 ` [PATCH 02/11] block: move struct biovec_slab to bio.c Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 04/11] block: streamline bvec_alloc Christoph Hellwig
` (9 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
Clean up bvec_alloc a little by factoring out a helper for the gfp_t
manipulations.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 2c359dadfdf6dc..c2152c4bf8a317 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -159,6 +159,16 @@ void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
}
}
+/*
+ * Make the first allocation restricted and don't dump info on allocation
+ * failures, since we'll fall back to the mempool in case of failure.
+ */
+static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
+{
+ return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
+ __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
+}
+
struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
mempool_t *pool)
{
@@ -199,20 +209,12 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
bvl = mempool_alloc(pool, gfp_mask);
} else {
struct biovec_slab *bvs = bvec_slabs + *idx;
- gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
-
- /*
- * Make this allocation restricted and don't dump info on
- * allocation failures, since we'll fallback to the mempool
- * in case of failure.
- */
- __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
/*
* Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
* is set, retry with the 1-entry mempool
*/
- bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
+ bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
*idx = BVEC_POOL_MAX;
goto fallback;
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 04/11] block: streamline bvec_alloc
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (2 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 03/11] block: factor out a bvec_alloc_gfp helper Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 05/11] block: remove the 1 and 4 vec bvec_slabs entries Christoph Hellwig
` (8 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
Avoid the pointless goto by trying the slab allocation first and falling
through to the mempool.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 26 ++++++++++----------------
1 file changed, 10 insertions(+), 16 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index c2152c4bf8a317..321b3479a154da 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -172,8 +172,6 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
mempool_t *pool)
{
- struct bio_vec *bvl;
-
/*
* see comment near bvec_array define!
*/
@@ -201,28 +199,24 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
}
/*
- * idx now points to the pool we want to allocate from. only the
- * 1-vec entry pool is mempool backed.
+ * Try a slab allocation first for all smaller allocations. If that
+ * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
+ * The mempool is sized to handle up to BIO_MAX_PAGES entries.
*/
- if (*idx == BVEC_POOL_MAX) {
-fallback:
- bvl = mempool_alloc(pool, gfp_mask);
- } else {
+ if (*idx < BVEC_POOL_MAX) {
struct biovec_slab *bvs = bvec_slabs + *idx;
+ struct bio_vec *bvl;
- /*
- * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
- * is set, retry with the 1-entry mempool
- */
bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
- if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
- *idx = BVEC_POOL_MAX;
- goto fallback;
+ if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) {
+ (*idx)++;
+ return bvl;
}
+ *idx = BVEC_POOL_MAX;
}
(*idx)++;
- return bvl;
+ return mempool_alloc(pool, gfp_mask);
}
void bio_uninit(struct bio *bio)
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 05/11] block: remove the 1 and 4 vec bvec_slabs entries
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (3 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 04/11] block: streamline bvec_alloc Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 06/11] block: turn the nr_iovecs argument to bio_alloc* into an unsigned short Christoph Hellwig
` (7 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
All bios with up to 4 bvecs use the inline bvecs in the bio itself, so
don't bother to define bvec_slabs entries for them. Also decruftify
the bvec_slabs definition and initialization while we're at it.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 53 ++++++++++++++++-------------------------------------
1 file changed, 16 insertions(+), 37 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 321b3479a154da..ae241252ea14e5 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -25,23 +25,17 @@
#include "blk.h"
#include "blk-rq-qos.h"
-struct biovec_slab {
+static struct biovec_slab {
int nr_vecs;
char *name;
struct kmem_cache *slab;
+} bvec_slabs[] __read_mostly = {
+ { .nr_vecs = 16, .name = "biovec-16" },
+ { .nr_vecs = 64, .name = "biovec-64" },
+ { .nr_vecs = 128, .name = "biovec-128" },
+ { .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
};
-/*
- * if you change this list, also change bvec_alloc or things will
- * break badly! cannot be bigger than what you can fit into an
- * unsigned short
- */
-#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
-static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
- BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
-};
-#undef BV
-
/*
* fs_bio_set is the bio_set containing bio and iovec memory pools used by
* IO code that does not need private memory pools.
@@ -176,12 +170,7 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
* see comment near bvec_array define!
*/
switch (nr) {
- case 1:
- *idx = 0;
- break;
- case 2 ... 4:
- *idx = 1;
- break;
+ /* smaller bios use inline vecs */
case 5 ... 16:
*idx = 2;
break;
@@ -1613,31 +1602,21 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
}
EXPORT_SYMBOL(bioset_init_from_src);
-static void __init biovec_init_slabs(void)
+static int __init init_bio(void)
{
int i;
- for (i = 0; i < BVEC_POOL_NR; i++) {
- int size;
- struct biovec_slab *bvs = bvec_slabs + i;
-
- if (bvs->nr_vecs <= BIO_INLINE_VECS) {
- bvs->slab = NULL;
- continue;
- }
-
- size = bvs->nr_vecs * sizeof(struct bio_vec);
- bvs->slab = kmem_cache_create(bvs->name, size, 0,
- SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
- }
-}
-
-static int __init init_bio(void)
-{
BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
bio_integrity_init();
- biovec_init_slabs();
+
+ for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
+ struct biovec_slab *bvs = bvec_slabs + i;
+
+ bvs->slab = kmem_cache_create(bvs->name,
+ bvs->nr_vecs * sizeof(struct bio_vec), 0,
+ SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
+ }
if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
panic("bio: can't allocate bios\n");
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 06/11] block: turn the nr_iovecs argument to bio_alloc* into an unsigned short
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (4 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 05/11] block: remove the 1 and 4 vec bvec_slabs entries Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 07/11] block: remove a layer of indentation in bio_iov_iter_get_pages Christoph Hellwig
` (6 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
The bi_max_vecs and bi_vcnt fields are defined as unsigned short, so
don't allow passing larger values in.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 4 ++--
include/linux/bio.h | 7 ++++---
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index ae241252ea14e5..3d28d4723f6f2f 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -407,7 +407,7 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
*
* Returns: Pointer to new bio on success, NULL on failure.
*/
-struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
+struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,
struct bio_set *bs)
{
gfp_t saved_gfp = gfp_mask;
@@ -493,7 +493,7 @@ EXPORT_SYMBOL(bio_alloc_bioset);
*
* Returns: Pointer to new bio on success, NULL on failure.
*/
-struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
+struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs)
{
struct bio *bio;
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 4a84207dd99663..9ceeb8ecdb7f23 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -407,8 +407,9 @@ extern void bioset_exit(struct bio_set *);
extern int biovec_init_pool(mempool_t *pool, int pool_entries);
extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
-extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
-struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs);
+struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs,
+ struct bio_set *bs);
+struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
extern void bio_put(struct bio *);
extern void __bio_clone_fast(struct bio *, struct bio *);
@@ -416,7 +417,7 @@ extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
extern struct bio_set fs_bio_set;
-static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
+static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs)
{
return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
}
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 07/11] block: remove a layer of indentation in bio_iov_iter_get_pages
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (5 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 06/11] block: turn the nr_iovecs argument to bio_alloc* into an unsigned short Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 08/11] block: set BIO_NO_PAGE_REF in bio_iov_bvec_set Christoph Hellwig
` (5 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
Remove a pointless layer of indentation after a return statement.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 3d28d4723f6f2f..dd3b2a01c9bfaa 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1081,15 +1081,15 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
bio_iov_bvec_set(bio, iter);
bio_set_flag(bio, BIO_NO_PAGE_REF);
return 0;
- } else {
- do {
- if (bio_op(bio) == REQ_OP_ZONE_APPEND)
- ret = __bio_iov_append_get_pages(bio, iter);
- else
- ret = __bio_iov_iter_get_pages(bio, iter);
- } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
}
+ do {
+ if (bio_op(bio) == REQ_OP_ZONE_APPEND)
+ ret = __bio_iov_append_get_pages(bio, iter);
+ else
+ ret = __bio_iov_iter_get_pages(bio, iter);
+ } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
+
/* don't account direct I/O as memory stall */
bio_clear_flag(bio, BIO_WORKINGSET);
return bio->bi_vcnt ? 0 : ret;
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 08/11] block: set BIO_NO_PAGE_REF in bio_iov_bvec_set
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (6 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 07/11] block: remove a layer of indentation in bio_iov_iter_get_pages Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 09/11] block: mark the bio as cloned " Christoph Hellwig
` (4 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
bio_iov_bvec_set assigns the foreign bvec, so setting the NO_PAGE_REF
directly there seems like the best fit.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index dd3b2a01c9bfaa..f753201238273b 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -941,6 +941,7 @@ static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
bio->bi_io_vec = (struct bio_vec *)iter->bvec;
bio->bi_iter.bi_bvec_done = iter->iov_offset;
bio->bi_iter.bi_size = iter->count;
+ bio_set_flag(bio, BIO_NO_PAGE_REF);
iov_iter_advance(iter, iter->count);
return 0;
@@ -1078,9 +1079,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
if (iov_iter_is_bvec(iter)) {
if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
return -EINVAL;
- bio_iov_bvec_set(bio, iter);
- bio_set_flag(bio, BIO_NO_PAGE_REF);
- return 0;
+ return bio_iov_bvec_set(bio, iter);
}
do {
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 09/11] block: mark the bio as cloned in bio_iov_bvec_set
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (7 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 08/11] block: set BIO_NO_PAGE_REF in bio_iov_bvec_set Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-02 17:19 ` [PATCH 10/11] md/raid10: remove dead code in reshape_request Christoph Hellwig
` (3 subsequent siblings)
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
bio_iov_bvec_set clones the bio_vecs from the iter, and thus should be
treated like a cloned bio in every respect. That also includes not
touching bi_max_vecs as that is a property of the bio allocation and not
its current payload.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/block/bio.c b/block/bio.c
index f753201238273b..a36f955cd120b0 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -937,11 +937,11 @@ static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);
bio->bi_vcnt = iter->nr_segs;
- bio->bi_max_vecs = iter->nr_segs;
bio->bi_io_vec = (struct bio_vec *)iter->bvec;
bio->bi_iter.bi_bvec_done = iter->iov_offset;
bio->bi_iter.bi_size = iter->count;
bio_set_flag(bio, BIO_NO_PAGE_REF);
+ bio_set_flag(bio, BIO_CLONED);
iov_iter_advance(iter, iter->count);
return 0;
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 10/11] md/raid10: remove dead code in reshape_request
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (8 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 09/11] block: mark the bio as cloned " Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-04 17:57 ` Song Liu
2021-02-02 17:19 ` [PATCH 11/11] block: use bi_max_vecs to find the bvec pool Christoph Hellwig
` (2 subsequent siblings)
12 siblings, 1 reply; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
A bio allocated by bio_alloc_bioset comes pre-zeroed, no need to
clear random fields.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
drivers/md/raid10.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index e1eefbec15d444..a9ae7d113492c9 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -4539,10 +4539,6 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
read_bio->bi_private = r10_bio;
read_bio->bi_end_io = end_reshape_read;
bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
- read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
- read_bio->bi_status = 0;
- read_bio->bi_vcnt = 0;
- read_bio->bi_iter.bi_size = 0;
r10_bio->master_bio = read_bio;
r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* [PATCH 11/11] block: use bi_max_vecs to find the bvec pool
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (9 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 10/11] md/raid10: remove dead code in reshape_request Christoph Hellwig
@ 2021-02-02 17:19 ` Christoph Hellwig
2021-02-07 16:21 ` cleanup bvec allocation Christoph Hellwig
2021-02-08 15:33 ` Jens Axboe
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-02 17:19 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
Instead of encoding of the bvec pool using magic bio flags, just use
a helper to find the pool based on the max_vecs value.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
block/bio-integrity.c | 11 ++--
block/bio.c | 104 ++++++++++++++++----------------------
block/blk.h | 6 +--
include/linux/bio.h | 1 -
include/linux/blk_types.h | 29 +----------
5 files changed, 51 insertions(+), 100 deletions(-)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 19617fa326c35f..dfa652122a2dc8 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs,
if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
if (bip->bip_vec)
bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
- bip->bip_slab);
+ bip->bip_max_vcnt);
mempool_free(bip, &bs->bio_integrity_pool);
} else {
kfree(bip);
@@ -70,14 +70,11 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
memset(bip, 0, sizeof(*bip));
if (nr_vecs > inline_vecs) {
- unsigned long idx = 0;
-
- bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
- &bs->bvec_integrity_pool);
+ bip->bip_max_vcnt = nr_vecs;
+ bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
+ &bip->bip_max_vcnt, gfp_mask);
if (!bip->bip_vec)
goto err;
- bip->bip_max_vcnt = bvec_nr_vecs(idx);
- bip->bip_slab = idx;
} else {
bip->bip_vec = bip->bip_inline_vecs;
bip->bip_max_vcnt = inline_vecs;
diff --git a/block/bio.c b/block/bio.c
index a36f955cd120b0..a0eabe2f8b07a5 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -36,6 +36,24 @@ static struct biovec_slab {
{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
};
+static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
+{
+ switch (nr_vecs) {
+ /* smaller bios use inline vecs */
+ case 5 ... 16:
+ return &bvec_slabs[0];
+ case 17 ... 64:
+ return &bvec_slabs[1];
+ case 65 ... 128:
+ return &bvec_slabs[2];
+ case 129 ... BIO_MAX_PAGES:
+ return &bvec_slabs[3];
+ default:
+ BUG();
+ return NULL;
+ }
+}
+
/*
* fs_bio_set is the bio_set containing bio and iovec memory pools used by
* IO code that does not need private memory pools.
@@ -131,26 +149,14 @@ static void bio_put_slab(struct bio_set *bs)
mutex_unlock(&bio_slab_lock);
}
-unsigned int bvec_nr_vecs(unsigned short idx)
+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
{
- return bvec_slabs[--idx].nr_vecs;
-}
+ BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);
-void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
-{
- if (!idx)
- return;
- idx--;
-
- BIO_BUG_ON(idx >= BVEC_POOL_NR);
-
- if (idx == BVEC_POOL_MAX) {
+ if (nr_vecs == BIO_MAX_PAGES)
mempool_free(bv, pool);
- } else {
- struct biovec_slab *bvs = bvec_slabs + idx;
-
- kmem_cache_free(bvs->slab, bv);
- }
+ else if (nr_vecs > BIO_INLINE_VECS)
+ kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
}
/*
@@ -163,48 +169,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
}
-struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
- mempool_t *pool)
+struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
+ gfp_t gfp_mask)
{
+ struct biovec_slab *bvs = biovec_slab(*nr_vecs);
+
+ if (WARN_ON_ONCE(!bvs))
+ return NULL;
+
/*
- * see comment near bvec_array define!
+ * Upgrade the nr_vecs request to take full advantage of the allocation.
+ * We also rely on this in the bvec_free path.
*/
- switch (nr) {
- /* smaller bios use inline vecs */
- case 5 ... 16:
- *idx = 2;
- break;
- case 17 ... 64:
- *idx = 3;
- break;
- case 65 ... 128:
- *idx = 4;
- break;
- case 129 ... BIO_MAX_PAGES:
- *idx = 5;
- break;
- default:
- return NULL;
- }
+ *nr_vecs = bvs->nr_vecs;
/*
* Try a slab allocation first for all smaller allocations. If that
* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
* The mempool is sized to handle up to BIO_MAX_PAGES entries.
*/
- if (*idx < BVEC_POOL_MAX) {
- struct biovec_slab *bvs = bvec_slabs + *idx;
+ if (*nr_vecs < BIO_MAX_PAGES) {
struct bio_vec *bvl;
bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
- if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) {
- (*idx)++;
+ if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
return bvl;
- }
- *idx = BVEC_POOL_MAX;
+ *nr_vecs = BIO_MAX_PAGES;
}
- (*idx)++;
return mempool_alloc(pool, gfp_mask);
}
@@ -231,7 +223,7 @@ static void bio_free(struct bio *bio)
bio_uninit(bio);
if (bs) {
- bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
+ bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
/*
* If we have front padding, adjust the bio pointer before freeing
@@ -275,12 +267,8 @@ EXPORT_SYMBOL(bio_init);
*/
void bio_reset(struct bio *bio)
{
- unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
-
bio_uninit(bio);
-
memset(bio, 0, BIO_RESET_BYTES);
- bio->bi_flags = flags;
atomic_set(&bio->__bi_remaining, 1);
}
EXPORT_SYMBOL(bio_reset);
@@ -453,22 +441,18 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,
bio = p + bs->front_pad;
if (nr_iovecs > BIO_INLINE_VECS) {
- unsigned long idx = 0;
struct bio_vec *bvl = NULL;
- bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
+ bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
if (!bvl && gfp_mask != saved_gfp) {
punt_bios_to_rescuer(bs);
gfp_mask = saved_gfp;
- bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx,
- &bs->bvec_pool);
+ bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
}
-
if (unlikely(!bvl))
goto err_free;
- bio_init(bio, bvl, bvec_nr_vecs(idx));
- bio->bi_flags |= idx << BVEC_POOL_OFFSET;
+ bio_init(bio, bvl, nr_iovecs);
} else if (nr_iovecs) {
bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
} else {
@@ -644,7 +628,7 @@ EXPORT_SYMBOL(bio_put);
*/
void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
{
- BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
+ WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);
/*
* most users will be overriding ->bi_bdev with a new target,
@@ -934,7 +918,7 @@ EXPORT_SYMBOL_GPL(bio_release_pages);
static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
{
- WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);
+ WARN_ON_ONCE(bio->bi_max_vecs);
bio->bi_vcnt = iter->nr_segs;
bio->bi_io_vec = (struct bio_vec *)iter->bvec;
@@ -1495,7 +1479,7 @@ EXPORT_SYMBOL_GPL(bio_trim);
*/
int biovec_init_pool(mempool_t *pool, int pool_entries)
{
- struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
+ struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
return mempool_init_slab_pool(pool, pool_entries, bp->slab);
}
@@ -1605,8 +1589,6 @@ static int __init init_bio(void)
{
int i;
- BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
-
bio_integrity_init();
for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
diff --git a/block/blk.h b/block/blk.h
index e022a0d0f2ce45..bfc4d526f6261c 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -56,9 +56,9 @@ void blk_free_flush_queue(struct blk_flush_queue *q);
void blk_freeze_queue(struct request_queue *q);
#define BIO_INLINE_VECS 4
-struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
-void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
-unsigned int bvec_nr_vecs(unsigned short idx);
+struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
+ gfp_t gfp_mask);
+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
static inline bool biovec_phys_mergeable(struct request_queue *q,
struct bio_vec *vec1, struct bio_vec *vec2)
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 9ceeb8ecdb7f23..3cbbaf76906edb 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -329,7 +329,6 @@ struct bio_integrity_payload {
struct bvec_iter bip_iter;
- unsigned short bip_slab; /* slab the bip came from */
unsigned short bip_vcnt; /* # of integrity bio_vecs */
unsigned short bip_max_vcnt; /* integrity bio_vec slots */
unsigned short bip_flags; /* control flags */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 1bc6f6a01070fc..db026b6ec15ab7 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -227,7 +227,7 @@ struct bio {
* top bits REQ_OP. Use
* accessors.
*/
- unsigned short bi_flags; /* status, etc and bvec pool number */
+ unsigned short bi_flags; /* BIO_* below */
unsigned short bi_ioprio;
unsigned short bi_write_hint;
blk_status_t bi_status;
@@ -307,33 +307,6 @@ enum {
BIO_FLAG_LAST
};
-/* See BVEC_POOL_OFFSET below before adding new flags */
-
-/*
- * We support 6 different bvec pools, the last one is magic in that it
- * is backed by a mempool.
- */
-#define BVEC_POOL_NR 6
-#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)
-
-/*
- * Top 3 bits of bio flags indicate the pool the bvecs came from. We add
- * 1 to the actual index so that 0 indicates that there are no bvecs to be
- * freed.
- */
-#define BVEC_POOL_BITS (3)
-#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
-#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
-#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
-# error "BVEC_POOL_BITS is too small"
-#endif
-
-/*
- * Flags starting here get preserved by bio_reset() - this includes
- * only BVEC_POOL_IDX()
- */
-#define BIO_RESET_BITS BVEC_POOL_OFFSET
-
typedef __u32 __bitwise blk_mq_req_flags_t;
/*
--
2.29.2
^ permalink raw reply related [flat|nested] 15+ messages in thread* Re: cleanup bvec allocation
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (10 preceding siblings ...)
2021-02-02 17:19 ` [PATCH 11/11] block: use bi_max_vecs to find the bvec pool Christoph Hellwig
@ 2021-02-07 16:21 ` Christoph Hellwig
2021-02-08 15:33 ` Jens Axboe
12 siblings, 0 replies; 15+ messages in thread
From: Christoph Hellwig @ 2021-02-07 16:21 UTC (permalink / raw)
To: Jens Axboe; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
On Tue, Feb 02, 2021 at 06:19:18PM +0100, Christoph Hellwig wrote:
> Hi Jens,
>
> This series cleans up various lose ends in the bvec allocator.
Any comments?
^ permalink raw reply [flat|nested] 15+ messages in thread* Re: cleanup bvec allocation
2021-02-02 17:19 cleanup bvec allocation Christoph Hellwig
` (11 preceding siblings ...)
2021-02-07 16:21 ` cleanup bvec allocation Christoph Hellwig
@ 2021-02-08 15:33 ` Jens Axboe
12 siblings, 0 replies; 15+ messages in thread
From: Jens Axboe @ 2021-02-08 15:33 UTC (permalink / raw)
To: Christoph Hellwig; +Cc: Song Liu, Martin K. Petersen, linux-block, linux-raid
On 2/2/21 10:19 AM, Christoph Hellwig wrote:
> Hi Jens,
>
> This series cleans up various lose ends in the bvec allocator.
Applied, thanks.
--
Jens Axboe
^ permalink raw reply [flat|nested] 15+ messages in thread