From: Jens Axboe <axboe@kernel.dk>
To: linux-aio@kvack.org, linux-block@vger.kernel.org,
linux-api@vger.kernel.org
Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com,
jannh@google.com, viro@ZenIV.linux.org.uk,
Jens Axboe <axboe@kernel.dk>
Subject: [PATCH 10/19] io_uring: batch io_kiocb allocation
Date: Mon, 11 Feb 2019 12:00:40 -0700 [thread overview]
Message-ID: <20190211190049.7888-12-axboe@kernel.dk> (raw)
In-Reply-To: <20190211190049.7888-1-axboe@kernel.dk>
Similarly to how we use the state->ios_left to know how many references
to get to a file, we can use it to allocate the io_kiocb's we need in
bulk.
Reviewed-by: Hannes Reinecke <hare@suse.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 45 ++++++++++++++++++++++++++++++++++++++-------
1 file changed, 38 insertions(+), 7 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 7358dd1dbf3f..e330252dc5de 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -168,6 +168,13 @@ struct io_kiocb {
struct io_submit_state {
struct blk_plug plug;
+ /*
+ * io_kiocb alloc cache
+ */
+ void *reqs[IO_IOPOLL_BATCH];
+ unsigned int free_reqs;
+ unsigned int cur_req;
+
/*
* File reference cache
*/
@@ -305,20 +312,40 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
wake_up(&ctx->wait);
}
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+ struct io_submit_state *state)
{
struct io_kiocb *req;
if (!percpu_ref_tryget(&ctx->refs))
return NULL;
- req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
- if (req) {
- req->ctx = ctx;
- req->flags = 0;
- return req;
+ if (!state) {
+ req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
+ if (unlikely(!req))
+ goto out;
+ } else if (!state->free_reqs) {
+ size_t sz;
+ int ret;
+
+ sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+ ret = kmem_cache_alloc_bulk(req_cachep, __GFP_NOWARN, sz,
+ state->reqs);
+ if (unlikely(ret <= 0))
+ goto out;
+ state->free_reqs = ret - 1;
+ state->cur_req = 1;
+ req = state->reqs[0];
+ } else {
+ req = state->reqs[state->cur_req];
+ state->free_reqs--;
+ state->cur_req++;
}
+ req->ctx = ctx;
+ req->flags = 0;
+ return req;
+out:
io_ring_drop_ctx_refs(ctx, 1);
return NULL;
}
@@ -1007,7 +1034,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
if (unlikely(s->sqe->flags))
return -EINVAL;
- req = io_get_req(ctx);
+ req = io_get_req(ctx, state);
if (unlikely(!req))
return -EAGAIN;
@@ -1041,6 +1068,9 @@ static void io_submit_state_end(struct io_submit_state *state)
{
blk_finish_plug(&state->plug);
io_file_put(state, NULL);
+ if (state->free_reqs)
+ kmem_cache_free_bulk(req_cachep, state->free_reqs,
+ &state->reqs[state->cur_req]);
}
/*
@@ -1050,6 +1080,7 @@ static void io_submit_state_start(struct io_submit_state *state,
struct io_ring_ctx *ctx, unsigned max_ios)
{
blk_start_plug(&state->plug);
+ state->free_reqs = 0;
state->file = NULL;
state->ios_left = max_ios;
}
--
2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: Jens Axboe <axboe@kernel.dk>
To: linux-aio@kvack.org, linux-block@vger.kernel.org,
linux-api@vger.kernel.org
Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com,
jannh@google.com, viro@ZenIV.linux.org.uk,
Jens Axboe <axboe@kernel.dk>
Subject: [PATCH 10/19] io_uring: batch io_kiocb allocation
Date: Mon, 11 Feb 2019 12:00:40 -0700 [thread overview]
Message-ID: <20190211190049.7888-12-axboe@kernel.dk> (raw)
In-Reply-To: <20190211190049.7888-1-axboe@kernel.dk>
Similarly to how we use the state->ios_left to know how many references
to get to a file, we can use it to allocate the io_kiocb's we need in
bulk.
Reviewed-by: Hannes Reinecke <hare@suse.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 45 ++++++++++++++++++++++++++++++++++++++-------
1 file changed, 38 insertions(+), 7 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 7358dd1dbf3f..e330252dc5de 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -168,6 +168,13 @@ struct io_kiocb {
struct io_submit_state {
struct blk_plug plug;
+ /*
+ * io_kiocb alloc cache
+ */
+ void *reqs[IO_IOPOLL_BATCH];
+ unsigned int free_reqs;
+ unsigned int cur_req;
+
/*
* File reference cache
*/
@@ -305,20 +312,40 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
wake_up(&ctx->wait);
}
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+ struct io_submit_state *state)
{
struct io_kiocb *req;
if (!percpu_ref_tryget(&ctx->refs))
return NULL;
- req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
- if (req) {
- req->ctx = ctx;
- req->flags = 0;
- return req;
+ if (!state) {
+ req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
+ if (unlikely(!req))
+ goto out;
+ } else if (!state->free_reqs) {
+ size_t sz;
+ int ret;
+
+ sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+ ret = kmem_cache_alloc_bulk(req_cachep, __GFP_NOWARN, sz,
+ state->reqs);
+ if (unlikely(ret <= 0))
+ goto out;
+ state->free_reqs = ret - 1;
+ state->cur_req = 1;
+ req = state->reqs[0];
+ } else {
+ req = state->reqs[state->cur_req];
+ state->free_reqs--;
+ state->cur_req++;
}
+ req->ctx = ctx;
+ req->flags = 0;
+ return req;
+out:
io_ring_drop_ctx_refs(ctx, 1);
return NULL;
}
@@ -1007,7 +1034,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
if (unlikely(s->sqe->flags))
return -EINVAL;
- req = io_get_req(ctx);
+ req = io_get_req(ctx, state);
if (unlikely(!req))
return -EAGAIN;
@@ -1041,6 +1068,9 @@ static void io_submit_state_end(struct io_submit_state *state)
{
blk_finish_plug(&state->plug);
io_file_put(state, NULL);
+ if (state->free_reqs)
+ kmem_cache_free_bulk(req_cachep, state->free_reqs,
+ &state->reqs[state->cur_req]);
}
/*
@@ -1050,6 +1080,7 @@ static void io_submit_state_start(struct io_submit_state *state,
struct io_ring_ctx *ctx, unsigned max_ios)
{
blk_start_plug(&state->plug);
+ state->free_reqs = 0;
state->file = NULL;
state->ios_left = max_ios;
}
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
next prev parent reply other threads:[~2019-02-11 19:01 UTC|newest]
Thread overview: 115+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-02-11 19:00 [PATCHSET v15] io_uring IO interface Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 01/19] fs: add an iopoll method to struct file_operations Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH] io_uring: add io_uring_event cache hit information Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 02/19] block: wire up block device iopoll method Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 03/19] block: add bio_set_polled() helper Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 04/19] iomap: wire up the iopoll method Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 05/19] Add io_uring IO interface Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 06/19] io_uring: add fsync support Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 07/19] io_uring: support for IO polling Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 08/19] fs: add fget_many() and fput_many() Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 09/19] io_uring: use fget/fput_many() for file references Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` Jens Axboe [this message]
2019-02-11 19:00 ` [PATCH 10/19] io_uring: batch io_kiocb allocation Jens Axboe
2019-02-11 19:00 ` [PATCH 11/19] block: implement bio helper to add iter bvec pages to bio Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-20 22:58 ` Ming Lei
2019-02-20 22:58 ` Ming Lei
2019-02-21 17:45 ` Jens Axboe
2019-02-21 17:45 ` Jens Axboe
2019-02-26 3:46 ` Eric Biggers
2019-02-26 3:46 ` Eric Biggers
2019-02-26 4:34 ` Jens Axboe
2019-02-26 4:34 ` Jens Axboe
2019-02-26 15:54 ` Jens Axboe
2019-02-26 15:54 ` Jens Axboe
2019-02-27 1:21 ` Ming Lei
2019-02-27 1:21 ` Ming Lei
2019-02-27 1:47 ` Jens Axboe
2019-02-27 1:47 ` Jens Axboe
2019-02-27 1:53 ` Ming Lei
2019-02-27 1:53 ` Ming Lei
2019-02-27 1:57 ` Jens Axboe
2019-02-27 1:57 ` Jens Axboe
2019-02-27 2:03 ` Jens Axboe
2019-02-27 2:21 ` Ming Lei
2019-02-27 2:21 ` Ming Lei
2019-02-27 2:28 ` Jens Axboe
2019-02-27 2:28 ` Jens Axboe
2019-02-27 2:37 ` Ming Lei
2019-02-27 2:37 ` Ming Lei
2019-02-27 2:43 ` Jens Axboe
2019-02-27 2:43 ` Jens Axboe
2019-02-27 3:09 ` Ming Lei
2019-02-27 3:09 ` Ming Lei
2019-02-27 3:37 ` Jens Axboe
2019-02-27 3:37 ` Jens Axboe
2019-02-27 3:43 ` Jens Axboe
2019-02-27 3:43 ` Jens Axboe
2019-02-27 3:44 ` Ming Lei
2019-02-27 3:44 ` Ming Lei
2019-02-27 4:05 ` Jens Axboe
2019-02-27 4:05 ` Jens Axboe
2019-02-27 4:06 ` Jens Axboe
2019-02-27 4:06 ` Jens Axboe
2019-02-27 19:42 ` Christoph Hellwig
2019-02-27 19:42 ` Christoph Hellwig
2019-02-28 8:37 ` Ming Lei
2019-02-28 8:37 ` Ming Lei
2019-02-27 23:35 ` Ming Lei
2019-02-27 23:35 ` Ming Lei
2019-03-08 7:55 ` Christoph Hellwig
2019-03-08 7:55 ` Christoph Hellwig
2019-03-08 9:12 ` Ming Lei
2019-03-08 9:12 ` Ming Lei
2019-03-08 8:18 ` Christoph Hellwig
2019-03-08 8:18 ` Christoph Hellwig
2019-02-11 19:00 ` [PATCH 12/19] io_uring: add support for pre-mapped user IO buffers Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-19 19:08 ` Jann Horn
2019-02-19 19:08 ` Jann Horn
2019-02-22 22:29 ` Jens Axboe
2019-02-22 22:29 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 13/19] net: split out functions related to registering inflight socket files Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 14/19] io_uring: add file set registration Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-19 16:12 ` Jann Horn
2019-02-19 16:12 ` Jann Horn
2019-02-22 22:29 ` Jens Axboe
2019-02-22 22:29 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 15/19] io_uring: add submission polling Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 16/19] io_uring: add io_kiocb ref count Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 17/19] io_uring: add support for IORING_OP_POLL Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 18/19] io_uring: allow workqueue item to handle multiple buffered requests Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-11 19:00 ` [PATCH 19/19] io_uring: add io_uring_event cache hit information Jens Axboe
2019-02-11 19:00 ` Jens Axboe
2019-02-21 12:10 ` [PATCHSET v15] io_uring IO interface Marek Majkowski
2019-02-21 12:10 ` Marek Majkowski
2019-02-21 17:48 ` Jens Axboe
2019-02-21 17:48 ` Jens Axboe
2019-02-22 15:01 ` Marek Majkowski
2019-02-22 15:01 ` Marek Majkowski
2019-02-22 22:32 ` Jens Axboe
2019-02-22 22:32 ` Jens Axboe
-- strict thread matches above, loose matches on Subject: below --
2019-02-09 21:13 [PATCHSET v14] " Jens Axboe
2019-02-09 21:13 ` [PATCH 10/19] io_uring: batch io_kiocb allocation Jens Axboe
2019-02-09 21:13 ` Jens Axboe
2019-02-08 17:34 [PATCHSET v13] io_uring IO interface Jens Axboe
2019-02-08 17:34 ` [PATCH 10/19] io_uring: batch io_kiocb allocation Jens Axboe
2019-02-08 17:34 ` Jens Axboe
2019-02-09 9:43 ` Hannes Reinecke
2019-02-09 9:43 ` Hannes Reinecke
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190211190049.7888-12-axboe@kernel.dk \
--to=axboe@kernel.dk \
--cc=avi@scylladb.com \
--cc=hch@lst.de \
--cc=jannh@google.com \
--cc=jmoyer@redhat.com \
--cc=linux-aio@kvack.org \
--cc=linux-api@vger.kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=viro@ZenIV.linux.org.uk \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.