From: Jens Axboe <axboe@kernel.dk> To: linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, linux-block@vger.kernel.org, linux-arch@vger.kernel.org Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com, Jens Axboe <axboe@kernel.dk> Subject: [PATCH 11/16] io_uring: batch io_kiocb allocation Date: Mon, 14 Jan 2019 19:55:26 -0700 [thread overview] Message-ID: <20190115025531.13985-12-axboe@kernel.dk> (raw) In-Reply-To: <20190115025531.13985-1-axboe@kernel.dk> Similarly to how we use the state->ios_left to know how many references to get to a file, we can use it to allocate the io_kiocb's we need in bulk. Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/io_uring.c | 66 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0d7c44a2d424..d0e4e37592fe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -140,6 +140,13 @@ struct io_submit_state { struct list_multi req_list; unsigned int req_count; + /* + * io_kiocb alloc cache + */ + void *reqs[IO_IOPOLL_BATCH]; + unsigned int free_reqs; + unsigned int cur_req; + /* * File reference cache */ @@ -244,29 +251,52 @@ static void io_fill_cq_error(struct io_ring_ctx *ctx, struct sqe_submit *s, wake_up(&ctx->wait); } -static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) +static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) { + percpu_ref_put_many(&ctx->refs, refs); + + if (waitqueue_active(&ctx->wait)) + wake_up(&ctx->wait); +} + +static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, + struct io_submit_state *state) +{ + gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN; struct io_kiocb *req; if (!percpu_ref_tryget(&ctx->refs)) return NULL; - req = kmem_cache_alloc(req_cachep, GFP_ATOMIC | __GFP_NOWARN); - if (!req) - return NULL; - - req->ctx = ctx; - INIT_LIST_HEAD(&req->list); - req->flags = 0; - return req; -} + if (!state) + req = kmem_cache_alloc(req_cachep, gfp); + else if (!state->free_reqs) { + size_t sz; + int ret; + + sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); + ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, + state->reqs); + if (ret <= 0) + goto out; + state->free_reqs = ret - 1; + state->cur_req = 1; + req = state->reqs[0]; + } else { + req = state->reqs[state->cur_req]; + state->free_reqs--; + state->cur_req++; + } -static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) -{ - percpu_ref_put_many(&ctx->refs, refs); + if (req) { + req->ctx = ctx; + req->flags = 0; + return req; + } - if (waitqueue_active(&ctx->wait)) - wake_up(&ctx->wait); +out: + io_ring_drop_ctx_refs(ctx, 1); + return NULL; } static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr) @@ -910,7 +940,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, struct io_kiocb *req; ssize_t ret; - req = io_get_req(ctx); + req = io_get_req(ctx, state); if (unlikely(!req)) return -EAGAIN; @@ -947,6 +977,9 @@ static void io_submit_state_end(struct io_submit_state *state) if (!list_empty(&state->req_list.list)) io_flush_state_reqs(state->ctx, state); io_file_put(state, NULL); + if (state->free_reqs) + kmem_cache_free_bulk(req_cachep, state->free_reqs, + &state->reqs[state->cur_req]); } /* @@ -958,6 +991,7 @@ static void io_submit_state_start(struct io_submit_state *state, state->ctx = ctx; INIT_LIST_HEAD(&state->req_list.list); state->req_count = 0; + state->free_reqs = 0; state->file = NULL; state->ios_left = max_ios; #ifdef CONFIG_BLOCK -- 2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: Jens Axboe <axboe@kernel.dk> To: linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, linux-block@vger.kernel.org, linux-arch@vger.kernel.org Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com, Jens Axboe <axboe@kernel.dk> Subject: [PATCH 11/16] io_uring: batch io_kiocb allocation Date: Mon, 14 Jan 2019 19:55:26 -0700 [thread overview] Message-ID: <20190115025531.13985-12-axboe@kernel.dk> (raw) In-Reply-To: <20190115025531.13985-1-axboe@kernel.dk> Similarly to how we use the state->ios_left to know how many references to get to a file, we can use it to allocate the io_kiocb's we need in bulk. Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/io_uring.c | 66 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0d7c44a2d424..d0e4e37592fe 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -140,6 +140,13 @@ struct io_submit_state { struct list_multi req_list; unsigned int req_count; + /* + * io_kiocb alloc cache + */ + void *reqs[IO_IOPOLL_BATCH]; + unsigned int free_reqs; + unsigned int cur_req; + /* * File reference cache */ @@ -244,29 +251,52 @@ static void io_fill_cq_error(struct io_ring_ctx *ctx, struct sqe_submit *s, wake_up(&ctx->wait); } -static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) +static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) { + percpu_ref_put_many(&ctx->refs, refs); + + if (waitqueue_active(&ctx->wait)) + wake_up(&ctx->wait); +} + +static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, + struct io_submit_state *state) +{ + gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN; struct io_kiocb *req; if (!percpu_ref_tryget(&ctx->refs)) return NULL; - req = kmem_cache_alloc(req_cachep, GFP_ATOMIC | __GFP_NOWARN); - if (!req) - return NULL; - - req->ctx = ctx; - INIT_LIST_HEAD(&req->list); - req->flags = 0; - return req; -} + if (!state) + req = kmem_cache_alloc(req_cachep, gfp); + else if (!state->free_reqs) { + size_t sz; + int ret; + + sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs)); + ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, + state->reqs); + if (ret <= 0) + goto out; + state->free_reqs = ret - 1; + state->cur_req = 1; + req = state->reqs[0]; + } else { + req = state->reqs[state->cur_req]; + state->free_reqs--; + state->cur_req++; + } -static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) -{ - percpu_ref_put_many(&ctx->refs, refs); + if (req) { + req->ctx = ctx; + req->flags = 0; + return req; + } - if (waitqueue_active(&ctx->wait)) - wake_up(&ctx->wait); +out: + io_ring_drop_ctx_refs(ctx, 1); + return NULL; } static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr) @@ -910,7 +940,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, struct io_kiocb *req; ssize_t ret; - req = io_get_req(ctx); + req = io_get_req(ctx, state); if (unlikely(!req)) return -EAGAIN; @@ -947,6 +977,9 @@ static void io_submit_state_end(struct io_submit_state *state) if (!list_empty(&state->req_list.list)) io_flush_state_reqs(state->ctx, state); io_file_put(state, NULL); + if (state->free_reqs) + kmem_cache_free_bulk(req_cachep, state->free_reqs, + &state->reqs[state->cur_req]); } /* @@ -958,6 +991,7 @@ static void io_submit_state_start(struct io_submit_state *state, state->ctx = ctx; INIT_LIST_HEAD(&state->req_list.list); state->req_count = 0; + state->free_reqs = 0; state->file = NULL; state->ios_left = max_ios; #ifdef CONFIG_BLOCK -- 2.17.1 -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to majordomo@kvack.org. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
next prev parent reply other threads:[~2019-01-15 2:56 UTC|newest] Thread overview: 72+ messages / expand[flat|nested] mbox.gz Atom feed top 2019-01-15 2:55 (unknown), Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 01/16] fs: add an iopoll method to struct file_operations Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 02/16] block: wire up block device iopoll method Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 03/16] block: add bio_set_polled() helper Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 04/16] iomap: wire up the iopoll method Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 05/16] Add io_uring IO interface Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 16:51 ` Jonathan Corbet 2019-01-15 16:51 ` Jonathan Corbet 2019-01-15 16:55 ` Jens Axboe 2019-01-15 16:55 ` Jens Axboe 2019-01-15 17:26 ` Jens Axboe 2019-01-15 17:26 ` Jens Axboe 2019-01-16 10:41 ` Arnd Bergmann 2019-01-16 10:41 ` Arnd Bergmann 2019-01-16 11:00 ` Arnd Bergmann 2019-01-16 11:00 ` Arnd Bergmann 2019-01-16 15:12 ` Jens Axboe 2019-01-16 15:12 ` Jens Axboe 2019-01-16 15:16 ` Arnd Bergmann 2019-01-16 15:16 ` Arnd Bergmann 2019-01-16 15:25 ` Jens Axboe 2019-01-16 15:25 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 06/16] io_uring: add fsync support Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 07/16] io_uring: support for IO polling Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 08/16] io_uring: add submission side request cache Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 09/16] fs: add fget_many() and fput_many() Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 10/16] io_uring: use fget/fput_many() for file references Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` Jens Axboe [this message] 2019-01-15 2:55 ` [PATCH 11/16] io_uring: batch io_kiocb allocation Jens Axboe 2019-01-15 2:55 ` [PATCH 12/16] block: implement bio helper to add iter bvec pages to bio Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 13/16] io_uring: add support for pre-mapped user IO buffers Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-16 10:53 ` Arnd Bergmann 2019-01-16 15:14 ` Jens Axboe 2019-01-16 15:14 ` Jens Axboe 2019-01-16 15:32 ` Jens Axboe 2019-01-16 15:32 ` Jens Axboe 2019-01-16 15:41 ` Arnd Bergmann 2019-01-16 15:47 ` Jens Axboe 2019-01-16 15:47 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 14/16] io_uring: add submission polling Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 15/16] io_uring: add file registration Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-16 10:45 ` Arnd Bergmann 2019-01-16 10:45 ` Arnd Bergmann 2019-01-16 15:15 ` Jens Axboe 2019-01-16 15:15 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 16/16] io_uring: add io_uring_event cache hit information Jens Axboe 2019-01-15 2:55 ` Jens Axboe -- strict thread matches above, loose matches on Subject: below -- 2019-01-12 21:29 [PATCHSET v3] io_uring IO interface Jens Axboe 2019-01-12 21:30 ` [PATCH 11/16] io_uring: batch io_kiocb allocation Jens Axboe 2019-01-12 21:30 ` Jens Axboe 2019-01-08 16:56 [PATCHSET v1] io_uring IO interface Jens Axboe 2019-01-08 16:56 ` [PATCH 11/16] io_uring: batch io_kiocb allocation Jens Axboe 2019-01-08 16:56 ` Jens Axboe 2019-01-09 12:13 ` Christoph Hellwig 2019-01-09 16:57 ` Jens Axboe 2019-01-09 16:57 ` Jens Axboe 2019-01-09 19:03 ` Christoph Hellwig 2019-01-09 20:08 ` Jens Axboe 2019-01-09 20:08 ` Jens Axboe
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20190115025531.13985-12-axboe@kernel.dk \ --to=axboe@kernel.dk \ --cc=avi@scylladb.com \ --cc=hch@lst.de \ --cc=jmoyer@redhat.com \ --cc=linux-aio@kvack.org \ --cc=linux-arch@vger.kernel.org \ --cc=linux-block@vger.kernel.org \ --cc=linux-fsdevel@vger.kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.