From: Jens Axboe <axboe@kernel.dk> To: linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, linux-block@vger.kernel.org, linux-arch@vger.kernel.org Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com, Jens Axboe <axboe@kernel.dk> Subject: [PATCH 10/16] io_uring: use fget/fput_many() for file references Date: Mon, 14 Jan 2019 19:55:25 -0700 [thread overview] Message-ID: <20190115025531.13985-11-axboe@kernel.dk> (raw) In-Reply-To: <20190115025531.13985-1-axboe@kernel.dk> On the submission side, add file reference batching to the io_submit_state. We get as many references as the number of iocbs we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed. On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap(). Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/io_uring.c | 98 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 62f31f20f3d5..0d7c44a2d424 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -139,6 +139,15 @@ struct io_submit_state { */ struct list_multi req_list; unsigned int req_count; + + /* + * File reference cache + */ + struct file *file; + unsigned int fd; + unsigned int has_refs; + unsigned int used_refs; + unsigned int ios_left; }; static struct kmem_cache *req_cachep; @@ -316,9 +325,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, struct list_head *done) { void *reqs[IO_IOPOLL_BATCH]; + int file_count, to_free; + struct file *file = NULL; struct io_kiocb *req; - int to_free = 0; + file_count = to_free = 0; while (!list_empty(done)) { req = list_first_entry(done, struct io_kiocb, list); list_del(&req->list); @@ -328,11 +339,27 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, reqs[to_free++] = req; (*nr_events)++; - fput(req->rw.ki_filp); + /* + * Batched puts of the same file, to avoid dirtying the + * file usage count multiple times, if avoidable. + */ + if (!file) { + file = req->rw.ki_filp; + file_count = 1; + } else if (file == req->rw.ki_filp) { + file_count++; + } else { + fput_many(file, file_count); + file = req->rw.ki_filp; + file_count = 1; + } + if (to_free == ARRAY_SIZE(reqs)) io_free_req_many(ctx, reqs, &to_free); } + if (file) + fput_many(file, file_count); if (to_free) io_free_req_many(ctx, reqs, &to_free); } @@ -522,14 +549,56 @@ static void io_iopoll_req_issued(struct io_submit_state *state, io_iopoll_req_add_state(state, req); } +static void io_file_put(struct io_submit_state *state, struct file *file) +{ + if (!state) { + fput(file); + } else if (state->file) { + int diff = state->has_refs - state->used_refs; + + if (diff) + fput_many(state->file, diff); + state->file = NULL; + } +} + +/* + * Get as many references to a file as we have IOs left in this submission, + * assuming most submissions are for one file, or at least that each file + * has more than one submission. + */ +static struct file *io_file_get(struct io_submit_state *state, int fd) +{ + if (!state) + return fget(fd); + + if (state->file) { + if (state->fd == fd) { + state->used_refs++; + state->ios_left--; + return state->file; + } + io_file_put(state, NULL); + } + state->file = fget_many(fd, state->ios_left); + if (!state->file) + return NULL; + + state->fd = fd; + state->has_refs = state->ios_left; + state->used_refs = 1; + state->ios_left--; + return state->file; +} + static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct io_ring_ctx *ctx = req->ctx; struct kiocb *kiocb = &req->rw; int ret; - kiocb->ki_filp = fget(sqe->fd); + kiocb->ki_filp = io_file_get(state, sqe->fd); if (unlikely(!kiocb->ki_filp)) return -EBADF; kiocb->ki_pos = sqe->off; @@ -568,7 +637,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, } return 0; out_fput: - fput(kiocb->ki_filp); + io_file_put(state, kiocb->ki_filp); return ret; } @@ -607,7 +676,7 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw, } static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw; @@ -615,7 +684,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, struct file *file; ssize_t ret; - ret = io_prep_rw(req, sqe, force_nonblock); + ret = io_prep_rw(req, sqe, force_nonblock, state); if (ret) return ret; file = kiocb->ki_filp; @@ -650,7 +719,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, } static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw; @@ -658,7 +727,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, struct file *file; ssize_t ret; - ret = io_prep_rw(req, sqe, force_nonblock); + ret = io_prep_rw(req, sqe, force_nonblock, state); if (ret) return ret; file = kiocb->ki_filp; @@ -771,10 +840,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret = io_nop(req, sqe); break; case IORING_OP_READV: - ret = io_read(req, sqe, force_nonblock); + ret = io_read(req, sqe, force_nonblock, state); break; case IORING_OP_WRITEV: - ret = io_write(req, sqe, force_nonblock); + ret = io_write(req, sqe, force_nonblock, state); break; case IORING_OP_FSYNC: ret = io_fsync(req, sqe, force_nonblock); @@ -877,17 +946,20 @@ static void io_submit_state_end(struct io_submit_state *state) blk_finish_plug(&state->plug); if (!list_empty(&state->req_list.list)) io_flush_state_reqs(state->ctx, state); + io_file_put(state, NULL); } /* * Start submission side cache. */ static void io_submit_state_start(struct io_submit_state *state, - struct io_ring_ctx *ctx) + struct io_ring_ctx *ctx, unsigned max_ios) { state->ctx = ctx; INIT_LIST_HEAD(&state->req_list.list); state->req_count = 0; + state->file = NULL; + state->ios_left = max_ios; #ifdef CONFIG_BLOCK state->plug_cb.callback = io_state_unplug; blk_start_plug(&state->plug); @@ -933,7 +1005,7 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) int i, ret = 0, submit = 0; if (to_submit > IO_PLUG_THRESHOLD) { - io_submit_state_start(&state, ctx); + io_submit_state_start(&state, ctx, to_submit); statep = &state; } -- 2.17.1
WARNING: multiple messages have this Message-ID (diff)
From: Jens Axboe <axboe@kernel.dk> To: linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, linux-block@vger.kernel.org, linux-arch@vger.kernel.org Cc: hch@lst.de, jmoyer@redhat.com, avi@scylladb.com, Jens Axboe <axboe@kernel.dk> Subject: [PATCH 10/16] io_uring: use fget/fput_many() for file references Date: Mon, 14 Jan 2019 19:55:25 -0700 [thread overview] Message-ID: <20190115025531.13985-11-axboe@kernel.dk> (raw) In-Reply-To: <20190115025531.13985-1-axboe@kernel.dk> On the submission side, add file reference batching to the io_submit_state. We get as many references as the number of iocbs we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed. On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap(). Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/io_uring.c | 98 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 62f31f20f3d5..0d7c44a2d424 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -139,6 +139,15 @@ struct io_submit_state { */ struct list_multi req_list; unsigned int req_count; + + /* + * File reference cache + */ + struct file *file; + unsigned int fd; + unsigned int has_refs; + unsigned int used_refs; + unsigned int ios_left; }; static struct kmem_cache *req_cachep; @@ -316,9 +325,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, struct list_head *done) { void *reqs[IO_IOPOLL_BATCH]; + int file_count, to_free; + struct file *file = NULL; struct io_kiocb *req; - int to_free = 0; + file_count = to_free = 0; while (!list_empty(done)) { req = list_first_entry(done, struct io_kiocb, list); list_del(&req->list); @@ -328,11 +339,27 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, reqs[to_free++] = req; (*nr_events)++; - fput(req->rw.ki_filp); + /* + * Batched puts of the same file, to avoid dirtying the + * file usage count multiple times, if avoidable. + */ + if (!file) { + file = req->rw.ki_filp; + file_count = 1; + } else if (file == req->rw.ki_filp) { + file_count++; + } else { + fput_many(file, file_count); + file = req->rw.ki_filp; + file_count = 1; + } + if (to_free == ARRAY_SIZE(reqs)) io_free_req_many(ctx, reqs, &to_free); } + if (file) + fput_many(file, file_count); if (to_free) io_free_req_many(ctx, reqs, &to_free); } @@ -522,14 +549,56 @@ static void io_iopoll_req_issued(struct io_submit_state *state, io_iopoll_req_add_state(state, req); } +static void io_file_put(struct io_submit_state *state, struct file *file) +{ + if (!state) { + fput(file); + } else if (state->file) { + int diff = state->has_refs - state->used_refs; + + if (diff) + fput_many(state->file, diff); + state->file = NULL; + } +} + +/* + * Get as many references to a file as we have IOs left in this submission, + * assuming most submissions are for one file, or at least that each file + * has more than one submission. + */ +static struct file *io_file_get(struct io_submit_state *state, int fd) +{ + if (!state) + return fget(fd); + + if (state->file) { + if (state->fd == fd) { + state->used_refs++; + state->ios_left--; + return state->file; + } + io_file_put(state, NULL); + } + state->file = fget_many(fd, state->ios_left); + if (!state->file) + return NULL; + + state->fd = fd; + state->has_refs = state->ios_left; + state->used_refs = 1; + state->ios_left--; + return state->file; +} + static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct io_ring_ctx *ctx = req->ctx; struct kiocb *kiocb = &req->rw; int ret; - kiocb->ki_filp = fget(sqe->fd); + kiocb->ki_filp = io_file_get(state, sqe->fd); if (unlikely(!kiocb->ki_filp)) return -EBADF; kiocb->ki_pos = sqe->off; @@ -568,7 +637,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, } return 0; out_fput: - fput(kiocb->ki_filp); + io_file_put(state, kiocb->ki_filp); return ret; } @@ -607,7 +676,7 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw, } static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw; @@ -615,7 +684,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, struct file *file; ssize_t ret; - ret = io_prep_rw(req, sqe, force_nonblock); + ret = io_prep_rw(req, sqe, force_nonblock, state); if (ret) return ret; file = kiocb->ki_filp; @@ -650,7 +719,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, } static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, - bool force_nonblock) + bool force_nonblock, struct io_submit_state *state) { struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw; @@ -658,7 +727,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, struct file *file; ssize_t ret; - ret = io_prep_rw(req, sqe, force_nonblock); + ret = io_prep_rw(req, sqe, force_nonblock, state); if (ret) return ret; file = kiocb->ki_filp; @@ -771,10 +840,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret = io_nop(req, sqe); break; case IORING_OP_READV: - ret = io_read(req, sqe, force_nonblock); + ret = io_read(req, sqe, force_nonblock, state); break; case IORING_OP_WRITEV: - ret = io_write(req, sqe, force_nonblock); + ret = io_write(req, sqe, force_nonblock, state); break; case IORING_OP_FSYNC: ret = io_fsync(req, sqe, force_nonblock); @@ -877,17 +946,20 @@ static void io_submit_state_end(struct io_submit_state *state) blk_finish_plug(&state->plug); if (!list_empty(&state->req_list.list)) io_flush_state_reqs(state->ctx, state); + io_file_put(state, NULL); } /* * Start submission side cache. */ static void io_submit_state_start(struct io_submit_state *state, - struct io_ring_ctx *ctx) + struct io_ring_ctx *ctx, unsigned max_ios) { state->ctx = ctx; INIT_LIST_HEAD(&state->req_list.list); state->req_count = 0; + state->file = NULL; + state->ios_left = max_ios; #ifdef CONFIG_BLOCK state->plug_cb.callback = io_state_unplug; blk_start_plug(&state->plug); @@ -933,7 +1005,7 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) int i, ret = 0, submit = 0; if (to_submit > IO_PLUG_THRESHOLD) { - io_submit_state_start(&state, ctx); + io_submit_state_start(&state, ctx, to_submit); statep = &state; } -- 2.17.1 -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to majordomo@kvack.org. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
next prev parent reply other threads:[~2019-01-15 2:56 UTC|newest] Thread overview: 64+ messages / expand[flat|nested] mbox.gz Atom feed top 2019-01-15 2:55 (unknown), Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 01/16] fs: add an iopoll method to struct file_operations Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 02/16] block: wire up block device iopoll method Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 03/16] block: add bio_set_polled() helper Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 04/16] iomap: wire up the iopoll method Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 05/16] Add io_uring IO interface Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 16:51 ` Jonathan Corbet 2019-01-15 16:51 ` Jonathan Corbet 2019-01-15 16:55 ` Jens Axboe 2019-01-15 16:55 ` Jens Axboe 2019-01-15 17:26 ` Jens Axboe 2019-01-15 17:26 ` Jens Axboe 2019-01-16 10:41 ` Arnd Bergmann 2019-01-16 10:41 ` Arnd Bergmann 2019-01-16 11:00 ` Arnd Bergmann 2019-01-16 11:00 ` Arnd Bergmann 2019-01-16 15:12 ` Jens Axboe 2019-01-16 15:12 ` Jens Axboe 2019-01-16 15:16 ` Arnd Bergmann 2019-01-16 15:16 ` Arnd Bergmann 2019-01-16 15:25 ` Jens Axboe 2019-01-16 15:25 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 06/16] io_uring: add fsync support Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 07/16] io_uring: support for IO polling Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 08/16] io_uring: add submission side request cache Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 09/16] fs: add fget_many() and fput_many() Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` Jens Axboe [this message] 2019-01-15 2:55 ` [PATCH 10/16] io_uring: use fget/fput_many() for file references Jens Axboe 2019-01-15 2:55 ` [PATCH 11/16] io_uring: batch io_kiocb allocation Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 12/16] block: implement bio helper to add iter bvec pages to bio Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 13/16] io_uring: add support for pre-mapped user IO buffers Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-16 10:53 ` Arnd Bergmann 2019-01-16 15:14 ` Jens Axboe 2019-01-16 15:14 ` Jens Axboe 2019-01-16 15:32 ` Jens Axboe 2019-01-16 15:32 ` Jens Axboe 2019-01-16 15:41 ` Arnd Bergmann 2019-01-16 15:47 ` Jens Axboe 2019-01-16 15:47 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 14/16] io_uring: add submission polling Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 15/16] io_uring: add file registration Jens Axboe 2019-01-15 2:55 ` Jens Axboe 2019-01-16 10:45 ` Arnd Bergmann 2019-01-16 10:45 ` Arnd Bergmann 2019-01-16 15:15 ` Jens Axboe 2019-01-16 15:15 ` Jens Axboe 2019-01-15 2:55 ` [PATCH 16/16] io_uring: add io_uring_event cache hit information Jens Axboe 2019-01-15 2:55 ` Jens Axboe -- strict thread matches above, loose matches on Subject: below -- 2019-01-12 21:29 [PATCHSET v3] io_uring IO interface Jens Axboe 2019-01-12 21:30 ` [PATCH 10/16] io_uring: use fget/fput_many() for file references Jens Axboe 2019-01-12 21:30 ` Jens Axboe
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20190115025531.13985-11-axboe@kernel.dk \ --to=axboe@kernel.dk \ --cc=avi@scylladb.com \ --cc=hch@lst.de \ --cc=jmoyer@redhat.com \ --cc=linux-aio@kvack.org \ --cc=linux-arch@vger.kernel.org \ --cc=linux-block@vger.kernel.org \ --cc=linux-fsdevel@vger.kernel.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.