[PATCH 5/5] io_uring: leverage completion cache for poll requests

[Hao Xu <haoxu@linux.alibaba.com>]

Leverage completion cache to handle completions of poll requests in a
batch.
Good thing is we save compl_nr - 1 completion_lock and
io_cqring_ev_posted.
Bad thing is compl_nr extra ifs in flush_completion.

Signed-off-by: Hao Xu <haoxu@linux.alibaba.com>
---
 fs/io_uring.c | 64 +++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 49 insertions(+), 15 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index b1d6c3a1d3cd..0f72cb0bf79a 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1099,6 +1099,8 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
 				 unsigned int issue_flags, u32 slot_index);
 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
 
+static bool io_complete_poll(struct io_kiocb *req);
+
 static struct kmem_cache *req_cachep;
 
 static const struct file_operations io_uring_fops;
@@ -2333,6 +2335,11 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 	for (i = 0; i < nr; i++) {
 		struct io_kiocb *req = state->compl_reqs[i];
 
+		if (req->opcode == IORING_OP_POLL_ADD) {
+			if (!io_complete_poll(req))
+				state->compl_reqs[i] = NULL;
+			continue;
+		}
 		__io_cqring_fill_event(ctx, req->user_data, req->result,
 					req->compl.cflags);
 	}
@@ -2344,7 +2351,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 	for (i = 0; i < nr; i++) {
 		struct io_kiocb *req = state->compl_reqs[i];
 
-		if (req_ref_put_and_test(req))
+		if (req && req_ref_put_and_test(req))
 			io_req_free_batch(&rb, req, &ctx->submit_state);
 	}
 
@@ -5360,6 +5367,23 @@ static inline void io_poll_complete(struct io_kiocb *req, __poll_t mask)
 	return;
 }
 
+static bool io_complete_poll(struct io_kiocb *req)
+{
+	bool done;
+
+	done = __io_poll_complete(req, req->result);
+	if (done) {
+		io_poll_remove_double(req);
+		hash_del(&req->hash_node);
+		req->poll.done = true;
+	} else {
+		req->result = 0;
+		add_wait_queue(req->poll.head, &req->poll.wait);
+	}
+
+	return done;
+}
+
 static void io_poll_task_func(struct io_kiocb *req, bool *locked)
 {
 	struct io_ring_ctx *ctx = req->ctx;
@@ -5367,18 +5391,10 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
 
 	if (io_poll_rewait(req, &req->poll)) {
 		spin_unlock(&ctx->completion_lock);
-	} else {
+	} else if (!*locked) {
 		bool done;
 
-		done = __io_poll_complete(req, req->result);
-		if (done) {
-			io_poll_remove_double(req);
-			hash_del(&req->hash_node);
-			req->poll.done = true;
-		} else {
-			req->result = 0;
-			add_wait_queue(req->poll.head, &req->poll.wait);
-		}
+		done = io_complete_poll(req);
 		io_commit_cqring(ctx);
 		spin_unlock(&ctx->completion_lock);
 		io_cqring_ev_posted(ctx);
@@ -5388,6 +5404,13 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
 			if (nxt)
 				io_req_task_submit(nxt, locked);
 		}
+	} else {
+		struct io_submit_state *state = &ctx->submit_state;
+
+		spin_unlock(&ctx->completion_lock);
+		state->compl_reqs[state->compl_nr++] = req;
+		if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
+			io_submit_flush_completions(ctx);
 	}
 }
 
@@ -5833,6 +5856,7 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_poll_table ipt;
 	__poll_t mask;
+	bool locked = current == req->task;
 
 	ipt.pt._qproc = io_poll_queue_proc;
 
@@ -5841,14 +5865,24 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
 
 	if (mask) { /* no async, we'd stolen it */
 		ipt.error = 0;
-		io_poll_complete(req, mask);
+		if (!locked)
+			io_poll_complete(req, mask);
 	}
 	spin_unlock(&ctx->completion_lock);
 
 	if (mask) {
-		io_cqring_ev_posted(ctx);
-		if (poll->events & EPOLLONESHOT)
-			io_put_req(req);
+		if (!locked) {
+			io_cqring_ev_posted(ctx);
+			if (poll->events & EPOLLONESHOT)
+				io_put_req(req);
+		} else {
+			struct io_submit_state *state = &ctx->submit_state;
+
+			req->result = mask;
+			state->compl_reqs[state->compl_nr++] = req;
+			if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
+				io_submit_flush_completions(ctx);
+		}
 	}
 	return ipt.error;
 }
-- 
2.24.4