[PATCH for-next 1/1] io_uring: optinise io_uring_task layout

[PATCH for-next 1/1] io_uring: optinise io_uring_task layout

[Pavel Begunkov]

task_work bits of io_uring_task are split into two cache lines causing
extra cache bouncing, place them into a separate cache line. Also move
the most used submission path fields closer together, so there are hot.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/tctx.h | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/io_uring/tctx.h b/io_uring/tctx.h
index dde82ce4d8e2..dead0ed00429 100644
--- a/io_uring/tctx.h
+++ b/io_uring/tctx.h
@@ -7,22 +7,24 @@
 
 struct io_uring_task {
 	/* submission side */
-	int			cached_refs;
-	struct xarray		xa;
-	struct wait_queue_head	wait;
-	const struct io_ring_ctx *last;
-	struct io_wq		*io_wq;
-	struct percpu_counter	inflight;
-	atomic_t		inflight_tracked;
-	atomic_t		in_idle;
-
-	spinlock_t		task_lock;
-	struct io_wq_work_list	task_list;
-	struct io_wq_work_list	prio_task_list;
-	struct callback_head	task_work;
-	bool			task_running;
-
-	struct file		*registered_rings[IO_RINGFD_REG_MAX];
+	int				cached_refs;
+	const struct io_ring_ctx 	*last;
+	struct io_wq			*io_wq;
+	struct file			*registered_rings[IO_RINGFD_REG_MAX];
+
+	struct xarray			xa;
+	struct wait_queue_head		wait;
+	atomic_t			in_idle;
+	atomic_t			inflight_tracked;
+	struct percpu_counter		inflight;
+
+	struct { /* task_work */
+		spinlock_t		task_lock;
+		bool			task_running;
+		struct io_wq_work_list	task_list;
+		struct io_wq_work_list	prio_task_list;
+		struct callback_head	task_work;
+	} ____cacheline_aligned_in_smp;
 };
 
 struct io_tctx_node {
-- 
2.36.1

Re: [PATCH for-next 1/1] io_uring: optinise io_uring_task layout

[Jens Axboe]

On 6/20/22 8:27 AM, Pavel Begunkov wrote:
> task_work bits of io_uring_task are split into two cache lines causing
> extra cache bouncing, place them into a separate cache line. Also move
> the most used submission path fields closer together, so there are hot.

Thanks, applied. The current layout is really stupid...

-- 
Jens Axboe