On 17/12/2019 02:22, Pavel Begunkov wrote:
> Move io_queue_link_head() to links handling code in io_submit_sqe(),
> so it wouldn't need extra checks and would have better data locality.
> 
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---
>  fs/io_uring.c | 32 ++++++++++++++------------------
>  1 file changed, 14 insertions(+), 18 deletions(-)
> 
> diff --git a/fs/io_uring.c b/fs/io_uring.c
> index bac9e711e38d..a880ed1409cb 100644
> --- a/fs/io_uring.c
> +++ b/fs/io_uring.c
> @@ -3373,13 +3373,15 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
>  			  struct io_kiocb **link)
>  {
>  	struct io_ring_ctx *ctx = req->ctx;
> +	unsigned int sqe_flags;
>  	int ret;
>  
> +	sqe_flags = READ_ONCE(req->sqe->flags);
>  	req->user_data = READ_ONCE(req->sqe->user_data);
>  	trace_io_uring_submit_sqe(ctx, req->user_data, true, req->in_async);
>  
>  	/* enforce forwards compatibility on users */
> -	if (unlikely(req->sqe->flags & ~SQE_VALID_FLAGS)) {
> +	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
>  		ret = -EINVAL;
>  		goto err_req;
>  	}
> @@ -3402,10 +3404,10 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
>  	if (*link) {
>  		struct io_kiocb *head = *link;
>  
> -		if (req->sqe->flags & IOSQE_IO_DRAIN)
> +		if (sqe_flags & IOSQE_IO_DRAIN)
>  			head->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN;
>  
> -		if (req->sqe->flags & IOSQE_IO_HARDLINK)
> +		if (sqe_flags & IOSQE_IO_HARDLINK)
>  			req->flags |= REQ_F_HARDLINK;
>  
>  		if (io_alloc_async_ctx(req)) {
> @@ -3421,9 +3423,15 @@ static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
>  		}
>  		trace_io_uring_link(ctx, req, head);
>  		list_add_tail(&req->link_list, &head->link_list);
> -	} else if (req->sqe->flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
> +
> +		/* last request of a link, enqueue the link */
> +		if (!(sqe_flags & IOSQE_IO_LINK)) {

This looks suspicious (as well as in the current revision). Returning back
to my questions a few days ago can sqe->flags have IOSQE_IO_HARDLINK, but not
IOSQE_IO_LINK? I don't find any check.

In other words, should it be as follows?
!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))

> +			io_queue_link_head(head);
> +			*link = NULL;
> +		}
> +	} else if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
>  		req->flags |= REQ_F_LINK;
> -		if (req->sqe->flags & IOSQE_IO_HARDLINK)
> +		if (sqe_flags & IOSQE_IO_HARDLINK)
>  			req->flags |= REQ_F_HARDLINK;
>  
>  		INIT_LIST_HEAD(&req->link_list);
> @@ -3540,10 +3548,8 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  	}
>  
>  	for (i = 0; i < nr; i++) {
> -		struct io_kiocb *req;
> -		unsigned int sqe_flags;
> +		struct io_kiocb *req = io_get_req(ctx, statep);
>  
> -		req = io_get_req(ctx, statep);
>  		if (unlikely(!req)) {
>  			if (!submitted)
>  				submitted = -EAGAIN;
> @@ -3563,8 +3569,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  		}
>  
>  		submitted++;
> -		sqe_flags = req->sqe->flags;
> -
>  		req->ring_file = ring_file;
>  		req->ring_fd = ring_fd;
>  		req->has_user = *mm != NULL;
> @@ -3572,14 +3576,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  		req->needs_fixed_file = async;
>  		if (!io_submit_sqe(req, statep, &link))
>  			break;
> -		/*
> -		 * If previous wasn't linked and we have a linked command,
> -		 * that's the end of the chain. Submit the previous link.
> -		 */
> -		if (!(sqe_flags & IOSQE_IO_LINK) && link) {
> -			io_queue_link_head(link);
> -			link = NULL;
> -		}
>  	}
>  
>  	if (link)
> 

-- 
Pavel Begunkov