Re: [RESEND PATCH 4/4] aio: use iovec array rather than the single one

Re: [RESEND PATCH 4/4] aio: use iovec array rather than the single one

[Jeff Moyer]

Gu Zheng <guz.fnst@cn.fujitsu.com> writes:

> Previously, we only offer a single iovec to handle all the read/write cases, so
> the PREADV/PWRITEV request always need to alloc more iovec buffer when copying
> user vectors.
> If we use a tmp iovec array rather than the single one, some small PREADV/PWRITEV
> workloads(vector size small than the tmp buffer) will not need to alloc more
> iovec buffer when copying user vectors.

Hi, Gu,

This still doesn't explain why you decided to look into this.  Did you
notice a performance issue in this path?  Do you have benchmarks that
show some speedup due to this change?

Thanks,
Jeff

>
> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
>  fs/aio.c |   10 +++++-----
>  1 files changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/fs/aio.c b/fs/aio.c
> index f1fede2..df3491a 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -1267,12 +1267,12 @@ static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
>  	if (compat)
>  		ret = compat_rw_copy_check_uvector(rw,
>  				(struct compat_iovec __user *)buf,
> -				*nr_segs, 1, *iovec, iovec);
> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>  	else
>  #endif
>  		ret = rw_copy_check_uvector(rw,
>  				(struct iovec __user *)buf,
> -				*nr_segs, 1, *iovec, iovec);
> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>  	if (ret < 0)
>  		return ret;
>  
> @@ -1309,7 +1309,7 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
>  	fmode_t mode;
>  	aio_rw_op *rw_op;
>  	rw_iter_op *iter_op;
> -	struct iovec inline_vec, *iovec = &inline_vec;
> +	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
>  	struct iov_iter iter;
>  
>  	switch (opcode) {
> @@ -1344,7 +1344,7 @@ rw_common:
>  		if (!ret)
>  			ret = rw_verify_area(rw, file, &req->ki_pos, req->ki_nbytes);
>  		if (ret < 0) {
> -			if (iovec != &inline_vec)
> +			if (iovec != inline_vecs)
>  				kfree(iovec);
>  			return ret;
>  		}
> @@ -1391,7 +1391,7 @@ rw_common:
>  		return -EINVAL;
>  	}
>  
> -	if (iovec != &inline_vec)
> +	if (iovec != inline_vecs)
>  		kfree(iovec);
>  
>  	if (ret != -EIOCBQUEUED) {

Re: [RESEND PATCH 4/4] aio: use iovec array rather than the single one

[Gu Zheng]

Hi Jeff,
On 07/23/2014 09:25 PM, Jeff Moyer wrote:

> Gu Zheng <guz.fnst@cn.fujitsu.com> writes:
> 
>> Previously, we only offer a single iovec to handle all the read/write cases, so
>> the PREADV/PWRITEV request always need to alloc more iovec buffer when copying
>> user vectors.
>> If we use a tmp iovec array rather than the single one, some small PREADV/PWRITEV
>> workloads(vector size small than the tmp buffer) will not need to alloc more
>> iovec buffer when copying user vectors.
> 
> Hi, Gu,
> 
> This still doesn't explain why you decided to look into this.

The comment is clear, just want to avoid some needless memory allocation in
the io submit path.

> Did you
> notice a performance issue in this path?  Do you have benchmarks that
> show some speedup due to this change?

Just some common tests based on fio, it gains a slight improvement(~3%) when the
iodepth in [5,6,7] than before. I did not paste these info here, because I think
other guys(especially the guys have production environment) who are interested
in this can give us more meaningful feedback.

Thanks,
Gu

> 
> Thanks,
> Jeff
> 
>>
>> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
>> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
>> ---
>>  fs/aio.c |   10 +++++-----
>>  1 files changed, 5 insertions(+), 5 deletions(-)
>>
>> diff --git a/fs/aio.c b/fs/aio.c
>> index f1fede2..df3491a 100644
>> --- a/fs/aio.c
>> +++ b/fs/aio.c
>> @@ -1267,12 +1267,12 @@ static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
>>  	if (compat)
>>  		ret = compat_rw_copy_check_uvector(rw,
>>  				(struct compat_iovec __user *)buf,
>> -				*nr_segs, 1, *iovec, iovec);
>> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>>  	else
>>  #endif
>>  		ret = rw_copy_check_uvector(rw,
>>  				(struct iovec __user *)buf,
>> -				*nr_segs, 1, *iovec, iovec);
>> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>>  	if (ret < 0)
>>  		return ret;
>>  
>> @@ -1309,7 +1309,7 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
>>  	fmode_t mode;
>>  	aio_rw_op *rw_op;
>>  	rw_iter_op *iter_op;
>> -	struct iovec inline_vec, *iovec = &inline_vec;
>> +	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
>>  	struct iov_iter iter;
>>  
>>  	switch (opcode) {
>> @@ -1344,7 +1344,7 @@ rw_common:
>>  		if (!ret)
>>  			ret = rw_verify_area(rw, file, &req->ki_pos, req->ki_nbytes);
>>  		if (ret < 0) {
>> -			if (iovec != &inline_vec)
>> +			if (iovec != inline_vecs)
>>  				kfree(iovec);
>>  			return ret;
>>  		}
>> @@ -1391,7 +1391,7 @@ rw_common:
>>  		return -EINVAL;
>>  	}
>>  
>> -	if (iovec != &inline_vec)
>> +	if (iovec != inline_vecs)
>>  		kfree(iovec);
>>  
>>  	if (ret != -EIOCBQUEUED) {
> .
> 

Re: [RESEND PATCH 1/4] aio: remove the needless registration of ring file's private_data

[Benjamin LaHaise]

On Wed, Jul 23, 2014 at 06:03:51PM +0800, Gu Zheng wrote:
> Remove the registration of ring file's private_data, we do not use
> it.

Thanks Gu -- applied to my aio-next tree at 
git://git.kvack.org/~bcrl/aio-next.git .

		-ben

> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
>  fs/aio.c |    1 -
>  1 files changed, 0 insertions(+), 1 deletions(-)
> 
> diff --git a/fs/aio.c b/fs/aio.c
> index 1c9c5f0..94546ab 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -192,7 +192,6 @@ static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
>  	}
>  
>  	file->f_flags = O_RDWR;
> -	file->private_data = ctx;
>  	return file;
>  }
>  
> -- 
> 1.7.7

-- 
"Thought is the essence of where you are now."

Re: [RESEND PATCH 2/4] aio: use the macro rather than the inline magic number

[Benjamin LaHaise]

On Wed, Jul 23, 2014 at 06:03:52PM +0800, Gu Zheng wrote:
> Replace the inline magic number with the ready-made macro(AIO_RING_MAGIC),
> just clean up.

Thanks Gu -- applied to my aio-next tree at   
git://git.kvack.org/~bcrl/aio-next.git .

		-ben

> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
>  fs/aio.c |    2 +-
>  1 files changed, 1 insertions(+), 1 deletions(-)
> 
> diff --git a/fs/aio.c b/fs/aio.c
> index 94546ab..e0f838d 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -201,7 +201,7 @@ static struct dentry *aio_mount(struct file_system_type *fs_type,
>  	static const struct dentry_operations ops = {
>  		.d_dname	= simple_dname,
>  	};
> -	return mount_pseudo(fs_type, "aio:", NULL, &ops, 0xa10a10a1);
> +	return mount_pseudo(fs_type, "aio:", NULL, &ops, AIO_RING_MAGIC);
>  }
>  
>  /* aio_setup
> -- 
> 1.7.7

-- 
"Thought is the essence of where you are now."

Re: [RESEND PATCH 3/4] aio: fix some comments

[Benjamin LaHaise]

On Wed, Jul 23, 2014 at 06:03:53PM +0800, Gu Zheng wrote:
> The function comments of aio_run_iocb and aio_read_events are out of date, so
> fix them here.

Thanks Gu -- applied to my aio-next tree at   
git://git.kvack.org/~bcrl/aio-next.git .

		-ben

> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
>  fs/aio.c |    7 +++----
>  1 files changed, 3 insertions(+), 4 deletions(-)
> 
> diff --git a/fs/aio.c b/fs/aio.c
> index e0f838d..f1fede2 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -1044,7 +1044,7 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
>  }
>  EXPORT_SYMBOL(aio_complete);
>  
> -/* aio_read_events
> +/* aio_read_events_ring
>   *	Pull an event off of the ioctx's event ring.  Returns the number of
>   *	events fetched
>   */
> @@ -1296,9 +1296,8 @@ static ssize_t aio_setup_single_vector(struct kiocb *kiocb,
>  }
>  
>  /*
> - * aio_setup_iocb:
> - *	Performs the initial checks and aio retry method
> - *	setup for the kiocb at the time of io submission.
> + * aio_run_iocb:
> + *	Performs the initial checks and io submission.
>   */
>  static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
>  			    char __user *buf, bool compat)
> -- 
> 1.7.7

-- 
"Thought is the essence of where you are now."

Re: [RESEND PATCH 4/4] aio: use iovec array rather than the single one

[Benjamin LaHaise]

On Wed, Jul 23, 2014 at 06:03:54PM +0800, Gu Zheng wrote:
> Previously, we only offer a single iovec to handle all the read/write cases, so
> the PREADV/PWRITEV request always need to alloc more iovec buffer when copying
> user vectors.
> If we use a tmp iovec array rather than the single one, some small PREADV/PWRITEV
> workloads(vector size small than the tmp buffer) will not need to alloc more
> iovec buffer when copying user vectors.

Thanks Gu -- applied to my aio-next tree at   
git://git.kvack.org/~bcrl/aio-next.git .

		-ben

> Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
> Signed-off-by: Gu Zheng <guz.fnst@cn.fujitsu.com>
> ---
>  fs/aio.c |   10 +++++-----
>  1 files changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/fs/aio.c b/fs/aio.c
> index f1fede2..df3491a 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -1267,12 +1267,12 @@ static ssize_t aio_setup_vectored_rw(struct kiocb *kiocb,
>  	if (compat)
>  		ret = compat_rw_copy_check_uvector(rw,
>  				(struct compat_iovec __user *)buf,
> -				*nr_segs, 1, *iovec, iovec);
> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>  	else
>  #endif
>  		ret = rw_copy_check_uvector(rw,
>  				(struct iovec __user *)buf,
> -				*nr_segs, 1, *iovec, iovec);
> +				*nr_segs, UIO_FASTIOV, *iovec, iovec);
>  	if (ret < 0)
>  		return ret;
>  
> @@ -1309,7 +1309,7 @@ static ssize_t aio_run_iocb(struct kiocb *req, unsigned opcode,
>  	fmode_t mode;
>  	aio_rw_op *rw_op;
>  	rw_iter_op *iter_op;
> -	struct iovec inline_vec, *iovec = &inline_vec;
> +	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
>  	struct iov_iter iter;
>  
>  	switch (opcode) {
> @@ -1344,7 +1344,7 @@ rw_common:
>  		if (!ret)
>  			ret = rw_verify_area(rw, file, &req->ki_pos, req->ki_nbytes);
>  		if (ret < 0) {
> -			if (iovec != &inline_vec)
> +			if (iovec != inline_vecs)
>  				kfree(iovec);
>  			return ret;
>  		}
> @@ -1391,7 +1391,7 @@ rw_common:
>  		return -EINVAL;
>  	}
>  
> -	if (iovec != &inline_vec)
> +	if (iovec != inline_vecs)
>  		kfree(iovec);
>  
>  	if (ret != -EIOCBQUEUED) {
> -- 
> 1.7.7

-- 
"Thought is the essence of where you are now."