Re: [PATCH v12 rdma-next 4/8] RDMA/efa: Use the common mmap_xa helpers

[Jason Gunthorpe <jgg@ziepe.ca>]

On Wed, Oct 30, 2019 at 11:44:13AM +0200, Michal Kalderon wrote:

> @@ -116,6 +122,13 @@ struct efa_ah {
>  	u8 id[EFA_GID_SIZE];
>  };
>  
> +struct efa_user_mmap_entry {
> +	struct rdma_user_mmap_entry rdma_entry;
> +	u64 address;
> +	size_t length;
> +	u8 mmap_flag;
> +};

There is no reason to move this struct out of efa_verbs.c

length is redundant with rdma_entry.npages

>  static int qp_mmap_entries_setup(struct efa_qp *qp,
>  				 struct efa_dev *dev,
>  				 struct efa_ucontext *ucontext,
>  				 struct efa_com_create_qp_params *params,
>  				 struct efa_ibv_create_qp_resp *resp)
>  {
> -	/*
> -	 * Once an entry is inserted it might be mmapped, hence cannot be
> -	 * cleaned up until dealloc_ucontext.
> -	 */
> -	resp->sq_db_mmap_key =
> -		mmap_entry_insert(dev, ucontext, qp,
> -				  dev->db_bar_addr + resp->sq_db_offset,
> -				  PAGE_SIZE, EFA_MMAP_IO_NC);
> -	if (resp->sq_db_mmap_key == EFA_MMAP_INVALID)
> +	size_t length;
> +	u64 address;
> +
> +	address = dev->db_bar_addr + resp->sq_db_offset;
> +	qp->sq_db_mmap_entry =
> +		efa_user_mmap_entry_insert(&ucontext->ibucontext,
> +					   address,
> +					   PAGE_SIZE, EFA_MMAP_IO_NC,
> +					   &resp->sq_db_mmap_key);

I'm still confused how this is OK for the lifetime, 'sq_db_offset'
comes from the device, does the device prevent re-use of the same
db_offset until the ucontext is closed? If so that deserves a comment
in here.

>  static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
> -		      struct vm_area_struct *vma, u64 key, u64 length)
> +		      struct vm_area_struct *vma, u64 key, size_t length)
>  {
> -	struct efa_mmap_entry *entry;
> +	struct rdma_user_mmap_entry *rdma_entry;
> +	struct efa_user_mmap_entry *entry;
>  	unsigned long va;
> +	int err = 0;
>  	u64 pfn;
> -	int err;
>  
> -	entry = mmap_entry_get(dev, ucontext, key, length);
> -	if (!entry) {
> +	rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, key,
> +					      vma);
> +	if (!rdma_entry) {
>  		ibdev_dbg(&dev->ibdev, "key[%#llx] does not have valid entry\n",
>  			  key);
>  		return -EINVAL;
>  	}
> +	entry = to_emmap(rdma_entry);
> +	if (entry->length != length) {
> +		ibdev_dbg(&dev->ibdev,
> +			  "key[%#llx] does not have valid length[%#zx] expected[%#zx]\n",
> +			  key, length, entry->length);
> +		err = -EINVAL;
> +		goto out;
> +	}

Should be in common code

Same with the VM_SHARED (whichi s only needed for the vm_insert_page
flow)

Also driver should not be messing with VM_EXEC as that is known to
cause problems.

> @@ -1648,16 +1615,16 @@ int efa_mmap(struct ib_ucontext *ibucontext,
>  {
>  	struct efa_ucontext *ucontext = to_eucontext(ibucontext);
>  	struct efa_dev *dev = to_edev(ibucontext->device);
> -	u64 length = vma->vm_end - vma->vm_start;
> +	size_t length = vma->vm_end - vma->vm_start;
>  	u64 key = vma->vm_pgoff << PAGE_SHIFT;
>  
>  	ibdev_dbg(&dev->ibdev,
> -		  "start %#lx, end %#lx, length = %#llx, key = %#llx\n",
> +		  "start %#lx, end %#lx, length = %#zx, key = %#llx\n",
>  		  vma->vm_start, vma->vm_end, length, key);
>  
>  	if (length % PAGE_SIZE != 0 || !(vma->vm_flags & VM_SHARED)) {

vm_end - vm_start is always page aligned

Jason