RE: [PATCH v8 rdma-next 3/7] RDMA/efa: Use the common mmap_xa helpers

[Michal Kalderon <mkalderon@marvell.com>]

> From: Gal Pressman <galpress@amazon.com>
> Sent: Sunday, September 1, 2019 9:57 AM
> 
> On 30/08/2019 9:15, Michal Kalderon wrote:
> >> From: Gal Pressman <galpress@amazon.com>
> >> Sent: Thursday, August 29, 2019 5:21 PM
> >>
> >> On 27/08/2019 16:28, Michal Kalderon wrote:
> >>> +static void efa_qp_user_mmap_entries_remove(struct efa_ucontext
> >> *ucontext,
> >>> +					    struct efa_qp *qp)
> >>> +{
> >>> +	rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp-
> >>> sq_db_mmap_key);
> >>> +	rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> >>> +				    qp->llq_desc_mmap_key);
> >>> +	rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp-
> >>> rq_mmap_key);
> >>> +	rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> >>> +qp->rq_db_mmap_key);
> >>
> >> Please remove the entries in reverse insertion order.
> > I don't mind fixing, but why ?
> 
> So the flows will be symmetric.
> 
> >>
> >>> +}
> >>> +
> >>> @@ -767,15 +726,17 @@ struct ib_qp *efa_create_qp(struct ib_pd
> >>> *ibpd,
> >>>
> >>>  	return &qp->ibqp;
> >>>
> >>> +err_remove_mmap_entries:
> >>> +	efa_qp_user_mmap_entries_remove(ucontext, qp);
> >>>  err_destroy_qp:
> >>>  	efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);
> >>>  err_free_mapped:
> >>> -	if (qp->rq_size) {
> >>> +	if (qp->rq_dma_addr)
> >>
> >> What's the difference?
> > Seemed a better query since it now only covers the rq_dma_addr
> unmapping.
> >
> >>
> >>>  		dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr,
> >> qp->rq_size,
> >>>  				 DMA_TO_DEVICE);
> >>> -		if (!rq_entry_inserted)
> >>> -			free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
> >>> -	}
> >>> +
> >>> +	if (qp->rq_mmap_key == RDMA_USER_MMAP_INVALID)
> >>> +		free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
> >>
> >> This should be inside the previous if statement, otherwise it might
> >> try to free pages that weren't allocated.
> > If they weren't allocated the key will be INVALID and they won't be freed.
> 
> If the key is INVALID you call free_pages_exact, but rq_cpu_addr might have
> never been allocated (if RQ is of size zero).
Right, thanks
> 
> >
> >>
> >>>  err_free_qp:
> >>>  	kfree(qp);
> >>>  err_out:
> >>> @@ -887,6 +848,7 @@ static int efa_destroy_cq_idx(struct efa_dev
> >>> *dev, int cq_idx)
> >>>
> >>>  void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)  {
> >>> +	struct efa_ucontext *ucontext;
> >>
> >> Reverse xmas tree.
> > ok
> >>
> >>>  	struct efa_dev *dev = to_edev(ibcq->device);
> >>>  	struct efa_cq *cq = to_ecq(ibcq);
> >>>
> >>> @@ -894,20 +856,33 @@ void efa_destroy_cq(struct ib_cq *ibcq, struct
> >> ib_udata *udata)
> >>>  		  "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",
> >>>  		  cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
> >>>
> >>> +	ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext,
> >>> +					     ibucontext);
> >>>  	efa_destroy_cq_idx(dev, cq->cq_idx);
> >>>  	dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
> >>>  			 DMA_FROM_DEVICE);
> >>> +	rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> >>> +				    cq->mmap_key);
> >>
> >> Entry removal should be first.
> > Why ? removing can lead to freeing, why would we want that before
> unmapping ?
> 
> Makes sense, thanks.
> 
> >>
> >>>  }
> >>>