From: Gal Pressman <galpress@amazon.com>
To: Michal Kalderon <michal.kalderon@marvell.com>
Cc: <mkalderon@marvell.com>, <aelior@marvell.com>, <jgg@ziepe.ca>,
<dledford@redhat.com>, <bmt@zurich.ibm.com>, <sleybo@amazon.com>,
<leon@kernel.org>, <linux-rdma@vger.kernel.org>,
Ariel Elior <ariel.elior@marvell.com>
Subject: Re: [PATCH v8 rdma-next 3/7] RDMA/efa: Use the common mmap_xa helpers
Date: Thu, 29 Aug 2019 17:20:53 +0300 [thread overview]
Message-ID: <c8781373-53ec-649e-1f1d-56cc17c229f9@amazon.com> (raw)
In-Reply-To: <20190827132846.9142-4-michal.kalderon@marvell.com>
On 27/08/2019 16:28, Michal Kalderon wrote:
> +static void efa_qp_user_mmap_entries_remove(struct efa_ucontext *ucontext,
> + struct efa_qp *qp)
> +{
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp->sq_db_mmap_key);
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> + qp->llq_desc_mmap_key);
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp->rq_mmap_key);
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp->rq_db_mmap_key);
Please remove the entries in reverse insertion order.
> +}
> +
> static int qp_mmap_entries_setup(struct efa_qp *qp,
> struct efa_dev *dev,
> struct efa_ucontext *ucontext,
> struct efa_com_create_qp_params *params,
> struct efa_ibv_create_qp_resp *resp)
> {
> - /*
> - * Once an entry is inserted it might be mmapped, hence cannot be
> - * cleaned up until dealloc_ucontext.
> - */
> - resp->sq_db_mmap_key =
> - mmap_entry_insert(dev, ucontext, qp,
> - dev->db_bar_addr + resp->sq_db_offset,
> - PAGE_SIZE, EFA_MMAP_IO_NC);
> - if (resp->sq_db_mmap_key == EFA_MMAP_INVALID)
> - return -ENOMEM;
> + u64 address;
> + u64 length;
> + int err;
> +
> + err = efa_user_mmap_entry_insert(&ucontext->ibucontext,
> + dev->db_bar_addr +
> + resp->sq_db_offset,
> + PAGE_SIZE, EFA_MMAP_IO_NC,
> + &qp->sq_db_mmap_key);
> + if (err)
> + return err;
>
> + resp->sq_db_mmap_key = qp->sq_db_mmap_key;
> resp->sq_db_offset &= ~PAGE_MASK;
>
> - resp->llq_desc_mmap_key =
> - mmap_entry_insert(dev, ucontext, qp,
> - dev->mem_bar_addr + resp->llq_desc_offset,
> - PAGE_ALIGN(params->sq_ring_size_in_bytes +
> - (resp->llq_desc_offset & ~PAGE_MASK)),
> - EFA_MMAP_IO_WC);
> - if (resp->llq_desc_mmap_key == EFA_MMAP_INVALID)
> - return -ENOMEM;
> + address = dev->mem_bar_addr + resp->llq_desc_offset;
> + length = PAGE_ALIGN(params->sq_ring_size_in_bytes +
> + (resp->llq_desc_offset & ~PAGE_MASK));
> +
> + err = efa_user_mmap_entry_insert(&ucontext->ibucontext,
> + address,
> + length,
> + EFA_MMAP_IO_WC,
> + &qp->llq_desc_mmap_key);
> + if (err)
> + goto err1;
>
> + resp->llq_desc_mmap_key = qp->llq_desc_mmap_key;
> resp->llq_desc_offset &= ~PAGE_MASK;
>
> if (qp->rq_size) {
> - resp->rq_db_mmap_key =
> - mmap_entry_insert(dev, ucontext, qp,
> - dev->db_bar_addr + resp->rq_db_offset,
> - PAGE_SIZE, EFA_MMAP_IO_NC);
> - if (resp->rq_db_mmap_key == EFA_MMAP_INVALID)
> - return -ENOMEM;
> + address = dev->db_bar_addr + resp->rq_db_offset;
>
> + err = efa_user_mmap_entry_insert(&ucontext->ibucontext,
> + address, PAGE_SIZE,
> + EFA_MMAP_IO_NC,
> + &qp->rq_db_mmap_key);
> + if (err)
> + goto err2;
> +
> + resp->rq_db_mmap_key = qp->rq_db_mmap_key;
> resp->rq_db_offset &= ~PAGE_MASK;
>
> - resp->rq_mmap_key =
> - mmap_entry_insert(dev, ucontext, qp,
> - virt_to_phys(qp->rq_cpu_addr),
> - qp->rq_size, EFA_MMAP_DMA_PAGE);
> - if (resp->rq_mmap_key == EFA_MMAP_INVALID)
> - return -ENOMEM;
> + address = virt_to_phys(qp->rq_cpu_addr);
> + err = efa_user_mmap_entry_insert(&ucontext->ibucontext,
> + address, qp->rq_size,
> + EFA_MMAP_DMA_PAGE,
> + &qp->rq_mmap_key);
> + if (err)
> + goto err3;
>
> + resp->rq_mmap_key = qp->rq_mmap_key;
> resp->rq_mmap_size = qp->rq_size;
> }
>
> return 0;
> +
> +err3:
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp->rq_db_mmap_key);
> +
> +err2:
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> + qp->llq_desc_mmap_key);
> +
> +err1:
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext, qp->sq_db_mmap_key);
I prefer meaningful goto labels, e.g err_remove_sq_db instead of err1.
> +
> + /* If any error occurred, we init the keys back to invalid */
> + efa_qp_init_keys(qp);
> +
> + return err;
> }
>
> static int efa_qp_validate_cap(struct efa_dev *dev,
> @@ -634,7 +594,6 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
> struct efa_dev *dev = to_edev(ibpd->device);
> struct efa_ibv_create_qp_resp resp = {};
> struct efa_ibv_create_qp cmd = {};
> - bool rq_entry_inserted = false;
> struct efa_ucontext *ucontext;
> struct efa_qp *qp;
> int err;
> @@ -687,6 +646,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
> goto err_out;
> }
>
> + efa_qp_init_keys(qp);
> create_qp_params.uarn = ucontext->uarn;
> create_qp_params.pd = to_epd(ibpd)->pdn;
>
> @@ -742,7 +702,6 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
> if (err)
> goto err_destroy_qp;
>
> - rq_entry_inserted = true;
> qp->qp_handle = create_qp_resp.qp_handle;
> qp->ibqp.qp_num = create_qp_resp.qp_num;
> qp->ibqp.qp_type = init_attr->qp_type;
> @@ -759,7 +718,7 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
> ibdev_dbg(&dev->ibdev,
> "Failed to copy udata for qp[%u]\n",
> create_qp_resp.qp_num);
> - goto err_destroy_qp;
> + goto err_remove_mmap_entries;
> }
> }
>
> @@ -767,15 +726,17 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
>
> return &qp->ibqp;
>
> +err_remove_mmap_entries:
> + efa_qp_user_mmap_entries_remove(ucontext, qp);
> err_destroy_qp:
> efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);
> err_free_mapped:
> - if (qp->rq_size) {
> + if (qp->rq_dma_addr)
What's the difference?
> dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size,
> DMA_TO_DEVICE);
> - if (!rq_entry_inserted)
> - free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
> - }
> +
> + if (qp->rq_mmap_key == RDMA_USER_MMAP_INVALID)
> + free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
This should be inside the previous if statement, otherwise it might try to free
pages that weren't allocated.
> err_free_qp:
> kfree(qp);
> err_out:
> @@ -887,6 +848,7 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx)
>
> void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
> {
> + struct efa_ucontext *ucontext;
Reverse xmas tree.
> struct efa_dev *dev = to_edev(ibcq->device);
> struct efa_cq *cq = to_ecq(ibcq);
>
> @@ -894,20 +856,33 @@ void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
> "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",
> cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
>
> + ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext,
> + ibucontext);
> efa_destroy_cq_idx(dev, cq->cq_idx);
> dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
> DMA_FROM_DEVICE);
> + rdma_user_mmap_entry_remove(&ucontext->ibucontext,
> + cq->mmap_key);
Entry removal should be first.
> }
>
> static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
> struct vm_area_struct *vma, u64 key, u64 length)
> {
> - struct efa_mmap_entry *entry;
> + struct rdma_user_mmap_entry *rdma_entry;
> + struct efa_user_mmap_entry *entry;
> unsigned long va;
> u64 pfn;
> int err;
>
> - entry = mmap_entry_get(dev, ucontext, key, length);
> - if (!entry) {
> + rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, key,
> + length, vma);
> + if (!rdma_entry) {
> ibdev_dbg(&dev->ibdev, "key[%#llx] does not have valid entry\n",
> key);
> return -EINVAL;
> }
> + entry = to_emmap(rdma_entry);
> + if (entry->length != length) {
> + ibdev_dbg(&dev->ibdev,
> + "key[%#llx] does not have valid length[%#llx] expected[%#llx]\n",
> + key, length, entry->length);
Need to put the entry.
> + return -EINVAL;
> + }
>
> ibdev_dbg(&dev->ibdev,
> "Mapping address[%#llx], length[%#llx], mmap_flag[%d]\n",
> - entry->address, length, entry->mmap_flag);
> + entry->address, entry->length, entry->mmap_flag);
>
> pfn = entry->address >> PAGE_SHIFT;
> switch (entry->mmap_flag) {
> @@ -1637,6 +1630,10 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
> &dev->ibdev,
> "Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n",
> entry->address, length, entry->mmap_flag, err);
> +
> + rdma_user_mmap_entry_put(&ucontext->ibucontext,
> + rdma_entry);
> +
> return err;
> }
>
next prev parent reply other threads:[~2019-08-29 14:21 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-08-27 13:28 [PATCH v8 rdma-next 0/7] RDMA/qedr: Use the doorbell overflow recovery mechanism for RDMA Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 1/7] RDMA/core: Move core content from ib_uverbs to ib_core Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 2/7] RDMA/core: Create mmap database and cookie helper functions Michal Kalderon
2019-08-29 11:35 ` Gal Pressman
2019-08-29 12:40 ` Jason Gunthorpe
2019-08-30 6:10 ` [EXT] " Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 3/7] RDMA/efa: Use the common mmap_xa helpers Michal Kalderon
2019-08-29 14:20 ` Gal Pressman [this message]
2019-08-30 6:15 ` Michal Kalderon
2019-09-01 6:56 ` Gal Pressman
2019-09-02 7:47 ` Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 4/7] RDMA/siw: " Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 5/7] RDMA/qedr: Use the common mmap API Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 6/7] RDMA/qedr: Add doorbell overflow recovery support Michal Kalderon
2019-08-27 13:28 ` [PATCH v8 rdma-next 7/7] RDMA/qedr: Add iWARP doorbell " Michal Kalderon
2019-08-28 14:41 ` [PATCH v8 rdma-next 0/7] RDMA/qedr: Use the doorbell overflow recovery mechanism for RDMA Gal Pressman
2019-08-30 12:07 ` [PATCH v8 rdma-next 4/7] RDMA/siw: Use the common mmap_xa helpers Bernard Metzler
2019-08-30 12:42 ` [EXT] " Michal Kalderon
2019-08-30 13:19 ` Bernard Metzler
2019-09-02 8:00 ` Michal Kalderon
2019-09-02 10:55 ` Bernard Metzler
2019-09-02 11:21 ` Michal Kalderon
2019-09-02 11:18 ` Bernard Metzler
2019-09-02 13:16 ` Michal Kalderon
2019-09-02 14:17 ` Bernard Metzler
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=c8781373-53ec-649e-1f1d-56cc17c229f9@amazon.com \
--to=galpress@amazon.com \
--cc=aelior@marvell.com \
--cc=ariel.elior@marvell.com \
--cc=bmt@zurich.ibm.com \
--cc=dledford@redhat.com \
--cc=jgg@ziepe.ca \
--cc=leon@kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=michal.kalderon@marvell.com \
--cc=mkalderon@marvell.com \
--cc=sleybo@amazon.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).