linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
* [PATCH] RDMA/rw: switch to dma_map_sgtable()
@ 2021-10-01 21:32 Logan Gunthorpe
  2021-10-05 18:22 ` Jason Gunthorpe
  0 siblings, 1 reply; 2+ messages in thread
From: Logan Gunthorpe @ 2021-10-01 21:32 UTC (permalink / raw)
  To: Doug Ledford, Jason Gunthorpe, linux-rdma, linux-kernel
  Cc: Zheng Yongjun, Wenpeng Liang, Leon Romanovsky, Weihang Li,
	Mark Bloch, Logan Gunthorpe

There are a couple of subtle error path bugs related to mapping the
sgls:

- In rdma_rw_ctx_init(), dma_unmap would be called with an sg that
  could have been incremented from the original call, as well as an
  nents that was not the original number of nents called when mapped.
- Similarly in rdma_rw_ctx_signature_init, both sg and prot_sg were
  unmapped with the incorrect number of nents.

To fix this, switch to the sgtable interface for mapping which
conveniently stores the original nents for unmapping. This will get
cleaned up further once the dma mapping interface supports P2PDMA and
pci_p2pdma_map_sg() can be removed. At that point the sgtable interface
will be preferred as it offers better error reporting for P2PDMA pages.

Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---

This patch was extracted from my P2PDMA patchset per Jason's request[1]
seeing it fixes some bugs.

[1] https://lore.kernel.org/all/20210928194325.GS3544071@ziepe.ca/

 drivers/infiniband/core/rw.c | 66 ++++++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 25 deletions(-)

diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index 5221cce65675..5a3bd41b331c 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -282,15 +282,22 @@ static void rdma_rw_unmap_sg(struct ib_device *dev, struct scatterlist *sg,
 		ib_dma_unmap_sg(dev, sg, sg_cnt, dir);
 }

-static int rdma_rw_map_sg(struct ib_device *dev, struct scatterlist *sg,
-			  u32 sg_cnt, enum dma_data_direction dir)
+static int rdma_rw_map_sgtable(struct ib_device *dev, struct sg_table *sgt,
+			       enum dma_data_direction dir)
 {
-	if (is_pci_p2pdma_page(sg_page(sg))) {
+	int nents;
+
+	if (is_pci_p2pdma_page(sg_page(sgt->sgl))) {
 		if (WARN_ON_ONCE(ib_uses_virt_dma(dev)))
 			return 0;
-		return pci_p2pdma_map_sg(dev->dma_device, sg, sg_cnt, dir);
+		nents = pci_p2pdma_map_sg(dev->dma_device, sgt->sgl,
+					  sgt->orig_nents, dir);
+		if (!nents)
+			return -EIO;
+		sgt->nents = nents;
+		return 0;
 	}
-	return ib_dma_map_sg(dev, sg, sg_cnt, dir);
+	return ib_dma_map_sgtable_attrs(dev, sgt, dir, 0);
 }

 /**
@@ -313,12 +320,16 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num,
 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
 {
 	struct ib_device *dev = qp->pd->device;
+	struct sg_table sgt = {
+		.sgl = sg,
+		.orig_nents = sg_cnt,
+	};
 	int ret;

-	ret = rdma_rw_map_sg(dev, sg, sg_cnt, dir);
-	if (!ret)
-		return -ENOMEM;
-	sg_cnt = ret;
+	ret = rdma_rw_map_sgtable(dev, &sgt, dir);
+	if (ret)
+		return ret;
+	sg_cnt = sgt.nents;

 	/*
 	 * Skip to the S/G entry that sg_offset falls into:
@@ -354,7 +365,7 @@ int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num,
 	return ret;

 out_unmap_sg:
-	rdma_rw_unmap_sg(dev, sg, sg_cnt, dir);
+	rdma_rw_unmap_sg(dev, sgt.sgl, sgt.orig_nents, dir);
 	return ret;
 }
 EXPORT_SYMBOL(rdma_rw_ctx_init);
@@ -385,6 +396,14 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 	struct ib_device *dev = qp->pd->device;
 	u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device,
 						    qp->integrity_en);
+	struct sg_table sgt = {
+		.sgl = sg,
+		.orig_nents = sg_cnt,
+	};
+	struct sg_table prot_sgt = {
+		.sgl = prot_sg,
+		.orig_nents = prot_sg_cnt,
+	};
 	struct ib_rdma_wr *rdma_wr;
 	int count = 0, ret;

@@ -394,18 +413,14 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		return -EINVAL;
 	}

-	ret = rdma_rw_map_sg(dev, sg, sg_cnt, dir);
-	if (!ret)
-		return -ENOMEM;
-	sg_cnt = ret;
+	ret = rdma_rw_map_sgtable(dev, &sgt, dir);
+	if (ret)
+		return ret;

 	if (prot_sg_cnt) {
-		ret = rdma_rw_map_sg(dev, prot_sg, prot_sg_cnt, dir);
-		if (!ret) {
-			ret = -ENOMEM;
+		ret = rdma_rw_map_sgtable(dev, &prot_sgt, dir);
+		if (ret)
 			goto out_unmap_sg;
-		}
-		prot_sg_cnt = ret;
 	}

 	ctx->type = RDMA_RW_SIG_MR;
@@ -426,10 +441,11 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,

 	memcpy(ctx->reg->mr->sig_attrs, sig_attrs, sizeof(struct ib_sig_attrs));

-	ret = ib_map_mr_sg_pi(ctx->reg->mr, sg, sg_cnt, NULL, prot_sg,
-			      prot_sg_cnt, NULL, SZ_4K);
+	ret = ib_map_mr_sg_pi(ctx->reg->mr, sg, sgt.nents, NULL, prot_sg,
+			      prot_sgt.nents, NULL, SZ_4K);
 	if (unlikely(ret)) {
-		pr_err("failed to map PI sg (%u)\n", sg_cnt + prot_sg_cnt);
+		pr_err("failed to map PI sg (%u)\n",
+		       sgt.nents + prot_sgt.nents);
 		goto out_destroy_sig_mr;
 	}

@@ -468,10 +484,10 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 out_free_ctx:
 	kfree(ctx->reg);
 out_unmap_prot_sg:
-	if (prot_sg_cnt)
-		rdma_rw_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
+	if (prot_sgt.nents)
+		rdma_rw_unmap_sg(dev, prot_sgt.sgl, prot_sgt.orig_nents, dir);
 out_unmap_sg:
-	rdma_rw_unmap_sg(dev, sg, sg_cnt, dir);
+	rdma_rw_unmap_sg(dev, sgt.sgl, sgt.orig_nents, dir);
 	return ret;
 }
 EXPORT_SYMBOL(rdma_rw_ctx_signature_init);

base-commit: 450f4f6aa1a369cc3ffadc1c7e27dfab3e90199f
--
2.30.2

^ permalink raw reply related	[flat|nested] 2+ messages in thread

* Re: [PATCH] RDMA/rw: switch to dma_map_sgtable()
  2021-10-01 21:32 [PATCH] RDMA/rw: switch to dma_map_sgtable() Logan Gunthorpe
@ 2021-10-05 18:22 ` Jason Gunthorpe
  0 siblings, 0 replies; 2+ messages in thread
From: Jason Gunthorpe @ 2021-10-05 18:22 UTC (permalink / raw)
  To: Logan Gunthorpe
  Cc: Doug Ledford, linux-rdma, linux-kernel, Zheng Yongjun,
	Wenpeng Liang, Leon Romanovsky, Weihang Li, Mark Bloch

On Fri, Oct 01, 2021 at 03:32:15PM -0600, Logan Gunthorpe wrote:
> There are a couple of subtle error path bugs related to mapping the
> sgls:
> 
> - In rdma_rw_ctx_init(), dma_unmap would be called with an sg that
>   could have been incremented from the original call, as well as an
>   nents that was not the original number of nents called when mapped.
> - Similarly in rdma_rw_ctx_signature_init, both sg and prot_sg were
>   unmapped with the incorrect number of nents.
> 
> To fix this, switch to the sgtable interface for mapping which
> conveniently stores the original nents for unmapping. This will get
> cleaned up further once the dma mapping interface supports P2PDMA and
> pci_p2pdma_map_sg() can be removed. At that point the sgtable interface
> will be preferred as it offers better error reporting for P2PDMA pages.
> 
> Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
 ---
>  drivers/infiniband/core/rw.c | 66 ++++++++++++++++++++++--------------
>  1 file changed, 41 insertions(+), 25 deletions(-)

Applied to for-next, thanks

Jason

^ permalink raw reply	[flat|nested] 2+ messages in thread

end of thread, other threads:[~2021-10-05 18:22 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-10-01 21:32 [PATCH] RDMA/rw: switch to dma_map_sgtable() Logan Gunthorpe
2021-10-05 18:22 ` Jason Gunthorpe

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).