linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Shunsuke Mie <mie@igel.co.jp>
To: Zhu Yanjun <zyjzyj2000@gmail.com>
Cc: "Shunsuke Mie" <mie@igel.co.jp>,
	"Christian König" <christian.koenig@amd.com>,
	"Alex Deucher" <alexander.deucher@amd.com>,
	"Daniel Vetter" <daniel.vetter@ffwll.ch>,
	"Doug Ledford" <dledford@redhat.com>,
	"Jason Gunthorpe" <jgg@ziepe.ca>,
	"Jianxin Xiong" <jianxin.xiong@intel.com>,
	"Leon Romanovsky" <leon@kernel.org>,
	linux-kernel@vger.kernel.org, linux-rdma@vger.kernel.org,
	dhobsong@igel.co.jp, taki@igel.co.jp, etom@igel.co.jp
Subject: [RFC PATCH 3/3] RDMA/rxe: Support dma-buf as memory region
Date: Wed,  8 Sep 2021 15:16:11 +0900	[thread overview]
Message-ID: <20210908061611.69823-4-mie@igel.co.jp> (raw)
In-Reply-To: <20210908061611.69823-1-mie@igel.co.jp>

Implement a ib device operation ‘reg_user_mr_dmabuf’. Import dma-buf
using the IB core API and map the memory area linked the dma-buf.

Signed-off-by: Shunsuke Mie <mie@igel.co.jp>
---
 drivers/infiniband/sw/rxe/rxe_loc.h   |   3 +
 drivers/infiniband/sw/rxe/rxe_mr.c    | 101 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.c |  36 +++++++++
 3 files changed, 140 insertions(+)

diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 1ddb20855dee..206d9d5f8bbf 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -75,6 +75,9 @@ u8 rxe_get_next_key(u32 last_key);
 void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr);
 int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,
 		     int access, struct rxe_mr *mr);
+int rxe_mr_dmabuf_init_user(struct rxe_pd *pd, int fd, u64 start, u64 length,
+			    u64 iova, int access, struct rxe_mr *mr);
+
 int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr);
 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		enum rxe_mr_copy_dir dir, u32 *crcp);
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 8b08705ed62a..846f52aad0de 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  */
 
+#include <linux/dma-buf.h>
+
 #include "rxe.h"
 #include "rxe_loc.h"
 
@@ -207,6 +209,105 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,
 	return err;
 }
 
+static int rxe_map_dmabuf_mr(struct rxe_mr *mr)
+{
+	struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem);
+	struct ib_umem *umem = mr->umem;
+	int err;
+
+	err = ib_umem_dmabuf_map_pages(umem_dmabuf);
+	if (err)
+		goto err1;
+
+	err = rxe_mr_gen_map(mr, umem);
+	if (err)
+		goto err2;
+
+	return ib_umem_num_pages(umem);
+
+err2:
+	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
+err1:
+	return err;
+}
+
+/* A function called from the dma-buf exporter when the mapped pages
+ * become invalid.
+ */
+static void rxe_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)
+{
+	int err;
+	struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
+	struct rxe_mr *mr = umem_dmabuf->private;
+
+	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
+
+	/* all of memory region is immediately mapped again */
+	err = rxe_map_dmabuf_mr(mr);
+	if (err)
+		pr_err("%s: failed to map the dma-buf region", __func__);
+}
+
+static struct dma_buf_attach_ops rxe_ib_dmabuf_attach_ops = {
+	.move_notify = rxe_ib_dmabuf_invalidate_cb,
+};
+
+/* initialize a umem and map all the areas of dma-buf. */
+int rxe_mr_dmabuf_init_user(struct rxe_pd *pd, int fd, u64 start, u64 length,
+			    u64 iova, int access, struct rxe_mr *mr)
+{
+	struct ib_umem_dmabuf *umem_dmabuf;
+	int num_buf;
+	int err;
+
+	umem_dmabuf = ib_umem_dmabuf_get(pd->ibpd.device, start, length, fd,
+					 access, &rxe_ib_dmabuf_attach_ops);
+	if (IS_ERR(umem_dmabuf)) {
+		err = PTR_ERR(umem_dmabuf);
+		pr_err("%s: failed to get umem_dmabuf (%d)", __func__, err);
+		goto err1;
+	}
+
+	umem_dmabuf->private = mr;
+
+	mr->umem = &umem_dmabuf->umem;
+	mr->umem->iova = iova;
+	num_buf = ib_umem_num_pages(mr->umem);
+
+	rxe_mr_init(access, mr);
+
+	err = rxe_mr_alloc(mr, num_buf);
+	if (err)
+		goto err1;
+
+	mr->page_shift = PAGE_SHIFT;
+	mr->page_mask = PAGE_SIZE - 1;
+
+	mr->ibmr.pd = &pd->ibpd;
+	mr->access = access;
+	mr->length = length;
+	mr->iova = iova;
+	mr->va = start;
+	mr->offset = ib_umem_offset(mr->umem);
+	mr->state = RXE_MR_STATE_VALID;
+	mr->type = RXE_MR_TYPE_MR;
+
+	err = rxe_map_dmabuf_mr(mr);
+	if (err) {
+		pr_err("%s: failed to map the dma-buf region", __func__);
+		goto err2;
+	}
+
+	return 0;
+
+err2:
+	for (i = 0; i < mr->num_map; i++)
+		kfree(mr->map[i]);
+	kfree(mr->map);
+err1:
+	return err;
+}
+
 int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr)
 {
 	int err;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index c223959ac174..4a38c20730b3 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -959,6 +959,39 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 	return ERR_PTR(err);
 }
 
+static struct ib_mr *rxe_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
+					    u64 length, u64 iova, int fd,
+					    int access, struct ib_udata *udata)
+{
+	int err;
+	struct rxe_dev *rxe = to_rdev(ibpd->device);
+	struct rxe_pd *pd = to_rpd(ibpd);
+	struct rxe_mr *mr;
+
+	mr = rxe_alloc(&rxe->mr_pool);
+	if (!mr) {
+		err = -ENOMEM;
+		goto err1;
+	}
+
+	rxe_add_index(mr);
+
+	rxe_add_ref(pd);
+
+	err = rxe_mr_dmabuf_init_user(pd, fd, start, length, iova, access, mr);
+	if (err)
+		goto err2;
+
+	return &mr->ibmr;
+
+err2:
+	rxe_drop_ref(pd);
+	rxe_drop_index(mr);
+	rxe_drop_ref(mr);
+err1:
+	return ERR_PTR(err);
+}
+
 static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 				  u32 max_num_sg)
 {
@@ -1139,6 +1172,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.query_qp = rxe_query_qp,
 	.query_srq = rxe_query_srq,
 	.reg_user_mr = rxe_reg_user_mr,
+	.reg_user_mr_dmabuf = rxe_reg_user_mr_dmabuf,
 	.req_notify_cq = rxe_req_notify_cq,
 	.resize_cq = rxe_resize_cq,
 
@@ -1181,6 +1215,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 	}
 	rxe->tfm = tfm;
 
+	dma_coerce_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64));
+
 	err = ib_register_device(dev, ibdev_name, NULL);
 	if (err)
 		pr_warn("%s failed with error %d\n", __func__, err);
-- 
2.17.1


  parent reply	other threads:[~2021-09-08  6:16 UTC|newest]

Thread overview: 20+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-08  6:16 [RFC PATCH 0/3] RDMA/rxe: Add dma-buf support Shunsuke Mie
2021-09-08  6:16 ` [RFC PATCH 1/3] RDMA/umem: Change for rdma devices has not dma device Shunsuke Mie
2021-09-08  6:26   ` Christoph Hellwig
2021-09-08  7:01     ` Shunsuke Mie
2021-09-08  7:19       ` Christoph Hellwig
2021-09-08  8:41         ` Shunsuke Mie
2021-09-08 11:18           ` Jason Gunthorpe
2021-09-08 13:33             ` Christian König
2021-09-08 19:22               ` Daniel Vetter
2021-09-08 23:33                 ` Jason Gunthorpe
2021-09-09  9:26                   ` Daniel Vetter
2021-09-10  1:46                     ` Shunsuke Mie
2021-09-13 19:22                       ` Daniel Vetter
2021-09-14  7:11                         ` Shunsuke Mie
2021-09-14  9:38                           ` Daniel Vetter
2021-09-14 10:13                             ` Shunsuke Mie
2021-09-08  6:16 ` [RFC PATCH 2/3] RDMA/rxe: Extract a mapping process into a function Shunsuke Mie
2021-09-08  6:16 ` Shunsuke Mie [this message]
2021-09-09  5:45 ` [RFC PATCH 0/3] RDMA/rxe: Add dma-buf support Zhu Yanjun
2021-09-10  2:00   ` Shunsuke Mie

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210908061611.69823-4-mie@igel.co.jp \
    --to=mie@igel.co.jp \
    --cc=alexander.deucher@amd.com \
    --cc=christian.koenig@amd.com \
    --cc=daniel.vetter@ffwll.ch \
    --cc=dhobsong@igel.co.jp \
    --cc=dledford@redhat.com \
    --cc=etom@igel.co.jp \
    --cc=jgg@ziepe.ca \
    --cc=jianxin.xiong@intel.com \
    --cc=leon@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-rdma@vger.kernel.org \
    --cc=taki@igel.co.jp \
    --cc=zyjzyj2000@gmail.com \
    --subject='Re: [RFC PATCH 3/3] RDMA/rxe: Support dma-buf as memory region' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).