From: Jianxin Xiong <jianxin.xiong@intel.com>
To: linux-rdma@vger.kernel.org
Cc: Jianxin Xiong <jianxin.xiong@intel.com>,
Doug Ledford <dledford@redhat.com>,
Jason Gunthorpe <jgg@ziepe.ca>,
Sumit Semwal <sumit.semwal@linaro.org>,
Leon Romanovsky <leon@kernel.org>,
Daniel Vetter <daniel.vetter@intel.com>
Subject: [RFC PATCH v2 1/3] RDMA/umem: Support importing dma-buf as user memory region
Date: Mon, 29 Jun 2020 10:31:41 -0700 [thread overview]
Message-ID: <1593451903-30959-2-git-send-email-jianxin.xiong@intel.com> (raw)
In-Reply-To: <1593451903-30959-1-git-send-email-jianxin.xiong@intel.com>
Dma-buf is a standard cross-driver buffer sharing mechanism that can be
used to support peer-to-peer access from RDMA devices.
Device memory exported via dma-buf can be associated with a file
descriptor. This is passed to the user space as a property associated
with the buffer allocation. When the buffer is registered as a memory
region, the file descriptor is passed to the RDMA driver along with
other parameters.
Implement the common code for importing dma-buf object, and pinning and
mapping dma-buf pages.
Signed-off-by: Jianxin Xiong <jianxin.xiong@intel.com>
Reviewed-by: Sean Hefty <sean.hefty@intel.com>
Acked-by: Michael J. Ruhl <michael.j.ruhl@intel.com>
---
drivers/infiniband/core/Makefile | 2 +-
drivers/infiniband/core/umem.c | 4 ++
drivers/infiniband/core/umem_dmabuf.c | 105 ++++++++++++++++++++++++++++++++++
drivers/infiniband/core/umem_dmabuf.h | 11 ++++
include/rdma/ib_umem.h | 14 ++++-
5 files changed, 134 insertions(+), 2 deletions(-)
create mode 100644 drivers/infiniband/core/umem_dmabuf.c
create mode 100644 drivers/infiniband/core/umem_dmabuf.h
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index d1b14887..0d4efa0 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -37,5 +37,5 @@ ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
uverbs_std_types_mr.o uverbs_std_types_counters.o \
uverbs_uapi.o uverbs_std_types_device.o \
uverbs_std_types_async_fd.o
-ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
+ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o umem_dmabuf.o
ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 82455a1..bf1a6c1 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -2,6 +2,7 @@
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Cisco Systems. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2020 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -42,6 +43,7 @@
#include <rdma/ib_umem_odp.h>
#include "uverbs.h"
+#include "umem_dmabuf.h"
static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
{
@@ -317,6 +319,8 @@ void ib_umem_release(struct ib_umem *umem)
{
if (!umem)
return;
+ if (umem->is_dmabuf)
+ return ib_umem_dmabuf_release(umem);
if (umem->is_odp)
return ib_umem_odp_release(to_ib_umem_odp(umem));
diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c
new file mode 100644
index 0000000..406ca64
--- /dev/null
+++ b/drivers/infiniband/core/umem_dmabuf.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright (c) 2020 Intel Corporation. All rights reserved.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+
+#include "uverbs.h"
+
+struct ib_umem_dmabuf {
+ struct ib_umem umem;
+ struct dma_buf *dmabuf;
+ struct dma_buf_attachment *attach;
+ struct sg_table *sgt;
+};
+
+static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem)
+{
+ return container_of(umem, struct ib_umem_dmabuf, umem);
+}
+
+struct ib_umem *ib_umem_dmabuf_get(struct ib_device *device,
+ unsigned long addr, size_t size,
+ int dmabuf_fd, int access)
+{
+ struct ib_umem_dmabuf *umem_dmabuf;
+ struct sg_table *sgt;
+ enum dma_data_direction dir;
+ long ret;
+ unsigned long end;
+
+ if (check_add_overflow(addr, size, &end))
+ return ERR_PTR(-EINVAL);
+
+ if (unlikely(PAGE_ALIGN(end) < PAGE_SIZE))
+ return ERR_PTR(-EINVAL);
+
+ if (access & IB_ACCESS_ON_DEMAND)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
+ if (!umem_dmabuf)
+ return ERR_PTR(-ENOMEM);
+
+ umem_dmabuf->umem.ibdev = device;
+ umem_dmabuf->umem.length = size;
+ umem_dmabuf->umem.address = addr;
+ umem_dmabuf->umem.writable = ib_access_writable(access);
+ umem_dmabuf->umem.is_dmabuf = 1;
+
+ umem_dmabuf->dmabuf = dma_buf_get(dmabuf_fd);
+ if (IS_ERR(umem_dmabuf->dmabuf)) {
+ ret = PTR_ERR(umem_dmabuf->dmabuf);
+ goto out_free_umem;
+ }
+
+ umem_dmabuf->attach = dma_buf_attach(umem_dmabuf->dmabuf,
+ device->dma_device);
+ if (IS_ERR(umem_dmabuf->attach)) {
+ ret = PTR_ERR(umem_dmabuf->attach);
+ goto out_release_dmabuf;
+ }
+
+ dir = umem_dmabuf->umem.writable ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
+ sgt = dma_buf_map_attachment(umem_dmabuf->attach, dir);
+ if (IS_ERR(sgt)) {
+ ret = PTR_ERR(sgt);
+ goto out_detach_dmabuf;
+ }
+
+ umem_dmabuf->sgt = sgt;
+ umem_dmabuf->umem.sg_head = *sgt;
+ umem_dmabuf->umem.nmap = sgt->nents;
+ return &umem_dmabuf->umem;
+
+out_detach_dmabuf:
+ dma_buf_detach(umem_dmabuf->dmabuf, umem_dmabuf->attach);
+
+out_release_dmabuf:
+ dma_buf_put(umem_dmabuf->dmabuf);
+
+out_free_umem:
+ kfree(umem_dmabuf);
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(ib_umem_dmabuf_get);
+
+void ib_umem_dmabuf_release(struct ib_umem *umem)
+{
+ enum dma_data_direction dir;
+ struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(umem);
+
+ dir = umem_dmabuf->umem.writable ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
+
+ /*
+ * Only use the original sgt returned from dma_buf_map_attachment(),
+ * otherwise the scatterlist may be freed twice due to the map caching
+ * mechanism.
+ */
+ dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt, dir);
+ dma_buf_detach(umem_dmabuf->dmabuf, umem_dmabuf->attach);
+ dma_buf_put(umem_dmabuf->dmabuf);
+ kfree(umem_dmabuf);
+}
diff --git a/drivers/infiniband/core/umem_dmabuf.h b/drivers/infiniband/core/umem_dmabuf.h
new file mode 100644
index 0000000..485f653
--- /dev/null
+++ b/drivers/infiniband/core/umem_dmabuf.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * Copyright (c) 2020 Intel Corporation. All rights reserved.
+ */
+
+#ifndef UMEM_DMABUF_H
+#define UMEM_DMABUF_H
+
+void ib_umem_dmabuf_release(struct ib_umem *umem);
+
+#endif /* UMEM_DMABUF_H */
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index e3518fd..78331ce 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2007 Cisco Systems. All rights reserved.
+ * Copyright (c) 2020 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -40,6 +41,7 @@
struct ib_ucontext;
struct ib_umem_odp;
+struct ib_umem_dmabuf;
struct ib_umem {
struct ib_device *ibdev;
@@ -48,6 +50,7 @@ struct ib_umem {
unsigned long address;
u32 writable : 1;
u32 is_odp : 1;
+ u32 is_dmabuf : 1;
struct work_struct work;
struct sg_table sg_head;
int nmap;
@@ -78,6 +81,9 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
unsigned long pgsz_bitmap,
unsigned long virt);
+struct ib_umem *ib_umem_dmabuf_get(struct ib_device *device,
+ unsigned long addr, size_t size,
+ int dmabuf_fd, int access);
#else /* CONFIG_INFINIBAND_USER_MEM */
@@ -100,7 +106,13 @@ static inline int ib_umem_find_best_pgsz(struct ib_umem *umem,
unsigned long virt) {
return -EINVAL;
}
+static inline struct ib_umem *ib_umem_dmabuf_get(struct ib_device *device,
+ unsigned long addr,
+ size_t size, int dmabuf_fd,
+ int access)
+{
+ return ERR_PTR(-EINVAL);
+}
#endif /* CONFIG_INFINIBAND_USER_MEM */
-
#endif /* IB_UMEM_H */
--
1.8.3.1
next prev parent reply other threads:[~2020-06-29 21:14 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-29 17:31 [RFC PATCH v2 0/3] RDMA: add dma-buf support Jianxin Xiong
2020-06-29 17:31 ` Jianxin Xiong [this message]
2020-06-30 19:04 ` [RFC PATCH v2 1/3] RDMA/umem: Support importing dma-buf as user memory region Xiong, Jianxin
2020-06-29 17:31 ` [RFC PATCH v2 2/3] RDMA/core: Expand the driver method 'reg_user_mr' to support dma-buf Jianxin Xiong
2020-06-30 19:04 ` Xiong, Jianxin
2020-06-29 17:31 ` [RFC PATCH v2 3/3] RDMA/uverbs: Add uverbs command for dma-buf based MR registration Jianxin Xiong
2020-06-30 19:05 ` Xiong, Jianxin
2020-06-29 18:51 ` [RFC PATCH v2 0/3] RDMA: add dma-buf support Jason Gunthorpe
2020-06-30 17:21 ` Xiong, Jianxin
2020-06-30 17:34 ` Jason Gunthorpe
2020-06-30 18:46 ` Xiong, Jianxin
2020-06-30 19:17 ` Jason Gunthorpe
2020-06-30 20:08 ` Xiong, Jianxin
2020-07-02 12:27 ` Jason Gunthorpe
2020-07-01 9:03 ` Christian König
2020-07-01 12:07 ` Daniel Vetter
2020-07-01 12:14 ` Daniel Vetter
2020-07-01 12:39 ` Jason Gunthorpe
2020-07-01 12:55 ` Christian König
2020-07-01 15:42 ` Daniel Vetter
2020-07-01 17:15 ` Jason Gunthorpe
2020-07-02 13:10 ` Daniel Vetter
2020-07-02 13:29 ` Jason Gunthorpe
2020-07-02 14:50 ` Christian König
2020-07-02 18:15 ` Daniel Vetter
2020-07-03 12:03 ` Jason Gunthorpe
2020-07-03 12:52 ` Daniel Vetter
2020-07-03 13:14 ` Jason Gunthorpe
2020-07-03 13:21 ` Christian König
2020-07-07 21:58 ` Xiong, Jianxin
2020-07-08 9:38 ` Christian König
2020-07-08 9:49 ` Daniel Vetter
2020-07-08 14:20 ` Christian König
2020-07-08 14:33 ` Alex Deucher
2020-06-30 18:56 ` Xiong, Jianxin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1593451903-30959-2-git-send-email-jianxin.xiong@intel.com \
--to=jianxin.xiong@intel.com \
--cc=daniel.vetter@intel.com \
--cc=dledford@redhat.com \
--cc=jgg@ziepe.ca \
--cc=leon@kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=sumit.semwal@linaro.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).