* [PATCH rdma-core 0/2] libhns: Add a new mmap implementation
@ 2021-09-30 8:37 Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 1/2] Update kernel headers Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 2/2] libhns: Add a new mmap implementation Wenpeng Liang
0 siblings, 2 replies; 3+ messages in thread
From: Wenpeng Liang @ 2021-09-30 8:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, linuxarm
Add a new mmap implementation for hns by using the new mmap entry API.
The "context->cq_tptr_base" is useless code and will be deleted later, so
its mmap implementation is no longer considered.
The related kernelspace series is named "RDMA/hns: Add a new mmap implementation".
Chengchang Tang (1):
libhns: Add a new mmap implementation
Wenpeng Liang (1):
Update kernel headers
kernel-headers/CMakeLists.txt | 5 ++
kernel-headers/rdma/hns-abi.h | 21 +++++++-
kernel-headers/rdma/irdma-abi.h | 2 +-
providers/hns/hns_roce_u.c | 86 ++++++++++++++++++++++++---------
providers/hns/hns_roce_u.h | 1 +
providers/hns/hns_roce_u_abi.h | 2 +-
6 files changed, 90 insertions(+), 27 deletions(-)
--
2.33.0
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH rdma-core 1/2] Update kernel headers
2021-09-30 8:37 [PATCH rdma-core 0/2] libhns: Add a new mmap implementation Wenpeng Liang
@ 2021-09-30 8:37 ` Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 2/2] libhns: Add a new mmap implementation Wenpeng Liang
1 sibling, 0 replies; 3+ messages in thread
From: Wenpeng Liang @ 2021-09-30 8:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, linuxarm
To commit ?? ("RDMA/hns: Add a new mmap implementation").
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
| 5 +++++
| 21 ++++++++++++++++++++-
| 2 +-
3 files changed, 26 insertions(+), 2 deletions(-)
--git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
index d9621ee2..3048d189 100644
--- a/kernel-headers/CMakeLists.txt
+++ b/kernel-headers/CMakeLists.txt
@@ -26,6 +26,11 @@ publish_internal_headers(rdma
rdma/vmw_pvrdma-abi.h
)
+publish_internal_headers(rdma/hfi
+ rdma/hfi/hfi1_ioctl.h
+ rdma/hfi/hfi1_user.h
+ )
+
publish_internal_headers(linux
linux/vfio.h
)
--git a/kernel-headers/rdma/hns-abi.h b/kernel-headers/rdma/hns-abi.h
index 42b17765..ce1e39f2 100644
--- a/kernel-headers/rdma/hns-abi.h
+++ b/kernel-headers/rdma/hns-abi.h
@@ -83,11 +83,30 @@ struct hns_roce_ib_create_qp_resp {
__aligned_u64 cap_flags;
};
+enum hns_roce_alloc_uctx_comp_flag {
+ HNS_ROCE_ALLOC_UCTX_COMP_CONFIG = 1 << 0,
+};
+
+enum hns_roce_alloc_uctx_resp_config {
+ HNS_ROCE_UCTX_RESP_MMAP_KEY_EN = 1 << 0,
+};
+
+enum hns_roce_alloc_uctx_req_config {
+ HNS_ROCE_UCTX_REQ_MMAP_KEY_EN = 1 << 0,
+};
+
+struct hns_roce_ib_alloc_ucontext {
+ __u32 comp;
+ __u32 config;
+};
+
struct hns_roce_ib_alloc_ucontext_resp {
__u32 qp_tab_size;
__u32 cqe_size;
__u32 srq_tab_size;
- __u32 reserved;
+ __u8 config;
+ __u8 rsv[3];
+ __aligned_u64 db_mmap_key;
};
struct hns_roce_ib_alloc_pd_resp {
--git a/kernel-headers/rdma/irdma-abi.h b/kernel-headers/rdma/irdma-abi.h
index 26b638a7..a7085e09 100644
--- a/kernel-headers/rdma/irdma-abi.h
+++ b/kernel-headers/rdma/irdma-abi.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB */
/*
* Copyright (c) 2006 - 2021 Intel Corporation. All rights reserved.
* Copyright (c) 2005 Topspin Communications. All rights reserved.
--
2.33.0
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH rdma-core 2/2] libhns: Add a new mmap implementation
2021-09-30 8:37 [PATCH rdma-core 0/2] libhns: Add a new mmap implementation Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 1/2] Update kernel headers Wenpeng Liang
@ 2021-09-30 8:37 ` Wenpeng Liang
1 sibling, 0 replies; 3+ messages in thread
From: Wenpeng Liang @ 2021-09-30 8:37 UTC (permalink / raw)
To: jgg, leon; +Cc: linux-rdma, linuxarm
From: Chengchang Tang <tangchengchang@huawei.com>
The new implementation prepares for subsequent new features and is
compatible with the old implementation.
In the new mmap implementation, the user space driver use the offset sent
by kernel space driver to complete the mmap instead of a hard-coded offset.
And the old implementation using hard-coded offset will not be extended in
the future.
Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
Signed-off-by: Wenpeng Liang <liangwenpeng@huawei.com>
---
providers/hns/hns_roce_u.c | 86 ++++++++++++++++++++++++----------
providers/hns/hns_roce_u.h | 1 +
providers/hns/hns_roce_u_abi.h | 2 +-
3 files changed, 64 insertions(+), 25 deletions(-)
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 3b31ad37..2531b097 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -95,16 +95,66 @@ static const struct verbs_context_ops hns_common_ops = {
.get_srq_num = hns_roce_u_get_srq_num,
};
+static int hns_roce_mmap(struct hns_roce_device *hr_dev,
+ struct hns_roce_context *context, int cmd_fd,
+ uint64_t db_mmap_key)
+{
+ context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, cmd_fd, db_mmap_key);
+ if (context->uar == MAP_FAILED)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int hns_roce_legacy_mmap(struct hns_roce_device *hr_dev,
+ struct hns_roce_context *context, int cmd_fd)
+{
+ off_t offset = 0;
+
+ context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, cmd_fd, offset);
+ if (context->uar == MAP_FAILED)
+ return -EINVAL;
+
+ offset += hr_dev->page_size;
+
+ if (hr_dev->hw_version == HNS_ROCE_HW_VER1) {
+ /*
+ * when vma->vm_pgoff is 1, the cq_tptr_base includes 64K CQ,
+ * a pointer of CQ need 2B size
+ */
+ context->cq_tptr_base = mmap(NULL, HNS_ROCE_CQ_DB_BUF_SIZE,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ cmd_fd, offset);
+ if (context->cq_tptr_base == MAP_FAILED)
+ goto db_free;
+ }
+
+ return 0;
+
+db_free:
+ munmap(context->uar, hr_dev->page_size);
+ context->uar = NULL;
+ return -EINVAL;
+}
+
+static void ucontext_set_cmd(struct hns_roce_alloc_ucontext *cmd)
+{
+ cmd->comp = HNS_ROCE_ALLOC_UCTX_COMP_CONFIG;
+ cmd->config = HNS_ROCE_UCTX_REQ_MMAP_KEY_EN;
+}
+
static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
int cmd_fd,
void *private_data)
{
struct hns_roce_device *hr_dev = to_hr_dev(ibdev);
struct hns_roce_alloc_ucontext_resp resp = {};
+ struct hns_roce_alloc_ucontext cmd = {};
struct ibv_device_attr dev_attrs;
struct hns_roce_context *context;
- struct ibv_get_context cmd;
- int offset = 0;
+ int ret;
int i;
context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
@@ -112,10 +162,14 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
if (!context)
return NULL;
- if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof(cmd),
+ ucontext_set_cmd(&cmd);
+ if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd),
&resp.ibv_resp, sizeof(resp)))
goto err_free;
+ context->mmap_key_support = !!(resp.config &
+ HNS_ROCE_UCTX_RESP_MMAP_KEY_EN);
+
if (!resp.cqe_size)
context->cqe_size = HNS_ROCE_CQE_SIZE;
else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE)
@@ -153,26 +207,14 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
context->max_cqe = dev_attrs.max_cqe;
context->max_srq_wr = dev_attrs.max_srq_wr;
context->max_srq_sge = dev_attrs.max_srq_sge;
+ if (context->mmap_key_support)
+ ret = hns_roce_mmap(hr_dev, context, cmd_fd, resp.db_mmap_key);
+ else
+ ret = hns_roce_legacy_mmap(hr_dev, context, cmd_fd);
- context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, cmd_fd, offset);
- if (context->uar == MAP_FAILED)
+ if (ret)
goto err_free;
- offset += hr_dev->page_size;
-
- if (hr_dev->hw_version == HNS_ROCE_HW_VER1) {
- /*
- * when vma->vm_pgoff is 1, the cq_tptr_base includes 64K CQ,
- * a pointer of CQ need 2B size
- */
- context->cq_tptr_base = mmap(NULL, HNS_ROCE_CQ_DB_BUF_SIZE,
- PROT_READ | PROT_WRITE, MAP_SHARED,
- cmd_fd, offset);
- if (context->cq_tptr_base == MAP_FAILED)
- goto db_free;
- }
-
pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
verbs_set_ops(&context->ibv_ctx, &hns_common_ops);
@@ -180,10 +222,6 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
return &context->ibv_ctx;
-db_free:
- munmap(context->uar, hr_dev->page_size);
- context->uar = NULL;
-
err_free:
verbs_uninit_context(&context->ibv_ctx);
free(context);
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 0d7abd81..126c9d3c 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -180,6 +180,7 @@ struct hns_roce_context {
unsigned int max_srq_sge;
int max_cqe;
unsigned int cqe_size;
+ bool mmap_key_support;
};
struct hns_roce_pd {
diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
index e56f9d35..295808b1 100644
--- a/providers/hns/hns_roce_u_abi.h
+++ b/providers/hns/hns_roce_u_abi.h
@@ -42,7 +42,7 @@ DECLARE_DRV_CMD(hns_roce_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
DECLARE_DRV_CMD(hns_roce_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
hns_roce_ib_create_cq, hns_roce_ib_create_cq_resp);
DECLARE_DRV_CMD(hns_roce_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
- empty, hns_roce_ib_alloc_ucontext_resp);
+ hns_roce_ib_alloc_ucontext, hns_roce_ib_alloc_ucontext_resp);
DECLARE_DRV_CMD(hns_roce_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
hns_roce_ib_create_qp, hns_roce_ib_create_qp_resp);
--
2.33.0
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-09-30 8:42 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-30 8:37 [PATCH rdma-core 0/2] libhns: Add a new mmap implementation Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 1/2] Update kernel headers Wenpeng Liang
2021-09-30 8:37 ` [PATCH rdma-core 2/2] libhns: Add a new mmap implementation Wenpeng Liang
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).