linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Bob Pearson <rpearsonhpe@gmail.com>
To: jgg@nvidia.com, xyjxyj2000@gmail.com, linux-rdma@vger.kernel.org
Cc: Bob Pearson <rpearsonhpe@gmail.com>
Subject: [PATCH 3/5] Providers/rxe: Support extended create srq
Date: Fri, 30 Jul 2021 10:21:56 -0500	[thread overview]
Message-ID: <20210730152157.67592-4-rpearsonhpe@gmail.com> (raw)
In-Reply-To: <20210730152157.67592-1-rpearsonhpe@gmail.com>

Add support for the ibv_create_srq_ex verb.

Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
---
 providers/rxe/rxe-abi.h |  2 ++
 providers/rxe/rxe.c     | 54 +++++++++++++++++++++++++++++++++++------
 providers/rxe/rxe.h     |  5 ++--
 3 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/providers/rxe/rxe-abi.h b/providers/rxe/rxe-abi.h
index 020201a9..07e90d81 100644
--- a/providers/rxe/rxe-abi.h
+++ b/providers/rxe/rxe-abi.h
@@ -51,6 +51,8 @@ DECLARE_DRV_CMD(urxe_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
 		empty, rxe_create_qp_resp);
 DECLARE_DRV_CMD(urxe_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		empty, rxe_create_srq_resp);
+DECLARE_DRV_CMD(urxe_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+		empty, rxe_create_srq_resp);
 DECLARE_DRV_CMD(urxe_modify_srq, IB_USER_VERBS_CMD_MODIFY_SRQ,
 		rxe_modify_srq_cmd, empty);
 DECLARE_DRV_CMD(urxe_resize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
diff --git a/providers/rxe/rxe.c b/providers/rxe/rxe.c
index 3bb9f01c..9cdddb8c 100644
--- a/providers/rxe/rxe.c
+++ b/providers/rxe/rxe.c
@@ -636,7 +636,7 @@ static struct ibv_srq *rxe_create_srq(struct ibv_pd *pd,
 	if (srq == NULL)
 		return NULL;
 
-	ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr, &cmd, sizeof(cmd),
+	ret = ibv_cmd_create_srq(pd, &srq->vsrq.srq, attr, &cmd, sizeof(cmd),
 				 &resp.ibv_resp, sizeof(resp));
 	if (ret) {
 		free(srq);
@@ -647,7 +647,7 @@ static struct ibv_srq *rxe_create_srq(struct ibv_pd *pd,
 			     PROT_READ | PROT_WRITE, MAP_SHARED,
 			     pd->context->cmd_fd, resp.mi.offset);
 	if ((void *)srq->rq.queue == MAP_FAILED) {
-		ibv_cmd_destroy_srq(&srq->ibv_srq);
+		ibv_cmd_destroy_srq(&srq->vsrq.srq);
 		free(srq);
 		return NULL;
 	}
@@ -656,7 +656,44 @@ static struct ibv_srq *rxe_create_srq(struct ibv_pd *pd,
 	srq->rq.max_sge = attr->attr.max_sge;
 	pthread_spin_init(&srq->rq.lock, PTHREAD_PROCESS_PRIVATE);
 
-	return &srq->ibv_srq;
+	return &srq->vsrq.srq;
+}
+
+static struct ibv_srq *rxe_create_srq_ex(struct ibv_context *context,
+				struct ibv_srq_init_attr_ex *attr_ex)
+{
+	struct rxe_srq *srq;
+	struct ibv_create_xsrq cmd = {};
+	size_t cmd_size = sizeof(cmd);
+	struct urxe_create_srq_ex_resp resp = {};
+	size_t resp_size = sizeof(resp);
+	int ret;
+
+	srq = calloc(1, sizeof(*srq));
+	if (!srq)
+		return NULL;
+
+	ret = ibv_cmd_create_srq_ex(context, &srq->vsrq, attr_ex,
+			  &cmd, cmd_size, &resp.ibv_resp, resp_size);
+	if (ret) {
+		free(srq);
+		return NULL;
+	}
+
+	srq->rq.queue = mmap(NULL, resp.mi.size,
+			     PROT_READ | PROT_WRITE, MAP_SHARED,
+			     context->cmd_fd, resp.mi.offset);
+	if ((void *)srq->rq.queue == MAP_FAILED) {
+		ibv_cmd_destroy_srq(&srq->vsrq.srq);
+		free(srq);
+		return NULL;
+	}
+
+	srq->mmap_info = resp.mi;
+	srq->rq.max_sge = attr_ex->attr.max_sge;
+	pthread_spin_init(&srq->rq.lock, PTHREAD_PROCESS_PRIVATE);
+
+	return &srq->vsrq.srq;
 }
 
 static int rxe_modify_srq(struct ibv_srq *ibsrq,
@@ -708,13 +745,13 @@ static int rxe_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr)
 	return ibv_cmd_query_srq(srq, attr, &cmd, sizeof(cmd));
 }
 
-static int rxe_destroy_srq(struct ibv_srq *ibvsrq)
+static int rxe_destroy_srq(struct ibv_srq *ibsrq)
 {
 	int ret;
-	struct rxe_srq *srq = to_rsrq(ibvsrq);
+	struct rxe_srq *srq = to_rsrq(ibsrq);
 	struct rxe_queue_buf *q = srq->rq.queue;
 
-	ret = ibv_cmd_destroy_srq(ibvsrq);
+	ret = ibv_cmd_destroy_srq(ibsrq);
 	if (!ret) {
 		if (srq->mmap_info.size)
 			munmap(q, srq->mmap_info.size);
@@ -765,11 +802,11 @@ out:
 	return rc;
 }
 
-static int rxe_post_srq_recv(struct ibv_srq *ibvsrq,
+static int rxe_post_srq_recv(struct ibv_srq *ibsrq,
 			     struct ibv_recv_wr *recv_wr,
 			     struct ibv_recv_wr **bad_recv_wr)
 {
-	struct rxe_srq *srq = to_rsrq(ibvsrq);
+	struct rxe_srq *srq = to_rsrq(ibsrq);
 	int rc = 0;
 
 	pthread_spin_lock(&srq->rq.lock);
@@ -1794,6 +1831,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
 	.resize_cq = rxe_resize_cq,
 	.destroy_cq = rxe_destroy_cq,
 	.create_srq = rxe_create_srq,
+	.create_srq_ex = rxe_create_srq_ex,
 	.modify_srq = rxe_modify_srq,
 	.query_srq = rxe_query_srq,
 	.destroy_srq = rxe_destroy_srq,
diff --git a/providers/rxe/rxe.h b/providers/rxe/rxe.h
index 6882d9c7..f3215f2e 100644
--- a/providers/rxe/rxe.h
+++ b/providers/rxe/rxe.h
@@ -91,10 +91,9 @@ struct rxe_qp {
 };
 
 struct rxe_srq {
-	struct ibv_srq		ibv_srq;
+	struct verbs_srq	vsrq;
 	struct mminfo		mmap_info;
 	struct rxe_wq		rq;
-	uint32_t		srq_num;
 };
 
 #define to_rxxx(xxx, type) container_of(ib##xxx, struct rxe_##type, ibv_##xxx)
@@ -121,7 +120,7 @@ static inline struct rxe_qp *to_rqp(struct ibv_qp *ibqp)
 
 static inline struct rxe_srq *to_rsrq(struct ibv_srq *ibsrq)
 {
-	return to_rxxx(srq, srq);
+	return container_of(ibsrq, struct rxe_srq, vsrq.srq);
 }
 
 static inline struct rxe_ah *to_rah(struct ibv_ah *ibah)
-- 
2.30.2


  parent reply	other threads:[~2021-07-30 15:23 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-07-30 15:21 [PATCH 0/5] Providers/rxe: Implement XRC transport for rxe Bob Pearson
2021-07-30 15:21 ` [PATCH 1/5] Update kernel headers Bob Pearson
2021-07-30 15:21 ` [PATCH 2/5] Providers/rxe: Support alloc/dealloc xrcd Bob Pearson
2021-07-30 15:21 ` Bob Pearson [this message]
2021-07-30 15:21 ` [PATCH 4/5] Providers/rxe: Support get srq number Bob Pearson
2021-07-30 15:21 ` [PATCH 5/5] Providers/rxe: Support XRC traffic Bob Pearson
2021-08-02  8:30   ` Leon Romanovsky
2021-08-13 21:39     ` Bob Pearson

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210730152157.67592-4-rpearsonhpe@gmail.com \
    --to=rpearsonhpe@gmail.com \
    --cc=jgg@nvidia.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=xyjxyj2000@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).