linux-rdma.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Bob Pearson <rpearsonhpe@gmail.com>
To: jgg@nvidia.com, zyjzyj2000@gmail.com, linux-rdma@vger.kernel.org
Cc: Bob Pearson <rpearsonhpe@gmail.com>
Subject: [PATCH for-next 08/13] RDMA/rxe: Support alloc/dealloc xrcd
Date: Thu, 29 Jul 2021 17:49:11 -0500	[thread overview]
Message-ID: <20210729224915.38986-9-rpearsonhpe@gmail.com> (raw)
In-Reply-To: <20210729224915.38986-1-rpearsonhpe@gmail.com>

Implement code to support ibv_alloc_xrcd and ibv_dealloc_xrcd verbs.

Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
---
 drivers/infiniband/sw/rxe/rxe.c       | 39 ++++++++++++++++-----------
 drivers/infiniband/sw/rxe/rxe_param.h |  1 +
 drivers/infiniband/sw/rxe/rxe_pool.c  |  6 +++++
 drivers/infiniband/sw/rxe/rxe_pool.h  |  1 +
 drivers/infiniband/sw/rxe/rxe_verbs.c | 19 +++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.h | 12 +++++++++
 6 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 8e0f9c489cab..fbbb3d6f172b 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -24,6 +24,7 @@ void rxe_dealloc(struct ib_device *ib_dev)
 
 	rxe_pool_cleanup(&rxe->uc_pool);
 	rxe_pool_cleanup(&rxe->pd_pool);
+	rxe_pool_cleanup(&rxe->xrcd_pool);
 	rxe_pool_cleanup(&rxe->ah_pool);
 	rxe_pool_cleanup(&rxe->srq_pool);
 	rxe_pool_cleanup(&rxe->qp_pool);
@@ -74,6 +75,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 			rxe->ndev->dev_addr);
 
 	rxe->max_ucontext			= RXE_MAX_UCONTEXT;
+	rxe->max_xrcd				= RXE_MAX_XRCD;
 }
 
 /* initialize port attributes */
@@ -130,62 +132,69 @@ static int rxe_init_pools(struct rxe_dev *rxe)
 	if (err)
 		goto err2;
 
+	err = rxe_pool_init(rxe, &rxe->xrcd_pool, RXE_TYPE_XRCD,
+			    rxe->max_xrcd);
+	if (err)
+		goto err3;
+
 	err = rxe_pool_init(rxe, &rxe->ah_pool, RXE_TYPE_AH,
 			    rxe->attr.max_ah);
 	if (err)
-		goto err3;
+		goto err4;
 
 	err = rxe_pool_init(rxe, &rxe->srq_pool, RXE_TYPE_SRQ,
 			    rxe->attr.max_srq);
 	if (err)
-		goto err4;
+		goto err5;
 
 	err = rxe_pool_init(rxe, &rxe->qp_pool, RXE_TYPE_QP,
 			    rxe->attr.max_qp);
 	if (err)
-		goto err5;
+		goto err6;
 
 	err = rxe_pool_init(rxe, &rxe->cq_pool, RXE_TYPE_CQ,
 			    rxe->attr.max_cq);
 	if (err)
-		goto err6;
+		goto err7;
 
 	err = rxe_pool_init(rxe, &rxe->mr_pool, RXE_TYPE_MR,
 			    rxe->attr.max_mr);
 	if (err)
-		goto err7;
+		goto err8;
 
 	err = rxe_pool_init(rxe, &rxe->mw_pool, RXE_TYPE_MW,
 			    rxe->attr.max_mw);
 	if (err)
-		goto err8;
+		goto err9;
 
 	err = rxe_pool_init(rxe, &rxe->mc_grp_pool, RXE_TYPE_MC_GRP,
 			    rxe->attr.max_mcast_grp);
 	if (err)
-		goto err9;
+		goto err10;
 
 	err = rxe_pool_init(rxe, &rxe->mc_elem_pool, RXE_TYPE_MC_ELEM,
 			    rxe->attr.max_total_mcast_qp_attach);
 	if (err)
-		goto err10;
+		goto err11;
 
 	return 0;
 
-err10:
+err11:
 	rxe_pool_cleanup(&rxe->mc_grp_pool);
-err9:
+err10:
 	rxe_pool_cleanup(&rxe->mw_pool);
-err8:
+err9:
 	rxe_pool_cleanup(&rxe->mr_pool);
-err7:
+err8:
 	rxe_pool_cleanup(&rxe->cq_pool);
-err6:
+err7:
 	rxe_pool_cleanup(&rxe->qp_pool);
-err5:
+err6:
 	rxe_pool_cleanup(&rxe->srq_pool);
-err4:
+err5:
 	rxe_pool_cleanup(&rxe->ah_pool);
+err4:
+	rxe_pool_cleanup(&rxe->xrcd_pool);
 err3:
 	rxe_pool_cleanup(&rxe->pd_pool);
 err2:
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index ec5c6331bee8..b843be4cc25a 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -79,6 +79,7 @@ enum rxe_device_param {
 	RXE_LOCAL_CA_ACK_DELAY		= 15,
 
 	RXE_MAX_UCONTEXT		= 512,
+	RXE_MAX_XRCD			= 512,
 
 	RXE_NUM_PORT			= 1,
 
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 342f090152d1..76caef5790b0 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -22,6 +22,12 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 		.elem_offset	= offsetof(struct rxe_pd, pelem),
 		.flags		= RXE_POOL_NO_ALLOC,
 	},
+	[RXE_TYPE_XRCD] = {
+		.name		= "rxe-xrcd",
+		.size		= sizeof(struct rxe_xrcd),
+		.elem_offset	= offsetof(struct rxe_xrcd, pelem),
+		.flags		= RXE_POOL_NO_ALLOC,
+	},
 	[RXE_TYPE_AH] = {
 		.name		= "rxe-ah",
 		.size		= sizeof(struct rxe_ah),
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h
index 1feca1bffced..1475b9374315 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.h
+++ b/drivers/infiniband/sw/rxe/rxe_pool.h
@@ -19,6 +19,7 @@ enum rxe_pool_flags {
 enum rxe_elem_type {
 	RXE_TYPE_UC,
 	RXE_TYPE_PD,
+	RXE_TYPE_XRCD,
 	RXE_TYPE_AH,
 	RXE_TYPE_SRQ,
 	RXE_TYPE_QP,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 7ff98a60ddcd..b4b993f1ce92 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -156,6 +156,22 @@ static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	return 0;
 }
 
+static int rxe_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+	struct rxe_dev *rxe = to_rdev(ibxrcd->device);
+	struct rxe_xrcd *xrcd = to_rxrcd(ibxrcd);
+
+	return rxe_add_to_pool(&rxe->xrcd_pool, xrcd);
+}
+
+static int rxe_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+	struct rxe_xrcd *xrcd = to_rxrcd(ibxrcd);
+
+	rxe_drop_ref(xrcd);
+	return 0;
+}
+
 static int rxe_create_ah(struct ib_ah *ibah,
 			 struct rdma_ah_init_attr *init_attr,
 			 struct ib_udata *udata)
@@ -1078,6 +1094,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.alloc_mw = rxe_alloc_mw,
 	.alloc_pd = rxe_alloc_pd,
 	.alloc_ucontext = rxe_alloc_ucontext,
+	.alloc_xrcd = rxe_alloc_xrcd,
 	.attach_mcast = rxe_attach_mcast,
 	.create_ah = rxe_create_ah,
 	.create_cq = rxe_create_cq,
@@ -1088,6 +1105,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.dealloc_mw = rxe_dealloc_mw,
 	.dealloc_pd = rxe_dealloc_pd,
 	.dealloc_ucontext = rxe_dealloc_ucontext,
+	.dealloc_xrcd = rxe_dealloc_xrcd,
 	.dereg_mr = rxe_dereg_mr,
 	.destroy_ah = rxe_destroy_ah,
 	.destroy_cq = rxe_destroy_cq,
@@ -1128,6 +1146,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
 	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
+	INIT_RDMA_OBJ_SIZE(ib_xrcd, rxe_xrcd, ibxrcd),
 };
 
 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 0a433f4c0222..5b75de74a992 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -43,6 +43,11 @@ struct rxe_pd {
 	struct rxe_pool_entry	pelem;
 };
 
+struct rxe_xrcd {
+	struct ib_xrcd		ibxrcd;
+	struct rxe_pool_entry	pelem;
+};
+
 struct rxe_ah {
 	struct ib_ah		ibah;
 	struct rxe_pool_entry	pelem;
@@ -384,6 +389,7 @@ struct rxe_dev {
 	struct ib_device	ib_dev;
 	struct ib_device_attr	attr;
 	int			max_ucontext;
+	int			max_xrcd;
 	int			max_inline_data;
 	struct mutex	usdev_lock;
 
@@ -393,6 +399,7 @@ struct rxe_dev {
 
 	struct rxe_pool		uc_pool;
 	struct rxe_pool		pd_pool;
+	struct rxe_pool		xrcd_pool;
 	struct rxe_pool		ah_pool;
 	struct rxe_pool		srq_pool;
 	struct rxe_pool		qp_pool;
@@ -434,6 +441,11 @@ static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
 	return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
 }
 
+static inline struct rxe_xrcd *to_rxrcd(struct ib_xrcd *xrcd)
+{
+	return xrcd ? container_of(xrcd, struct rxe_xrcd, ibxrcd) : NULL;
+}
+
 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
 {
 	return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
-- 
2.30.2


  parent reply	other threads:[~2021-07-29 22:50 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-07-29 22:49 [PATCH for-next 00/13] RDMA:rxe: Implement XRC for rxe Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 01/13] RDMA/rxe: Decouple rxe_pkt_info from sk_buff Bob Pearson
2021-08-27 13:01   ` Jason Gunthorpe
2021-07-29 22:49 ` [PATCH for-next 02/13] IB/core: Add xrc opcodes to ib_pack.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 03/13] RDMA/rxe: Extend rxe_send_wr to support xrceth Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 04/13] RDMA/rxe: Extend rxe_opcode.h to support xrc Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 05/13] RDMA/rxe: Add XRC ETH to rxe_hdr.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 06/13] RDMA/rxe: Add XRC QP type to rxe_wr_opcode_info Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 07/13] RDMA/rxe: Add XRC opcodes to rxe_opcode_info Bob Pearson
2021-07-29 22:49 ` Bob Pearson [this message]
2021-07-29 22:49 ` [PATCH for-next 09/13] RDMA/rxe: Extend SRQs to support extensions Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 10/13] RDMA/rxe: Compute next opcode for XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 11/13] RDMA/rxe: Extend rxe_verbs and rxe_qp to support XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 12/13] RDMA/rxe: Extend rxe send XRC packets Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 13/13] RDMA/rxe: Enable receiving " Bob Pearson

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210729224915.38986-9-rpearsonhpe@gmail.com \
    --to=rpearsonhpe@gmail.com \
    --cc=jgg@nvidia.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=zyjzyj2000@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).