From: Bob Pearson <rpearsonhpe@gmail.com>
To: jgg@nvidia.com, zyjzyj2000@gmail.com, linux-rdma@vger.kernel.org
Cc: Bob Pearson <rpearsonhpe@gmail.com>
Subject: [PATCH for-next 08/13] RDMA/rxe: Support alloc/dealloc xrcd
Date: Thu, 29 Jul 2021 17:49:11 -0500 [thread overview]
Message-ID: <20210729224915.38986-9-rpearsonhpe@gmail.com> (raw)
In-Reply-To: <20210729224915.38986-1-rpearsonhpe@gmail.com>
Implement code to support ibv_alloc_xrcd and ibv_dealloc_xrcd verbs.
Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
---
drivers/infiniband/sw/rxe/rxe.c | 39 ++++++++++++++++-----------
drivers/infiniband/sw/rxe/rxe_param.h | 1 +
drivers/infiniband/sw/rxe/rxe_pool.c | 6 +++++
drivers/infiniband/sw/rxe/rxe_pool.h | 1 +
drivers/infiniband/sw/rxe/rxe_verbs.c | 19 +++++++++++++
drivers/infiniband/sw/rxe/rxe_verbs.h | 12 +++++++++
6 files changed, 63 insertions(+), 15 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 8e0f9c489cab..fbbb3d6f172b 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -24,6 +24,7 @@ void rxe_dealloc(struct ib_device *ib_dev)
rxe_pool_cleanup(&rxe->uc_pool);
rxe_pool_cleanup(&rxe->pd_pool);
+ rxe_pool_cleanup(&rxe->xrcd_pool);
rxe_pool_cleanup(&rxe->ah_pool);
rxe_pool_cleanup(&rxe->srq_pool);
rxe_pool_cleanup(&rxe->qp_pool);
@@ -74,6 +75,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
rxe->ndev->dev_addr);
rxe->max_ucontext = RXE_MAX_UCONTEXT;
+ rxe->max_xrcd = RXE_MAX_XRCD;
}
/* initialize port attributes */
@@ -130,62 +132,69 @@ static int rxe_init_pools(struct rxe_dev *rxe)
if (err)
goto err2;
+ err = rxe_pool_init(rxe, &rxe->xrcd_pool, RXE_TYPE_XRCD,
+ rxe->max_xrcd);
+ if (err)
+ goto err3;
+
err = rxe_pool_init(rxe, &rxe->ah_pool, RXE_TYPE_AH,
rxe->attr.max_ah);
if (err)
- goto err3;
+ goto err4;
err = rxe_pool_init(rxe, &rxe->srq_pool, RXE_TYPE_SRQ,
rxe->attr.max_srq);
if (err)
- goto err4;
+ goto err5;
err = rxe_pool_init(rxe, &rxe->qp_pool, RXE_TYPE_QP,
rxe->attr.max_qp);
if (err)
- goto err5;
+ goto err6;
err = rxe_pool_init(rxe, &rxe->cq_pool, RXE_TYPE_CQ,
rxe->attr.max_cq);
if (err)
- goto err6;
+ goto err7;
err = rxe_pool_init(rxe, &rxe->mr_pool, RXE_TYPE_MR,
rxe->attr.max_mr);
if (err)
- goto err7;
+ goto err8;
err = rxe_pool_init(rxe, &rxe->mw_pool, RXE_TYPE_MW,
rxe->attr.max_mw);
if (err)
- goto err8;
+ goto err9;
err = rxe_pool_init(rxe, &rxe->mc_grp_pool, RXE_TYPE_MC_GRP,
rxe->attr.max_mcast_grp);
if (err)
- goto err9;
+ goto err10;
err = rxe_pool_init(rxe, &rxe->mc_elem_pool, RXE_TYPE_MC_ELEM,
rxe->attr.max_total_mcast_qp_attach);
if (err)
- goto err10;
+ goto err11;
return 0;
-err10:
+err11:
rxe_pool_cleanup(&rxe->mc_grp_pool);
-err9:
+err10:
rxe_pool_cleanup(&rxe->mw_pool);
-err8:
+err9:
rxe_pool_cleanup(&rxe->mr_pool);
-err7:
+err8:
rxe_pool_cleanup(&rxe->cq_pool);
-err6:
+err7:
rxe_pool_cleanup(&rxe->qp_pool);
-err5:
+err6:
rxe_pool_cleanup(&rxe->srq_pool);
-err4:
+err5:
rxe_pool_cleanup(&rxe->ah_pool);
+err4:
+ rxe_pool_cleanup(&rxe->xrcd_pool);
err3:
rxe_pool_cleanup(&rxe->pd_pool);
err2:
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index ec5c6331bee8..b843be4cc25a 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -79,6 +79,7 @@ enum rxe_device_param {
RXE_LOCAL_CA_ACK_DELAY = 15,
RXE_MAX_UCONTEXT = 512,
+ RXE_MAX_XRCD = 512,
RXE_NUM_PORT = 1,
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 342f090152d1..76caef5790b0 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -22,6 +22,12 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
.elem_offset = offsetof(struct rxe_pd, pelem),
.flags = RXE_POOL_NO_ALLOC,
},
+ [RXE_TYPE_XRCD] = {
+ .name = "rxe-xrcd",
+ .size = sizeof(struct rxe_xrcd),
+ .elem_offset = offsetof(struct rxe_xrcd, pelem),
+ .flags = RXE_POOL_NO_ALLOC,
+ },
[RXE_TYPE_AH] = {
.name = "rxe-ah",
.size = sizeof(struct rxe_ah),
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.h b/drivers/infiniband/sw/rxe/rxe_pool.h
index 1feca1bffced..1475b9374315 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.h
+++ b/drivers/infiniband/sw/rxe/rxe_pool.h
@@ -19,6 +19,7 @@ enum rxe_pool_flags {
enum rxe_elem_type {
RXE_TYPE_UC,
RXE_TYPE_PD,
+ RXE_TYPE_XRCD,
RXE_TYPE_AH,
RXE_TYPE_SRQ,
RXE_TYPE_QP,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 7ff98a60ddcd..b4b993f1ce92 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -156,6 +156,22 @@ static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
return 0;
}
+static int rxe_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+ struct rxe_dev *rxe = to_rdev(ibxrcd->device);
+ struct rxe_xrcd *xrcd = to_rxrcd(ibxrcd);
+
+ return rxe_add_to_pool(&rxe->xrcd_pool, xrcd);
+}
+
+static int rxe_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
+{
+ struct rxe_xrcd *xrcd = to_rxrcd(ibxrcd);
+
+ rxe_drop_ref(xrcd);
+ return 0;
+}
+
static int rxe_create_ah(struct ib_ah *ibah,
struct rdma_ah_init_attr *init_attr,
struct ib_udata *udata)
@@ -1078,6 +1094,7 @@ static const struct ib_device_ops rxe_dev_ops = {
.alloc_mw = rxe_alloc_mw,
.alloc_pd = rxe_alloc_pd,
.alloc_ucontext = rxe_alloc_ucontext,
+ .alloc_xrcd = rxe_alloc_xrcd,
.attach_mcast = rxe_attach_mcast,
.create_ah = rxe_create_ah,
.create_cq = rxe_create_cq,
@@ -1088,6 +1105,7 @@ static const struct ib_device_ops rxe_dev_ops = {
.dealloc_mw = rxe_dealloc_mw,
.dealloc_pd = rxe_dealloc_pd,
.dealloc_ucontext = rxe_dealloc_ucontext,
+ .dealloc_xrcd = rxe_dealloc_xrcd,
.dereg_mr = rxe_dereg_mr,
.destroy_ah = rxe_destroy_ah,
.destroy_cq = rxe_destroy_cq,
@@ -1128,6 +1146,7 @@ static const struct ib_device_ops rxe_dev_ops = {
INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
+ INIT_RDMA_OBJ_SIZE(ib_xrcd, rxe_xrcd, ibxrcd),
};
int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 0a433f4c0222..5b75de74a992 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -43,6 +43,11 @@ struct rxe_pd {
struct rxe_pool_entry pelem;
};
+struct rxe_xrcd {
+ struct ib_xrcd ibxrcd;
+ struct rxe_pool_entry pelem;
+};
+
struct rxe_ah {
struct ib_ah ibah;
struct rxe_pool_entry pelem;
@@ -384,6 +389,7 @@ struct rxe_dev {
struct ib_device ib_dev;
struct ib_device_attr attr;
int max_ucontext;
+ int max_xrcd;
int max_inline_data;
struct mutex usdev_lock;
@@ -393,6 +399,7 @@ struct rxe_dev {
struct rxe_pool uc_pool;
struct rxe_pool pd_pool;
+ struct rxe_pool xrcd_pool;
struct rxe_pool ah_pool;
struct rxe_pool srq_pool;
struct rxe_pool qp_pool;
@@ -434,6 +441,11 @@ static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
}
+static inline struct rxe_xrcd *to_rxrcd(struct ib_xrcd *xrcd)
+{
+ return xrcd ? container_of(xrcd, struct rxe_xrcd, ibxrcd) : NULL;
+}
+
static inline struct rxe_ah *to_rah(struct ib_ah *ah)
{
return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
--
2.30.2
next prev parent reply other threads:[~2021-07-29 22:50 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-29 22:49 [PATCH for-next 00/13] RDMA:rxe: Implement XRC for rxe Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 01/13] RDMA/rxe: Decouple rxe_pkt_info from sk_buff Bob Pearson
2021-08-27 13:01 ` Jason Gunthorpe
2021-07-29 22:49 ` [PATCH for-next 02/13] IB/core: Add xrc opcodes to ib_pack.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 03/13] RDMA/rxe: Extend rxe_send_wr to support xrceth Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 04/13] RDMA/rxe: Extend rxe_opcode.h to support xrc Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 05/13] RDMA/rxe: Add XRC ETH to rxe_hdr.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 06/13] RDMA/rxe: Add XRC QP type to rxe_wr_opcode_info Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 07/13] RDMA/rxe: Add XRC opcodes to rxe_opcode_info Bob Pearson
2021-07-29 22:49 ` Bob Pearson [this message]
2021-07-29 22:49 ` [PATCH for-next 09/13] RDMA/rxe: Extend SRQs to support extensions Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 10/13] RDMA/rxe: Compute next opcode for XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 11/13] RDMA/rxe: Extend rxe_verbs and rxe_qp to support XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 12/13] RDMA/rxe: Extend rxe send XRC packets Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 13/13] RDMA/rxe: Enable receiving " Bob Pearson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210729224915.38986-9-rpearsonhpe@gmail.com \
--to=rpearsonhpe@gmail.com \
--cc=jgg@nvidia.com \
--cc=linux-rdma@vger.kernel.org \
--cc=zyjzyj2000@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).