From: Jack Wang <jinpu.wang@ionos.com>
To: linux-rdma@vger.kernel.org
Cc: bvanassche@acm.org, leon@kernel.org, dledford@redhat.com,
jgg@ziepe.ca, haris.iqbal@ionos.com, jinpu.wang@ionos.com,
Jack Wang <jinpu.wang@cloud.ionos.com>,
Guoqing Jiang <guoqing.jiang@cloud.ionos.com>,
Leon Romanovsky <leonro@nvidia.com>
Subject: [PATCHv3 for-next 2/5] RDMA/rtrs-clt: Use minimal max_send_sge when create qp
Date: Mon, 14 Jun 2021 11:03:34 +0200 [thread overview]
Message-ID: <20210614090337.29557-3-jinpu.wang@ionos.com> (raw)
In-Reply-To: <20210614090337.29557-1-jinpu.wang@ionos.com>
From: Jack Wang <jinpu.wang@cloud.ionos.com>
We use device limit max_send_sge, which is suboptimal for memory usage.
We don't need that much for User Con, 1 is enough. And for IO con,
sess->max_segments + 1 is enough
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Signed-off-by: Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
---
drivers/infiniband/ulp/rtrs/rtrs-clt.c | 14 ++++++++------
drivers/infiniband/ulp/rtrs/rtrs-clt.h | 1 -
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index f1fd7ae9ac53..cd53edddfe1f 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -1417,7 +1417,6 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess)
sess->max_pages_per_mr =
min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,
ib_dev->attrs.max_fast_reg_page_list_len);
- sess->max_send_sge = ib_dev->attrs.max_send_sge;
}
static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,
@@ -1573,7 +1572,7 @@ static void destroy_con(struct rtrs_clt_con *con)
static int create_con_cq_qp(struct rtrs_clt_con *con)
{
struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
- u32 max_send_wr, max_recv_wr, cq_size;
+ u32 max_send_wr, max_recv_wr, cq_size, max_send_sge;
int err, cq_vector;
struct rtrs_msg_rkey_rsp *rsp;
@@ -1587,6 +1586,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
*/
max_send_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;
max_recv_wr = SERVICE_CON_QUEUE_DEPTH * 2 + 2;
+ max_send_sge = 1;
/* We must be the first here */
if (WARN_ON(sess->s.dev))
return -EINVAL;
@@ -1625,25 +1625,27 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
max_recv_wr =
min_t(int, sess->s.dev->ib_dev->attrs.max_qp_wr,
sess->queue_depth * 3 + 1);
+ max_send_sge = sess->clt->max_segments + 1;
}
+ cq_size = max_send_wr + max_recv_wr;
/* alloc iu to recv new rkey reply when server reports flags set */
if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {
- con->rsp_ius = rtrs_iu_alloc(max_recv_wr, sizeof(*rsp),
+ con->rsp_ius = rtrs_iu_alloc(cq_size, sizeof(*rsp),
GFP_KERNEL, sess->s.dev->ib_dev,
DMA_FROM_DEVICE,
rtrs_clt_rdma_done);
if (!con->rsp_ius)
return -ENOMEM;
- con->queue_size = max_recv_wr;
+ con->queue_size = cq_size;
}
cq_size = max_send_wr + max_recv_wr;
cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors;
if (con->c.cid >= sess->s.irq_con_num)
- err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,
+ err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,
cq_vector, cq_size, max_send_wr,
max_recv_wr, IB_POLL_DIRECT);
else
- err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,
+ err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge,
cq_vector, cq_size, max_send_wr,
max_recv_wr, IB_POLL_SOFTIRQ);
/*
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
index 919c9f96f25b..822a820540d4 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
@@ -141,7 +141,6 @@ struct rtrs_clt_sess {
u32 chunk_size;
size_t queue_depth;
u32 max_pages_per_mr;
- int max_send_sge;
u32 flags;
struct kobject kobj;
u8 for_new_clt;
--
2.25.1
next prev parent reply other threads:[~2021-06-14 9:03 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-14 9:03 [PATCHv3 for-next 0/5] Misc update for RTRS Jack Wang
2021-06-14 9:03 ` [PATCHv3 for-next 1/5] RDMA/rtrs-srv: Set minimal max_send_wr and max_recv_wr Jack Wang
2021-06-14 9:03 ` Jack Wang [this message]
2021-06-14 9:03 ` [PATCHv3 for-next 3/5] RDMA/rtrs: RDMA_RXE requires more number of WR Jack Wang
2021-06-14 9:03 ` [PATCHv3 for-next 4/5] RDMA/rtrs: Rename cq_size/queue_size to cq_num/queue_num Jack Wang
2021-06-14 9:03 ` [PATCHv3 for-next 5/5] RDMA/rtrs: Check device max_qp_wr limit when create QP Jack Wang
2021-06-18 16:51 ` [PATCHv3 for-next 0/5] Misc update for RTRS Jason Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210614090337.29557-3-jinpu.wang@ionos.com \
--to=jinpu.wang@ionos.com \
--cc=bvanassche@acm.org \
--cc=dledford@redhat.com \
--cc=guoqing.jiang@cloud.ionos.com \
--cc=haris.iqbal@ionos.com \
--cc=jgg@ziepe.ca \
--cc=jinpu.wang@cloud.ionos.com \
--cc=leon@kernel.org \
--cc=leonro@nvidia.com \
--cc=linux-rdma@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.