[PATCH 4/4] RDMA/srpt: Increase max_send_sge

[Bart Van Assche <bvanassche@acm.org>]

The ib_srpt driver limits max_send_sge to 16. Since that is a workaround
for an mlx4 bug that has been fixed, increase max_send_sge. For mlx4, do
not use the value advertised by the driver (32) since that causes QP's
to transition to the error status.

See also commit f95ccffc715b ("IB/mlx4: Use 4K pages for kernel QP's WQE
buffer").

Cc: Laurence Oberman <loberman@redhat.com>
Cc: Kamal Heib <kamalheib1@gmail.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
---
 drivers/infiniband/ulp/srpt/ib_srpt.c | 25 +++++++++++++++++++++++--
 drivers/infiniband/ulp/srpt/ib_srpt.h |  5 -----
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 1ad3cc7c553a..8d9b8664ea75 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1781,8 +1781,30 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	struct srpt_device *sdev = sport->sdev;
 	const struct ib_device_attr *attrs = &sdev->device->attrs;
 	int sq_size = sport->port_attrib.srp_sq_size;
+	unsigned int max_sge_delta = 0;
 	int i, ret;
 
+	switch (sdev->device->ops.driver_id) {
+	case RDMA_DRIVER_MLX4:
+		/*
+		 * The smallest max_sge_delta value that works with
+		 * ConnectX-3 firmware version 2.42.5000.
+		 */
+		max_sge_delta = 2;
+		break;
+	case RDMA_DRIVER_MTHCA:
+		/*
+		 * From the OFED release notes: In mem-free devices, RC
+		 * QPs can be created with a maximum of (max_sge - 1)
+		 * entries only. See also
+		 * https://git.openfabrics.org/?p=compat-rdma/docs.git;a=blob;f=release_notes/mthca_release_notes.txt;h=40f3c4ea77a07fe5ded888b8417530471e89d87b.
+		 */
+		max_sge_delta = 1;
+		break;
+	default:
+		break;
+	}
+
 	WARN_ON(ch->rq_size < 1);
 
 	ret = -ENOMEM;
@@ -1816,8 +1838,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	 */
 	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
 	qp_init->cap.max_rdma_ctxs = sq_size / 2;
-	qp_init->cap.max_send_sge = min(attrs->max_send_sge,
-					SRPT_MAX_SG_PER_WQE);
+	qp_init->cap.max_send_sge = attrs->max_send_sge - max_sge_delta;
 	qp_init->cap.max_recv_sge = 1;
 	qp_init->port_num = ch->sport->port;
 	if (sdev->use_srq)
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 2e1a69840857..f31c349d07a1 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -105,11 +105,6 @@ enum {
 	SRP_CMD_ACA = 0x4,
 
 	SRPT_DEF_SG_TABLESIZE = 128,
-	/*
-	 * An experimentally determined value that avoids that QP creation
-	 * fails due to "swiotlb buffer is full" on systems using the swiotlb.
-	 */
-	SRPT_MAX_SG_PER_WQE = 16,
 
 	MIN_SRPT_SQ_SIZE = 16,
 	DEF_SRPT_SQ_SIZE = 4096,