[PATCH rdma-core 0/7] libhns: Bugfix for hip08

[PATCH rdma-core 0/7] libhns: Bugfix for hip08

[Weihang Li]

Various fixes on library of hip08.

PR: https://github.com/linux-rdma/rdma-core/pull/623

Lijun Ou (4):
  libhns: Bugfix for assigning sl
  libhns: Bugfix for cleaning cq
  libhns: Bugfix for updating qp params
  libhns: Avoid null pointer operation

Weihang Li (1):
  libhns: Fix calculation errors with ilog32()

Xi Wang (1):
  libhns: Optimize bind_mw for fixing null pointer access

Yangyang Li (1):
  libhns: Return correct value of cqe num when flushing cqe failed

 providers/hns/hns_roce_u_hw_v2.c | 33 +++++++++++++++++++++++----------
 providers/hns/hns_roce_u_verbs.c | 32 +++++++++++++++++++-------------
 2 files changed, 42 insertions(+), 23 deletions(-)

-- 
2.8.1

[PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()

[Weihang Li]

Current calculation results using ilog32() is larger than expected, which
will lead to driver broken. The following is the log when QP creations
fails:

[   81.294844] hns3 0000:7d:00.0 hns_0: check SQ size error!
[   81.294848] hns3 0000:7d:00.0 hns_0: check SQ size error!
[   81.300225] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
[   81.300227] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for create qp
[   81.305602] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
[   81.305603] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for create qp
[   81.311589] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000 failed(-22)
[   81.318603] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000 failed(-22)

Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_verbs.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 9d222c0..bd5060d 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -645,7 +645,8 @@ static int hns_roce_calc_qp_buff_size(struct ibv_pd *pd, struct ibv_qp_cap *cap,
 	int page_size = to_hr_dev(pd->context->device)->page_size;
 
 	if (to_hr_dev(pd->context->device)->hw_version == HNS_ROCE_HW_VER1) {
-		qp->rq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_rq_wqe));
+		qp->rq.wqe_shift =
+				ilog32(sizeof(struct hns_roce_rc_rq_wqe)) - 1;
 
 		qp->buf_size = align((qp->sq.wqe_cnt << qp->sq.wqe_shift),
 				     page_size) +
@@ -662,7 +663,7 @@ static int hns_roce_calc_qp_buff_size(struct ibv_pd *pd, struct ibv_qp_cap *cap,
 	} else {
 		unsigned int rqwqe_size = HNS_ROCE_SGE_SIZE * cap->max_recv_sge;
 
-		qp->rq.wqe_shift = ilog32(rqwqe_size);
+		qp->rq.wqe_shift = ilog32(rqwqe_size) - 1;
 
 		if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE || type == IBV_QPT_UD)
 			qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
@@ -747,8 +748,8 @@ static void hns_roce_set_qp_params(struct ibv_pd *pd,
 		qp->rq.wqe_cnt = roundup_pow_of_two(attr->cap.max_recv_wr);
 	}
 
-	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe));
-	qp->sq.shift = ilog32(qp->sq.wqe_cnt);
+	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe)) - 1;
+	qp->sq.shift = ilog32(qp->sq.wqe_cnt) - 1;
 	qp->rq.max_gs = attr->cap.max_recv_sge;
 
 	if (to_hr_dev(pd->context->device)->hw_version == HNS_ROCE_HW_VER1) {
@@ -884,7 +885,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
 
 	cmd.buf_addr = (uintptr_t) qp->buf.buf;
 	cmd.log_sq_stride = qp->sq.wqe_shift;
-	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt);
+	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt) - 1;
 
 	pthread_mutex_lock(&context->qp_table_mutex);
 
-- 
2.8.1

[PATCH rdma-core 2/7] libhns: Optimize bind_mw for fixing null pointer access

[Weihang Li]

From: Xi Wang <wangxi11@huawei.com>

The argument checking flow in hns_roce_u_bind_mw() will leads to access on
a null address when the mr is not initialized in mw_bind.

Fixes: 47eff6e8624d ("libhns: Adjust the order of parameter checking")
Signed-off-by: Xi Wang <wangxi11@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_verbs.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index bd5060d..0acfd9a 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -186,7 +186,10 @@ int hns_roce_u_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
 	if (!bind_info->mr && bind_info->length)
 		return EINVAL;
 
-	if ((mw->pd != qp->pd) || (mw->pd != bind_info->mr->pd))
+	if (mw->pd != qp->pd)
+		return EINVAL;
+
+	if (bind_info->mr && (mw->pd != bind_info->mr->pd))
 		return EINVAL;
 
 	if (mw->type != IBV_MW_TYPE_1)
-- 
2.8.1

[PATCH rdma-core 3/7] libhns: Bugfix for assigning sl

[Weihang Li]

From: Lijun Ou <oulijun@huawei.com>

Only the field that corresponding mask is set can be modified, or its
original value may be covered.

Fixes: c24583975044 ("libhns: Add verbs of qp support")
Signed-off-by: Lijun Ou <oulijun@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_hw_v2.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 931f59d..b473f07 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -1103,8 +1103,10 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		pthread_spin_unlock(&hr_qp->sq.lock);
 	}
 
-	if (!ret && (attr_mask & IBV_QP_STATE) &&
-	    attr->qp_state == IBV_QPS_RESET) {
+	if (ret)
+		return ret;
+
+	if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET) {
 		hns_roce_v2_cq_clean(to_hr_cq(qp->recv_cq), qp->qp_num,
 				     qp->srq ? to_hr_srq(qp->srq) : NULL);
 		if (qp->send_cq != qp->recv_cq)
@@ -1114,10 +1116,11 @@ static int hns_roce_u_v2_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		hns_roce_init_qp_indices(to_hr_qp(qp));
 	}
 
-	if (!ret && (attr_mask & IBV_QP_PORT))
+	if (attr_mask & IBV_QP_PORT)
 		hr_qp->port_num = attr->port_num;
 
-	hr_qp->sl = attr->ah_attr.sl;
+	if (attr_mask & IBV_QP_AV)
+		hr_qp->sl = attr->ah_attr.sl;
 
 	return ret;
 }
-- 
2.8.1

[PATCH rdma-core 4/7] libhns: Bugfix for cleaning cq

[Weihang Li]

From: Lijun Ou <oulijun@huawei.com>

When a qp attached with a srq needs to be modified to reset or error
state, the corresponding cq should be cleaned and then the wqe placeholder
of the srq can be released.

Fixes: 6fe30a1a705f ("libhns: Introduce QP operations referred to hip08 RoCE device)
Signed-off-by: Lijun Ou <oulijun@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_hw_v2.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index b473f07..7f5a2ce 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -1040,6 +1040,8 @@ out:
 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
 				   struct hns_roce_srq *srq)
 {
+	bool is_rcqe;
+	int wqe_index;
 	int nfreed = 0;
 	uint32_t prod_index;
 	uint8_t owner_bit = 0;
@@ -1055,6 +1057,14 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
 		cqe = get_cqe_v2(cq, prod_index & cq->ibv_cq.cqe);
 		if ((roce_get_field(cqe->byte_16, CQE_BYTE_16_LCL_QPN_M,
 			      CQE_BYTE_16_LCL_QPN_S) & 0xffffff) == qpn) {
+			is_rcqe = roce_get_bit(cqe->byte_4, CQE_BYTE_4_S_R_S);
+
+			if (srq && is_rcqe) {
+				wqe_index = roce_get_field(cqe->byte_4,
+						CQE_BYTE_4_WQE_IDX_M,
+						CQE_BYTE_4_WQE_IDX_S);
+				hns_roce_free_srq_wqe(srq, wqe_index);
+			}
 			++nfreed;
 		} else if (nfreed) {
 			dest = get_cqe_v2(cq,
-- 
2.8.1

[PATCH rdma-core 5/7] libhns: Bugfix for updating qp params

[Weihang Li]

From: Lijun Ou <oulijun@huawei.com>

In hns_roce_u_create_qp(), it's necessary to update rq params after
ibv_cmd_create_qp() running successfully, and then we will use it when
posting work request.

Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
Signed-off-by: Lijun Ou <oulijun@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_verbs.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 0acfd9a..a9c78f8 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -771,19 +771,14 @@ static void hns_roce_set_qp_params(struct ibv_pd *pd,
 	}
 
 	/* limit by the context queried during alloc context */
-	qp->rq.max_post = min(ctx->max_qp_wr, qp->rq.wqe_cnt);
 	qp->sq.max_post = min(ctx->max_qp_wr, qp->sq.wqe_cnt);
 	qp->sq.max_gs = min(ctx->max_sge, qp->sq.max_gs);
-	qp->rq.max_gs = min(ctx->max_sge, qp->rq.max_gs);
 
 	qp->sq_signal_bits = attr->sq_sig_all ? 0 : 1;
-	qp->max_inline_data  = HNS_ROCE_MAX_INLINE_DATA_LEN;
+	qp->max_inline_data = HNS_ROCE_MAX_INLINE_DATA_LEN;
 
 	/* update attr for creating qp */
 	attr->cap.max_send_wr = qp->sq.max_post;
-	attr->cap.max_recv_wr = qp->rq.max_post;
-	attr->cap.max_send_sge = qp->sq.max_gs;
-	attr->cap.max_recv_sge = qp->rq.max_gs;
 	attr->cap.max_inline_data = qp->max_inline_data;
 }
 
@@ -906,7 +901,14 @@ struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
 	}
 	pthread_mutex_unlock(&context->qp_table_mutex);
 
-	qp->flags	= resp.cap_flags;
+	/* adjust rq maxima to not exceed reported device maxima */
+	attr->cap.max_recv_wr = min(context->max_qp_wr, attr->cap.max_recv_wr);
+	attr->cap.max_recv_sge = min(context->max_sge, attr->cap.max_recv_sge);
+	qp->rq.wqe_cnt = attr->cap.max_recv_wr;
+	qp->rq.max_gs = attr->cap.max_recv_sge;
+	qp->rq.max_post = attr->cap.max_recv_wr;
+
+	qp->flags = resp.cap_flags;
 
 	return &qp->ibv_qp;
 
-- 
2.8.1

[PATCH rdma-core 6/7] libhns: Avoid null pointer operation

[Weihang Li]

From: Lijun Ou <oulijun@huawei.com>

When recv cq or send cq doesn't exist, clean cq will lead to a null pointer
error in hns_roce_u_v2_destroy_qp().

Fixes: 6fe30a1a705f ("libhns: Introduce QP operations referred to hip08 RoCE device")
Signed-off-by: Lijun Ou <oulijun@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_hw_v2.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 7f5a2ce..64dea8e 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -1181,10 +1181,11 @@ static int hns_roce_u_v2_destroy_qp(struct ibv_qp *ibqp)
 
 	hns_roce_lock_cqs(ibqp);
 
-	__hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), ibqp->qp_num,
-			       ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
+	if (ibqp->recv_cq)
+		__hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), ibqp->qp_num,
+				       ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
 
-	if (ibqp->send_cq != ibqp->recv_cq)
+	if (ibqp->send_cq && ibqp->send_cq != ibqp->recv_cq)
 		__hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), ibqp->qp_num,
 				       NULL);
 
-- 
2.8.1

[PATCH rdma-core 7/7] libhns: Return correct value of cqe num when flushing cqe failed

[Weihang Li]

From: Yangyang Li <liyangyang20@huawei.com>

When flushing cqe failed, it will return a error code to
hns_roce_v2_poll_one() and no longer update cqe number which is necessary
for ULPs, that will lead to a process suspension.
Because error code of flush cqe is meaningless for ULPs, so we delete it.

Fixes: 321ec6d04c0b ("libhns: Package for polling cqe function")
Signed-off-by: Yangyang Li <liyangyang20@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 providers/hns/hns_roce_u_hw_v2.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c
index 64dea8e..a8d3d11 100644
--- a/providers/hns/hns_roce_u_hw_v2.c
+++ b/providers/hns/hns_roce_u_hw_v2.c
@@ -287,10 +287,9 @@ static int hns_roce_flush_cqe(struct hns_roce_qp **cur_qp, struct ibv_wc *wc)
 		attr.qp_state = IBV_QPS_ERR;
 		ret = hns_roce_u_v2_modify_qp(&(*cur_qp)->ibv_qp,
 						      &attr, attr_mask);
-		if (ret) {
+		if (ret)
 			fprintf(stderr, PFX "failed to modify qp!\n");
-			return ret;
-		}
+
 		(*cur_qp)->ibv_qp.state = IBV_QPS_ERR;
 	}
 
-- 
2.8.1

RE: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()

[Zengtao (B)]

> -----Original Message-----
> From: linux-rdma-owner@vger.kernel.org
> [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
> Sent: Thursday, November 21, 2019 9:19 AM
> To: jgg@ziepe.ca; leon@kernel.org
> Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
> Subject: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()
> 
> Current calculation results using ilog32() is larger than expected, which
> will lead to driver broken. The following is the log when QP creations
> fails:
> 
> [   81.294844] hns3 0000:7d:00.0 hns_0: check SQ size error!
> [   81.294848] hns3 0000:7d:00.0 hns_0: check SQ size error!
> [   81.300225] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
> [   81.300227] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
> create qp
> [   81.305602] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
> [   81.305603] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
> create qp
> [   81.311589] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
> failed(-22)
> [   81.318603] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
> failed(-22)
> 
> Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
> ---
>  providers/hns/hns_roce_u_verbs.c | 11 ++++++-----
>  1 file changed, 6 insertions(+), 5 deletions(-)
> 
> diff --git a/providers/hns/hns_roce_u_verbs.c
> b/providers/hns/hns_roce_u_verbs.c
> index 9d222c0..bd5060d 100644
> --- a/providers/hns/hns_roce_u_verbs.c
> +++ b/providers/hns/hns_roce_u_verbs.c
> @@ -645,7 +645,8 @@ static int hns_roce_calc_qp_buff_size(struct
> ibv_pd *pd, struct ibv_qp_cap *cap,
>  	int page_size = to_hr_dev(pd->context->device)->page_size;
> 
>  	if (to_hr_dev(pd->context->device)->hw_version ==
> HNS_ROCE_HW_VER1) {
> -		qp->rq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_rq_wqe));
> +		qp->rq.wqe_shift =
> +				ilog32(sizeof(struct hns_roce_rc_rq_wqe)) - 1;
> 
>  		qp->buf_size = align((qp->sq.wqe_cnt << qp->sq.wqe_shift),
>  				     page_size) +
> @@ -662,7 +663,7 @@ static int hns_roce_calc_qp_buff_size(struct
> ibv_pd *pd, struct ibv_qp_cap *cap,
>  	} else {
>  		unsigned int rqwqe_size = HNS_ROCE_SGE_SIZE *
> cap->max_recv_sge;
> 
> -		qp->rq.wqe_shift = ilog32(rqwqe_size);
> +		qp->rq.wqe_shift = ilog32(rqwqe_size) - 1;
> 
>  		if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE || type ==
> IBV_QPT_UD)
>  			qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
> @@ -747,8 +748,8 @@ static void hns_roce_set_qp_params(struct
> ibv_pd *pd,
>  		qp->rq.wqe_cnt =
> roundup_pow_of_two(attr->cap.max_recv_wr);
>  	}
> 
> -	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe));
> -	qp->sq.shift = ilog32(qp->sq.wqe_cnt);
> +	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe)) - 1;
> +	qp->sq.shift = ilog32(qp->sq.wqe_cnt) - 1;

One suggestion, it's better to introduce a new micro instead of ilog32(x) -1.

>  	qp->rq.max_gs = attr->cap.max_recv_sge;
> 
>  	if (to_hr_dev(pd->context->device)->hw_version ==
> HNS_ROCE_HW_VER1) {
> @@ -884,7 +885,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct
> ibv_pd *pd,
> 
>  	cmd.buf_addr = (uintptr_t) qp->buf.buf;
>  	cmd.log_sq_stride = qp->sq.wqe_shift;
> -	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt);
> +	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt) - 1;
> 
>  	pthread_mutex_lock(&context->qp_table_mutex);
> 
> --
> 2.8.1

RE: [PATCH rdma-core 2/7] libhns: Optimize bind_mw for fixing null pointer access

[Zengtao (B)]

> -----Original Message-----
> From: linux-rdma-owner@vger.kernel.org
> [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
> Sent: Thursday, November 21, 2019 9:19 AM
> To: jgg@ziepe.ca; leon@kernel.org
> Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
> Subject: [PATCH rdma-core 2/7] libhns: Optimize bind_mw for fixing null
> pointer access
> 
> From: Xi Wang <wangxi11@huawei.com>
> 
> The argument checking flow in hns_roce_u_bind_mw() will leads to access
> on
> a null address when the mr is not initialized in mw_bind.
> 
> Fixes: 47eff6e8624d ("libhns: Adjust the order of parameter checking")
> Signed-off-by: Xi Wang <wangxi11@huawei.com>
> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
> ---
>  providers/hns/hns_roce_u_verbs.c | 5 ++++-
>  1 file changed, 4 insertions(+), 1 deletion(-)
> 
> diff --git a/providers/hns/hns_roce_u_verbs.c
> b/providers/hns/hns_roce_u_verbs.c
> index bd5060d..0acfd9a 100644
> --- a/providers/hns/hns_roce_u_verbs.c
> +++ b/providers/hns/hns_roce_u_verbs.c
> @@ -186,7 +186,10 @@ int hns_roce_u_bind_mw(struct ibv_qp *qp,
> struct ibv_mw *mw,
>  	if (!bind_info->mr && bind_info->length)
>  		return EINVAL;
> 
> -	if ((mw->pd != qp->pd) || (mw->pd != bind_info->mr->pd))
> +	if (mw->pd != qp->pd)
> +		return EINVAL;
> +
> +	if (bind_info->mr && (mw->pd != bind_info->mr->pd))
>  		return EINVAL;
> 
Errno should also be set properly in this function, please refer to:
http://man7.org/linux/man-pages/man3/ibv_bind_mw.3.html

>  	if (mw->type != IBV_MW_TYPE_1)
> --
> 2.8.1

Re: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()

[Weihang Li]

On 2019/11/22 10:58, Zengtao (B) wrote:
>> -----Original Message-----
>> From: linux-rdma-owner@vger.kernel.org
>> [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
>> Sent: Thursday, November 21, 2019 9:19 AM
>> To: jgg@ziepe.ca; leon@kernel.org
>> Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
>> Subject: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()
>>
>> Current calculation results using ilog32() is larger than expected, which
>> will lead to driver broken. The following is the log when QP creations
>> fails:
>>
>> [   81.294844] hns3 0000:7d:00.0 hns_0: check SQ size error!
>> [   81.294848] hns3 0000:7d:00.0 hns_0: check SQ size error!
>> [   81.300225] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
>> [   81.300227] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
>> create qp
>> [   81.305602] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
>> [   81.305603] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
>> create qp
>> [   81.311589] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
>> failed(-22)
>> [   81.318603] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
>> failed(-22)
>>
>> Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
>> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
>> ---
>>  providers/hns/hns_roce_u_verbs.c | 11 ++++++-----
>>  1 file changed, 6 insertions(+), 5 deletions(-)
>>
>> diff --git a/providers/hns/hns_roce_u_verbs.c
>> b/providers/hns/hns_roce_u_verbs.c
>> index 9d222c0..bd5060d 100644
>> --- a/providers/hns/hns_roce_u_verbs.c
>> +++ b/providers/hns/hns_roce_u_verbs.c
>> @@ -645,7 +645,8 @@ static int hns_roce_calc_qp_buff_size(struct
>> ibv_pd *pd, struct ibv_qp_cap *cap,
>>  	int page_size = to_hr_dev(pd->context->device)->page_size;
>>
>>  	if (to_hr_dev(pd->context->device)->hw_version ==
>> HNS_ROCE_HW_VER1) {
>> -		qp->rq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_rq_wqe));
>> +		qp->rq.wqe_shift =
>> +				ilog32(sizeof(struct hns_roce_rc_rq_wqe)) - 1;
>>
>>  		qp->buf_size = align((qp->sq.wqe_cnt << qp->sq.wqe_shift),
>>  				     page_size) +
>> @@ -662,7 +663,7 @@ static int hns_roce_calc_qp_buff_size(struct
>> ibv_pd *pd, struct ibv_qp_cap *cap,
>>  	} else {
>>  		unsigned int rqwqe_size = HNS_ROCE_SGE_SIZE *
>> cap->max_recv_sge;
>>
>> -		qp->rq.wqe_shift = ilog32(rqwqe_size);
>> +		qp->rq.wqe_shift = ilog32(rqwqe_size) - 1;
>>
>>  		if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE || type ==
>> IBV_QPT_UD)
>>  			qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
>> @@ -747,8 +748,8 @@ static void hns_roce_set_qp_params(struct
>> ibv_pd *pd,
>>  		qp->rq.wqe_cnt =
>> roundup_pow_of_two(attr->cap.max_recv_wr);
>>  	}
>>
>> -	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe));
>> -	qp->sq.shift = ilog32(qp->sq.wqe_cnt);
>> +	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe)) - 1;
>> +	qp->sq.shift = ilog32(qp->sq.wqe_cnt) - 1;
> 
> One suggestion, it's better to introduce a new micro instead of ilog32(x) -1.

OK, thank you for your advice.

Weihang

> 
>>  	qp->rq.max_gs = attr->cap.max_recv_sge;
>>
>>  	if (to_hr_dev(pd->context->device)->hw_version ==
>> HNS_ROCE_HW_VER1) {
>> @@ -884,7 +885,7 @@ struct ibv_qp *hns_roce_u_create_qp(struct
>> ibv_pd *pd,
>>
>>  	cmd.buf_addr = (uintptr_t) qp->buf.buf;
>>  	cmd.log_sq_stride = qp->sq.wqe_shift;
>> -	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt);
>> +	cmd.log_sq_bb_count = ilog32(qp->sq.wqe_cnt) - 1;
>>
>>  	pthread_mutex_lock(&context->qp_table_mutex);
>>
>> --
>> 2.8.1
> 
> 
> .
> 

Re: [PATCH rdma-core 2/7] libhns: Optimize bind_mw for fixing null pointer access

[Weihang Li]

On 2019/11/22 11:02, Zengtao (B) wrote:
>> -----Original Message-----
>> From: linux-rdma-owner@vger.kernel.org
>> [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
>> Sent: Thursday, November 21, 2019 9:19 AM
>> To: jgg@ziepe.ca; leon@kernel.org
>> Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
>> Subject: [PATCH rdma-core 2/7] libhns: Optimize bind_mw for fixing null
>> pointer access
>>
>> From: Xi Wang <wangxi11@huawei.com>
>>
>> The argument checking flow in hns_roce_u_bind_mw() will leads to access
>> on
>> a null address when the mr is not initialized in mw_bind.
>>
>> Fixes: 47eff6e8624d ("libhns: Adjust the order of parameter checking")
>> Signed-off-by: Xi Wang <wangxi11@huawei.com>
>> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
>> ---
>>  providers/hns/hns_roce_u_verbs.c | 5 ++++-
>>  1 file changed, 4 insertions(+), 1 deletion(-)
>>
>> diff --git a/providers/hns/hns_roce_u_verbs.c
>> b/providers/hns/hns_roce_u_verbs.c
>> index bd5060d..0acfd9a 100644
>> --- a/providers/hns/hns_roce_u_verbs.c
>> +++ b/providers/hns/hns_roce_u_verbs.c
>> @@ -186,7 +186,10 @@ int hns_roce_u_bind_mw(struct ibv_qp *qp,
>> struct ibv_mw *mw,
>>  	if (!bind_info->mr && bind_info->length)
>>  		return EINVAL;
>>
>> -	if ((mw->pd != qp->pd) || (mw->pd != bind_info->mr->pd))
>> +	if (mw->pd != qp->pd)
>> +		return EINVAL;
>> +
>> +	if (bind_info->mr && (mw->pd != bind_info->mr->pd))
>>  		return EINVAL;
>>
> Errno should also be set properly in this function, please refer to:
> http://man7.org/linux/man-pages/man3/ibv_bind_mw.3.html
> 

Hi Zengtao,

Do you mean that all these conditions should return errno different
with each other?

IMHO, EINVAL is OK here, because all these returns is caused by "Invalid
Argument"

Thank you
Weihang


>>  	if (mw->type != IBV_MW_TYPE_1)
>> --
>> 2.8.1
> 
> 
> .
> 

Re: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()

[Jason Gunthorpe]

On Fri, Nov 22, 2019 at 02:58:44AM +0000, Zengtao (B) wrote:
> > From: linux-rdma-owner@vger.kernel.org
> > [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
> > Sent: Thursday, November 21, 2019 9:19 AM
> > To: jgg@ziepe.ca; leon@kernel.org
> > Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
> > Subject: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()
> > 
> > Current calculation results using ilog32() is larger than expected, which
> > will lead to driver broken. The following is the log when QP creations
> > fails:
> > 
> > [   81.294844] hns3 0000:7d:00.0 hns_0: check SQ size error!
> > [   81.294848] hns3 0000:7d:00.0 hns_0: check SQ size error!
> > [   81.300225] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
> > [   81.300227] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
> > create qp
> > [   81.305602] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
> > [   81.305603] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
> > create qp
> > [   81.311589] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
> > failed(-22)
> > [   81.318603] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
> > failed(-22)
> > 
> > Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
> > Signed-off-by: Weihang Li <liweihang@hisilicon.com>
> >  providers/hns/hns_roce_u_verbs.c | 11 ++++++-----
> >  1 file changed, 6 insertions(+), 5 deletions(-)
> > 
> > diff --git a/providers/hns/hns_roce_u_verbs.c
> > b/providers/hns/hns_roce_u_verbs.c
> > index 9d222c0..bd5060d 100644
> > +++ b/providers/hns/hns_roce_u_verbs.c
> > @@ -645,7 +645,8 @@ static int hns_roce_calc_qp_buff_size(struct
> > ibv_pd *pd, struct ibv_qp_cap *cap,
> >  	int page_size = to_hr_dev(pd->context->device)->page_size;
> > 
> >  	if (to_hr_dev(pd->context->device)->hw_version ==
> > HNS_ROCE_HW_VER1) {
> > -		qp->rq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_rq_wqe));
> > +		qp->rq.wqe_shift =
> > +				ilog32(sizeof(struct hns_roce_rc_rq_wqe)) - 1;
> > 
> >  		qp->buf_size = align((qp->sq.wqe_cnt << qp->sq.wqe_shift),
> >  				     page_size) +
> > @@ -662,7 +663,7 @@ static int hns_roce_calc_qp_buff_size(struct
> > ibv_pd *pd, struct ibv_qp_cap *cap,
> >  	} else {
> >  		unsigned int rqwqe_size = HNS_ROCE_SGE_SIZE *
> > cap->max_recv_sge;
> > 
> > -		qp->rq.wqe_shift = ilog32(rqwqe_size);
> > +		qp->rq.wqe_shift = ilog32(rqwqe_size) - 1;
> > 
> >  		if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE || type ==
> > IBV_QPT_UD)
> >  			qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
> > @@ -747,8 +748,8 @@ static void hns_roce_set_qp_params(struct
> > ibv_pd *pd,
> >  		qp->rq.wqe_cnt =
> > roundup_pow_of_two(attr->cap.max_recv_wr);
> >  	}
> > 
> > -	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe));
> > -	qp->sq.shift = ilog32(qp->sq.wqe_cnt);
> > +	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe)) - 1;
> > +	qp->sq.shift = ilog32(qp->sq.wqe_cnt) - 1;
> 
> One suggestion, it's better to introduce a new micro instead of ilog32(x) -1.

Is the -1 even correct?

I would have guessed something called shift wants to be:

   shift = ilog32(qp->sq.wqe_cnt - 1)

Such that 
   1 << shift == wqe_cnt
       When wqe_cnt is a power of two.
   1 << shift > wqe_cnt
       When wqe_cnt is not a power of two.

Jason

Re: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()

[Weihang Li]

On 2019/11/23 2:09, Jason Gunthorpe wrote:
> On Fri, Nov 22, 2019 at 02:58:44AM +0000, Zengtao (B) wrote:
>>> From: linux-rdma-owner@vger.kernel.org
>>> [mailto:linux-rdma-owner@vger.kernel.org] On Behalf Of Weihang Li
>>> Sent: Thursday, November 21, 2019 9:19 AM
>>> To: jgg@ziepe.ca; leon@kernel.org
>>> Cc: dledford@redhat.com; linux-rdma@vger.kernel.org; Linuxarm
>>> Subject: [PATCH rdma-core 1/7] libhns: Fix calculation errors with ilog32()
>>>
>>> Current calculation results using ilog32() is larger than expected, which
>>> will lead to driver broken. The following is the log when QP creations
>>> fails:
>>>
>>> [   81.294844] hns3 0000:7d:00.0 hns_0: check SQ size error!
>>> [   81.294848] hns3 0000:7d:00.0 hns_0: check SQ size error!
>>> [   81.300225] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
>>> [   81.300227] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
>>> create qp
>>> [   81.305602] hns3 0000:7d:00.0 hns_0: Sanity check sq size failed
>>> [   81.305603] hns3 0000:7d:00.0: hns_roce_set_user_sq_size error for
>>> create qp
>>> [   81.311589] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
>>> failed(-22)
>>> [   81.318603] hns3 0000:7d:00.0 hns_0: Create RC QP 0x000000
>>> failed(-22)
>>>
>>> Fixes: b6cd213b276f ("libhns: Refactor for creating qp")
>>> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
>>>  providers/hns/hns_roce_u_verbs.c | 11 ++++++-----
>>>  1 file changed, 6 insertions(+), 5 deletions(-)
>>>
>>> diff --git a/providers/hns/hns_roce_u_verbs.c
>>> b/providers/hns/hns_roce_u_verbs.c
>>> index 9d222c0..bd5060d 100644
>>> +++ b/providers/hns/hns_roce_u_verbs.c
>>> @@ -645,7 +645,8 @@ static int hns_roce_calc_qp_buff_size(struct
>>> ibv_pd *pd, struct ibv_qp_cap *cap,
>>>  	int page_size = to_hr_dev(pd->context->device)->page_size;
>>>
>>>  	if (to_hr_dev(pd->context->device)->hw_version ==
>>> HNS_ROCE_HW_VER1) {
>>> -		qp->rq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_rq_wqe));
>>> +		qp->rq.wqe_shift =
>>> +				ilog32(sizeof(struct hns_roce_rc_rq_wqe)) - 1;
>>>
>>>  		qp->buf_size = align((qp->sq.wqe_cnt << qp->sq.wqe_shift),
>>>  				     page_size) +
>>> @@ -662,7 +663,7 @@ static int hns_roce_calc_qp_buff_size(struct
>>> ibv_pd *pd, struct ibv_qp_cap *cap,
>>>  	} else {
>>>  		unsigned int rqwqe_size = HNS_ROCE_SGE_SIZE *
>>> cap->max_recv_sge;
>>>
>>> -		qp->rq.wqe_shift = ilog32(rqwqe_size);
>>> +		qp->rq.wqe_shift = ilog32(rqwqe_size) - 1;
>>>
>>>  		if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE || type ==
>>> IBV_QPT_UD)
>>>  			qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
>>> @@ -747,8 +748,8 @@ static void hns_roce_set_qp_params(struct
>>> ibv_pd *pd,
>>>  		qp->rq.wqe_cnt =
>>> roundup_pow_of_two(attr->cap.max_recv_wr);
>>>  	}
>>>
>>> -	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe));
>>> -	qp->sq.shift = ilog32(qp->sq.wqe_cnt);
>>> +	qp->sq.wqe_shift = ilog32(sizeof(struct hns_roce_rc_send_wqe)) - 1;
>>> +	qp->sq.shift = ilog32(qp->sq.wqe_cnt) - 1;
>>
>> One suggestion, it's better to introduce a new micro instead of ilog32(x) -1.
> 
> Is the -1 even correct?
> 
> I would have guessed something called shift wants to be:
> 
>    shift = ilog32(qp->sq.wqe_cnt - 1)
> 
> Such that 
>    1 << shift == wqe_cnt
>        When wqe_cnt is a power of two.
>    1 << shift > wqe_cnt
>        When wqe_cnt is not a power of two.
> 
> Jason
> 

That's right, shift = ilog32(n - 1) is always right, but result of
ilog32(n) - 1 is only correct when n is a power of two.

Will modify to ilog32(n - 1).

Thank you
Weihang


> .
>