[PATCH v3 0/5] RDMA/rxe: Do some cleanup

[PATCH v3 0/5] RDMA/rxe: Do some cleanup

[Xiao Yang]

V2->V3:
1) Rebase on the latest kernel
2) Drop the ternary expression

Xiao Yang (5):
  RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
  RDMA/rxe: Remove the common is_user member of struct rxe_qp
  RDMA/rxe: Change the is_user member of struct rxe_cq to bool
  RDMA/rxe: Set partial attributes when completion status !=
    IBV_WC_SUCCESS
  RDMA/rxe: Remove duplicate settings

 drivers/infiniband/sw/rxe/rxe_comp.c  | 51 +++++++++++++++------------
 drivers/infiniband/sw/rxe/rxe_cq.c    |  3 +-
 drivers/infiniband/sw/rxe/rxe_qp.c    |  5 +--
 drivers/infiniband/sw/rxe/rxe_req.c   |  4 +--
 drivers/infiniband/sw/rxe/rxe_resp.c  | 14 +++-----
 drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++----------------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  3 +-
 7 files changed, 49 insertions(+), 72 deletions(-)

-- 
2.25.1

[PATCH v3 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user

[Xiao Yang]

1) post_one_send() always processes kernel's send queue.
2) rxe_poll_cq() always processes kernel's completion queue.

Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
 drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++---------------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 267b5a9c345d..548771ecbaf8 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -625,7 +625,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
 	struct rxe_sq *sq = &qp->sq;
 	struct rxe_send_wqe *send_wqe;
 	unsigned long flags;
-	int full;
 
 	err = validate_send_wr(qp, ibwr, mask, length);
 	if (err)
@@ -633,27 +632,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
 
 	spin_lock_irqsave(&qp->sq.sq_lock, flags);
 
-	if (qp->is_user)
-		full = queue_full(sq->queue, QUEUE_TYPE_FROM_USER);
-	else
-		full = queue_full(sq->queue, QUEUE_TYPE_KERNEL);
-
-	if (unlikely(full)) {
+	if (unlikely(queue_full(sq->queue, QUEUE_TYPE_KERNEL))) {
 		spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
 		return -ENOMEM;
 	}
 
-	if (qp->is_user)
-		send_wqe = producer_addr(sq->queue, QUEUE_TYPE_FROM_USER);
-	else
-		send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
+	send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
 
 	init_send_wqe(qp, ibwr, mask, length, send_wqe);
 
-	if (qp->is_user)
-		advance_producer(sq->queue, QUEUE_TYPE_FROM_USER);
-	else
-		advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
+	advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
 
 	spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
 
@@ -845,18 +833,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 
 	spin_lock_irqsave(&cq->cq_lock, flags);
 	for (i = 0; i < num_entries; i++) {
-		if (cq->is_user)
-			cqe = queue_head(cq->queue, QUEUE_TYPE_TO_USER);
-		else
-			cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
+		cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
 		if (!cqe)
 			break;
 
 		memcpy(wc++, &cqe->ibwc, sizeof(*wc));
-		if (cq->is_user)
-			advance_consumer(cq->queue, QUEUE_TYPE_TO_USER);
-		else
-			advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
+
+		advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
 	}
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
-- 
2.25.1

[PATCH v3 2/5] RDMA/rxe: Remove the common is_user member of struct rxe_qp

[Xiao Yang]

The following commit introduced separate is_user members for struct
rxe_sq/rxe_rq/rxe_srq but no code uses the separate is_user member
of struct rxe_sq and lots of code still use the common is_user member
of struct rxe_qp.  So it is clear to make all code use separate is_user
members uniformly and remove the common is_user member.

Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
 drivers/infiniband/sw/rxe/rxe_comp.c  |  6 +++---
 drivers/infiniband/sw/rxe/rxe_qp.c    |  5 +++--
 drivers/infiniband/sw/rxe/rxe_req.c   |  4 ++--
 drivers/infiniband/sw/rxe/rxe_resp.c  | 10 +++++-----
 drivers/infiniband/sw/rxe/rxe_verbs.c | 12 +++---------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  1 -
 6 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index d2d802c776fd..148fe3dd3b32 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -142,7 +142,7 @@ static inline enum comp_state get_wqe(struct rxe_qp *qp,
 	/* we come here whether or not we found a response packet to see if
 	 * there are any posted WQEs
 	 */
-	if (qp->is_user)
+	if (qp->sq.is_user)
 		wqe = queue_head(qp->sq.queue, QUEUE_TYPE_FROM_USER);
 	else
 		wqe = queue_head(qp->sq.queue, QUEUE_TYPE_KERNEL);
@@ -385,7 +385,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 {
 	memset(cqe, 0, sizeof(*cqe));
 
-	if (!qp->is_user) {
+	if (!qp->sq.is_user) {
 		struct ib_wc		*wc	= &cqe->ibwc;
 
 		wc->wr_id		= wqe->wr.wr_id;
@@ -432,7 +432,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 	if (post)
 		make_send_cqe(qp, wqe, &cqe);
 
-	if (qp->is_user)
+	if (qp->sq.is_user)
 		advance_consumer(qp->sq.queue, QUEUE_TYPE_FROM_USER);
 	else
 		advance_consumer(qp->sq.queue, QUEUE_TYPE_KERNEL);
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 1ab6af7ddb25..5cbbb00089cf 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -248,7 +248,8 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
 		return err;
 	}
 
-	if (qp->is_user)
+	qp->sq.is_user = uresp;
+	if (qp->sq.is_user)
 		qp->req.wqe_index = producer_index(qp->sq.queue,
 						QUEUE_TYPE_FROM_USER);
 	else
@@ -313,7 +314,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
 	spin_lock_init(&qp->rq.producer_lock);
 	spin_lock_init(&qp->rq.consumer_lock);
 
-	qp->rq.is_user = qp->is_user;
+	qp->rq.is_user = uresp;
 
 	skb_queue_head_init(&qp->resp_pkts);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 3894197a82f6..61888e963774 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -49,7 +49,7 @@ static void req_retry(struct rxe_qp *qp)
 	unsigned int cons;
 	unsigned int prod;
 
-	if (qp->is_user) {
+	if (qp->sq.is_user) {
 		cons = consumer_index(q, QUEUE_TYPE_FROM_USER);
 		prod = producer_index(q, QUEUE_TYPE_FROM_USER);
 	} else {
@@ -121,7 +121,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
 	unsigned int cons;
 	unsigned int prod;
 
-	if (qp->is_user) {
+	if (qp->sq.is_user) {
 		wqe = queue_head(q, QUEUE_TYPE_FROM_USER);
 		cons = consumer_index(q, QUEUE_TYPE_FROM_USER);
 		prod = producer_index(q, QUEUE_TYPE_FROM_USER);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 5501227ddc65..c850f8b0080d 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -303,7 +303,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
 
 	spin_lock_bh(&srq->rq.consumer_lock);
 
-	if (qp->is_user)
+	if (srq->is_user)
 		wqe = queue_head(q, QUEUE_TYPE_FROM_USER);
 	else
 		wqe = queue_head(q, QUEUE_TYPE_KERNEL);
@@ -322,7 +322,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
 	memcpy(&qp->resp.srq_wqe, wqe, size);
 
 	qp->resp.wqe = &qp->resp.srq_wqe.wqe;
-	if (qp->is_user) {
+	if (srq->is_user) {
 		advance_consumer(q, QUEUE_TYPE_FROM_USER);
 		count = queue_count(q, QUEUE_TYPE_FROM_USER);
 	} else {
@@ -357,7 +357,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
 			qp->resp.status = IB_WC_WR_FLUSH_ERR;
 			return RESPST_COMPLETE;
 		} else if (!srq) {
-			if (qp->is_user)
+			if (qp->rq.is_user)
 				qp->resp.wqe = queue_head(qp->rq.queue,
 						QUEUE_TYPE_FROM_USER);
 			else
@@ -389,7 +389,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
 		if (srq)
 			return get_srq_wqe(qp);
 
-		if (qp->is_user)
+		if (qp->rq.is_user)
 			qp->resp.wqe = queue_head(qp->rq.queue,
 					QUEUE_TYPE_FROM_USER);
 		else
@@ -937,7 +937,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 
 	/* have copy for srq and reference for !srq */
 	if (!qp->srq) {
-		if (qp->is_user)
+		if (qp->rq.is_user)
 			advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_USER);
 		else
 			advance_consumer(qp->rq.queue, QUEUE_TYPE_KERNEL);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 548771ecbaf8..135b54e93963 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -413,14 +413,8 @@ static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,
 	if (err)
 		return err;
 
-	if (udata) {
-		if (udata->inlen)
-			return -EINVAL;
-
-		qp->is_user = true;
-	} else {
-		qp->is_user = false;
-	}
+	if (udata && udata->inlen)
+		return -EINVAL;
 
 	err = rxe_add_to_pool(&rxe->qp_pool, qp);
 	if (err)
@@ -709,7 +703,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 		return -EINVAL;
 	}
 
-	if (qp->is_user) {
+	if (qp->sq.is_user) {
 		/* Utilize process context to do protocol processing */
 		rxe_run_task(&qp->req.task, 0);
 		return 0;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index ac2a2148027f..ba74cedff9d1 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -215,7 +215,6 @@ struct rxe_qp {
 	struct ib_qp_attr	attr;
 	unsigned int		valid;
 	unsigned int		mtu;
-	bool			is_user;
 
 	struct rxe_pd		*pd;
 	struct rxe_srq		*srq;
-- 
2.25.1

[PATCH v3 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool

[Xiao Yang]

Make all is_user members of struct rxe_sq/rxe_cq/rxe_srq/rxe_cq
has the same type.

Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
 drivers/infiniband/sw/rxe/rxe_cq.c    | 3 +--
 drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
index aef288f164fd..8cc90a5424d9 100644
--- a/drivers/infiniband/sw/rxe/rxe_cq.c
+++ b/drivers/infiniband/sw/rxe/rxe_cq.c
@@ -81,8 +81,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
 		return err;
 	}
 
-	if (uresp)
-		cq->is_user = 1;
+	cq->is_user = uresp;
 
 	cq->is_dying = false;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index ba74cedff9d1..671884514ee1 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -64,7 +64,7 @@ struct rxe_cq {
 	spinlock_t		cq_lock;
 	u8			notify;
 	bool			is_dying;
-	int			is_user;
+	bool			is_user;
 	struct tasklet_struct	comp_task;
 };
 
-- 
2.25.1

[PATCH v3 4/5] RDMA/rxe: Set partial attributes when completion status != IBV_WC_SUCCESS

[Xiao Yang]

As ibv_poll_cq()'s manual said, only partial attributes are valid
when completion status != IBV_WC_SUCCESS.

Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
 drivers/infiniband/sw/rxe/rxe_comp.c | 45 +++++++++++++++-------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 148fe3dd3b32..6b71e020009b 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -383,30 +383,35 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,
 static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 			  struct rxe_cqe *cqe)
 {
+	struct ib_wc *wc = &cqe->ibwc;
+	struct ib_uverbs_wc *uwc = &cqe->uibwc;
+
 	memset(cqe, 0, sizeof(*cqe));
 
 	if (!qp->sq.is_user) {
-		struct ib_wc		*wc	= &cqe->ibwc;
-
-		wc->wr_id		= wqe->wr.wr_id;
-		wc->status		= wqe->status;
-		wc->opcode		= wr_to_wc_opcode(wqe->wr.opcode);
-		if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
-		    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
-			wc->wc_flags = IB_WC_WITH_IMM;
-		wc->byte_len		= wqe->dma.length;
-		wc->qp			= &qp->ibqp;
+		wc->wr_id = wqe->wr.wr_id;
+		wc->status = wqe->status;
+		wc->qp = &qp->ibqp;
 	} else {
-		struct ib_uverbs_wc	*uwc	= &cqe->uibwc;
-
-		uwc->wr_id		= wqe->wr.wr_id;
-		uwc->status		= wqe->status;
-		uwc->opcode		= wr_to_wc_opcode(wqe->wr.opcode);
-		if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
-		    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
-			uwc->wc_flags = IB_WC_WITH_IMM;
-		uwc->byte_len		= wqe->dma.length;
-		uwc->qp_num		= qp->ibqp.qp_num;
+		uwc->wr_id = wqe->wr.wr_id;
+		uwc->status = wqe->status;
+		uwc->qp_num = qp->ibqp.qp_num;
+	}
+
+	if (wqe->status == IB_WC_SUCCESS) {
+		if (!qp->sq.is_user) {
+			wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+			    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+				wc->wc_flags = IB_WC_WITH_IMM;
+			wc->byte_len = wqe->dma.length;
+		} else {
+			uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+			    wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+				uwc->wc_flags = IB_WC_WITH_IMM;
+			uwc->byte_len = wqe->dma.length;
+		}
 	}
 }
 
-- 
2.25.1

[PATCH v3 5/5] RDMA/rxe: Remove duplicate settings

[Xiao Yang]

Remove duplicate settings for vendor_err and qp_num.

Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
 drivers/infiniband/sw/rxe/rxe_resp.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index c850f8b0080d..6102ea36ed28 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -876,7 +876,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 		wc->opcode = (pkt->mask & RXE_IMMDT_MASK &&
 				pkt->mask & RXE_WRITE_MASK) ?
 					IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
-		wc->vendor_err = 0;
 		wc->byte_len = (pkt->mask & RXE_IMMDT_MASK &&
 				pkt->mask & RXE_WRITE_MASK) ?
 					qp->resp.length : wqe->dma.length - wqe->dma.resid;
@@ -897,8 +896,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 				uwc->ex.invalidate_rkey = ieth_rkey(pkt);
 			}
 
-			uwc->qp_num		= qp->ibqp.qp_num;
-
 			if (pkt->mask & RXE_DETH_MASK)
 				uwc->src_qp = deth_sqp(pkt);
 
@@ -930,7 +927,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 			if (pkt->mask & RXE_DETH_MASK)
 				wc->src_qp = deth_sqp(pkt);
 
-			wc->qp			= &qp->ibqp;
 			wc->port_num		= qp->attr.port_num;
 		}
 	}
-- 
2.25.1

Re: [PATCH v3 0/5] RDMA/rxe: Do some cleanup

[Jason Gunthorpe]

On Thu, Sep 16, 2021 at 08:46:47PM +0800, Xiao Yang wrote:
> V2->V3:
> 1) Rebase on the latest kernel
> 2) Drop the ternary expression
> 
> Xiao Yang (5):
>   RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
>   RDMA/rxe: Remove the common is_user member of struct rxe_qp
>   RDMA/rxe: Change the is_user member of struct rxe_cq to bool
>   RDMA/rxe: Set partial attributes when completion status !=
>     IBV_WC_SUCCESS
>   RDMA/rxe: Remove duplicate settings
> 
>  drivers/infiniband/sw/rxe/rxe_comp.c  | 51 +++++++++++++++------------
>  drivers/infiniband/sw/rxe/rxe_cq.c    |  3 +-
>  drivers/infiniband/sw/rxe/rxe_qp.c    |  5 +--
>  drivers/infiniband/sw/rxe/rxe_req.c   |  4 +--
>  drivers/infiniband/sw/rxe/rxe_resp.c  | 14 +++-----
>  drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++----------------
>  drivers/infiniband/sw/rxe/rxe_verbs.h |  3 +-
>  7 files changed, 49 insertions(+), 72 deletions(-)

This doesn't apply, can you rebase and resend please

Jason

Re: [PATCH v3 0/5] RDMA/rxe: Do some cleanup

[Yang, Xiao/杨 晓]

On 2021/9/28 22:48, Jason Gunthorpe wrote:
> On Thu, Sep 16, 2021 at 08:46:47PM +0800, Xiao Yang wrote:
>> V2->V3:
>> 1) Rebase on the latest kernel
>> 2) Drop the ternary expression
>>
>> Xiao Yang (5):
>>    RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
>>    RDMA/rxe: Remove the common is_user member of struct rxe_qp
>>    RDMA/rxe: Change the is_user member of struct rxe_cq to bool
>>    RDMA/rxe: Set partial attributes when completion status !=
>>      IBV_WC_SUCCESS
>>    RDMA/rxe: Remove duplicate settings
>>
>>   drivers/infiniband/sw/rxe/rxe_comp.c  | 51 +++++++++++++++------------
>>   drivers/infiniband/sw/rxe/rxe_cq.c    |  3 +-
>>   drivers/infiniband/sw/rxe/rxe_qp.c    |  5 +--
>>   drivers/infiniband/sw/rxe/rxe_req.c   |  4 +--
>>   drivers/infiniband/sw/rxe/rxe_resp.c  | 14 +++-----
>>   drivers/infiniband/sw/rxe/rxe_verbs.c | 41 +++++----------------
>>   drivers/infiniband/sw/rxe/rxe_verbs.h |  3 +-
>>   7 files changed, 49 insertions(+), 72 deletions(-)
> This doesn't apply, can you rebase and resend please
Hi Jason,

Sure, I will rebase and resend soon.

Best Regards,
Xiao Yang
> Jason