* [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
2021-09-02 8:46 [PATCH v2 0/5] RDMA/rxe: Do some cleanup Xiao Yang
@ 2021-09-02 8:46 ` Xiao Yang
2021-09-14 18:32 ` Jason Gunthorpe
2021-09-02 8:46 ` [PATCH v2 2/5] RDMA/rxe: Remove the common is_user member of struct rxe_qp Xiao Yang
` (3 subsequent siblings)
4 siblings, 1 reply; 12+ messages in thread
From: Xiao Yang @ 2021-09-02 8:46 UTC (permalink / raw)
To: linux-rdma; +Cc: rpearsonhpe, zyjzyj2000, jgg, leon, Xiao Yang
1) post_one_send() always processes kernel's send queue.
2) rxe_poll_cq() always processes kernel's completion queue.
Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++---------------------
1 file changed, 6 insertions(+), 23 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index c223959ac174..cdded9f64910 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -632,7 +632,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
struct rxe_sq *sq = &qp->sq;
struct rxe_send_wqe *send_wqe;
unsigned long flags;
- int full;
err = validate_send_wr(qp, ibwr, mask, length);
if (err)
@@ -640,27 +639,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
spin_lock_irqsave(&qp->sq.sq_lock, flags);
- if (qp->is_user)
- full = queue_full(sq->queue, QUEUE_TYPE_FROM_USER);
- else
- full = queue_full(sq->queue, QUEUE_TYPE_KERNEL);
-
- if (unlikely(full)) {
+ if (unlikely(queue_full(sq->queue, QUEUE_TYPE_KERNEL))) {
spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
return -ENOMEM;
}
- if (qp->is_user)
- send_wqe = producer_addr(sq->queue, QUEUE_TYPE_FROM_USER);
- else
- send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
+ send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
init_send_wqe(qp, ibwr, mask, length, send_wqe);
- if (qp->is_user)
- advance_producer(sq->queue, QUEUE_TYPE_FROM_USER);
- else
- advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
+ advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
@@ -852,18 +840,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
spin_lock_irqsave(&cq->cq_lock, flags);
for (i = 0; i < num_entries; i++) {
- if (cq->is_user)
- cqe = queue_head(cq->queue, QUEUE_TYPE_TO_USER);
- else
- cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
+ cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
if (!cqe)
break;
memcpy(wc++, &cqe->ibwc, sizeof(*wc));
- if (cq->is_user)
- advance_consumer(cq->queue, QUEUE_TYPE_TO_USER);
- else
- advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
+
+ advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
}
spin_unlock_irqrestore(&cq->cq_lock, flags);
--
2.23.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
2021-09-02 8:46 ` [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user Xiao Yang
@ 2021-09-14 18:32 ` Jason Gunthorpe
2021-09-16 9:16 ` yangx.jy
0 siblings, 1 reply; 12+ messages in thread
From: Jason Gunthorpe @ 2021-09-14 18:32 UTC (permalink / raw)
To: Xiao Yang; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On Thu, Sep 02, 2021 at 04:46:36PM +0800, Xiao Yang wrote:
> 1) post_one_send() always processes kernel's send queue.
> 2) rxe_poll_cq() always processes kernel's completion queue.
>
> Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
> Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
> drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++---------------------
> 1 file changed, 6 insertions(+), 23 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index c223959ac174..cdded9f64910 100644
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -632,7 +632,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
> struct rxe_sq *sq = &qp->sq;
> struct rxe_send_wqe *send_wqe;
> unsigned long flags;
> - int full;
>
> err = validate_send_wr(qp, ibwr, mask, length);
> if (err)
> @@ -640,27 +639,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
>
> spin_lock_irqsave(&qp->sq.sq_lock, flags);
>
> - if (qp->is_user)
> - full = queue_full(sq->queue, QUEUE_TYPE_FROM_USER);
> - else
> - full = queue_full(sq->queue, QUEUE_TYPE_KERNEL);
> -
> - if (unlikely(full)) {
> + if (unlikely(queue_full(sq->queue, QUEUE_TYPE_KERNEL))) {
> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
> return -ENOMEM;
> }
>
> - if (qp->is_user)
> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_FROM_USER);
> - else
> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
> + send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
>
> init_send_wqe(qp, ibwr, mask, length, send_wqe);
>
> - if (qp->is_user)
> - advance_producer(sq->queue, QUEUE_TYPE_FROM_USER);
> - else
> - advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
> + advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
>
> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
This bit looks OK
> @@ -852,18 +840,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
>
> spin_lock_irqsave(&cq->cq_lock, flags);
> for (i = 0; i < num_entries; i++) {
> - if (cq->is_user)
> - cqe = queue_head(cq->queue, QUEUE_TYPE_TO_USER);
> - else
> - cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
> + cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
> if (!cqe)
> break;
>
> memcpy(wc++, &cqe->ibwc, sizeof(*wc));
> - if (cq->is_user)
> - advance_consumer(cq->queue, QUEUE_TYPE_TO_USER);
> - else
> - advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
> +
> + advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
> }
But why is this OK?
It is used here:
.poll_cq = rxe_poll_cq,
Which is part of:
static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
[..]
ret = ib_poll_cq(cq, 1, &wc);
That is used called?
Jason
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
2021-09-14 18:32 ` Jason Gunthorpe
@ 2021-09-16 9:16 ` yangx.jy
2021-09-16 13:22 ` Jason Gunthorpe
0 siblings, 1 reply; 12+ messages in thread
From: yangx.jy @ 2021-09-16 9:16 UTC (permalink / raw)
To: Jason Gunthorpe; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On 2021/9/15 2:32, Jason Gunthorpe wrote:
> On Thu, Sep 02, 2021 at 04:46:36PM +0800, Xiao Yang wrote:
>> 1) post_one_send() always processes kernel's send queue.
>> 2) rxe_poll_cq() always processes kernel's completion queue.
>>
>> Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
>> Signed-off-by: Xiao Yang<yangx.jy@fujitsu.com>
>> drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++---------------------
>> 1 file changed, 6 insertions(+), 23 deletions(-)
>>
>> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
>> index c223959ac174..cdded9f64910 100644
>> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
>> @@ -632,7 +632,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
>> struct rxe_sq *sq =&qp->sq;
>> struct rxe_send_wqe *send_wqe;
>> unsigned long flags;
>> - int full;
>>
>> err = validate_send_wr(qp, ibwr, mask, length);
>> if (err)
>> @@ -640,27 +639,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
>>
>> spin_lock_irqsave(&qp->sq.sq_lock, flags);
>>
>> - if (qp->is_user)
>> - full = queue_full(sq->queue, QUEUE_TYPE_FROM_USER);
>> - else
>> - full = queue_full(sq->queue, QUEUE_TYPE_KERNEL);
>> -
>> - if (unlikely(full)) {
>> + if (unlikely(queue_full(sq->queue, QUEUE_TYPE_KERNEL))) {
>> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
>> return -ENOMEM;
>> }
>>
>> - if (qp->is_user)
>> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_FROM_USER);
>> - else
>> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
>> + send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
>>
>> init_send_wqe(qp, ibwr, mask, length, send_wqe);
>>
>> - if (qp->is_user)
>> - advance_producer(sq->queue, QUEUE_TYPE_FROM_USER);
>> - else
>> - advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
>> + advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
>>
>> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
> This bit looks OK
>
>> @@ -852,18 +840,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
>>
>> spin_lock_irqsave(&cq->cq_lock, flags);
>> for (i = 0; i< num_entries; i++) {
>> - if (cq->is_user)
>> - cqe = queue_head(cq->queue, QUEUE_TYPE_TO_USER);
>> - else
>> - cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
>> + cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
>> if (!cqe)
>> break;
>>
>> memcpy(wc++,&cqe->ibwc, sizeof(*wc));
>> - if (cq->is_user)
>> - advance_consumer(cq->queue, QUEUE_TYPE_TO_USER);
>> - else
>> - advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
>> +
>> + advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
>> }
> But why is this OK?
>
> It is used here:
>
> .poll_cq = rxe_poll_cq,
>
> Which is part of:
>
> static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
> [..]
>
> ret = ib_poll_cq(cq, 1,&wc);
>
> That is used called?
Hi Jason,
ib_uverbs_poll_cq() is called by ibv_cmd_poll_cq() in userspace but rxe
uses its own rxe_poll_cq() instead.
See the following code in rdma-core:
-------------------------------------------------------------------
libibverbs/cmd.c:
int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
{
...
ret = execute_cmd_write_no_uhw(ibcq->context,
IB_USER_VERBS_CMD_POLL_CQ, &cmd, sizeof(cmd), resp, rsize);
...
}
providers/rxe/rxe.c:
...
.poll_cq = rxe_poll_cq,
...
-------------------------------------------------------------------
In this case, rxe has no chance to call ib_uverbs_poll_cq from userspace.
rxe_poll_cq() can also be called by kthread, like this:
--------------------------------------------------------------
[ 1247.060587] Call Trace:
[ 1247.060592] __ib_process_cq+0x57/0x150 [ib_core]
[ 1247.060633] ib_cq_poll_work+0x26/0x80 [ib_core]
[ 1247.060671] process_one_work+0x1ec/0x390
[ 1247.060680] worker_thread+0x50/0x3a0
[ 1247.060687] ? process_one_work+0x390/0x390
[ 1247.060695] kthread+0x127/0x150
[ 1247.060701] ? set_kthread_struct+0x40/0x40
[ 1247.060706] ret_from_fork+0x22/0x30
[ 1247.060713] ---[ end trace 24a7d2217da4f2b5 ]---
--------------------------------------------------------------
By the way, I think the following code also indicates that
rxe_poll_cq() is always processed by kernel.
----------------------------------------------------------------------------------
memcpy(wc++, &cqe->ibwc, sizeof(*wc));
----------------------------------------------------------------------------------
Best Regards,
Xiao Yang
> Jason
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
2021-09-16 9:16 ` yangx.jy
@ 2021-09-16 13:22 ` Jason Gunthorpe
2021-09-17 6:00 ` yangx.jy
0 siblings, 1 reply; 12+ messages in thread
From: Jason Gunthorpe @ 2021-09-16 13:22 UTC (permalink / raw)
To: yangx.jy; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On Thu, Sep 16, 2021 at 09:16:35AM +0000, yangx.jy@fujitsu.com wrote:
> On 2021/9/15 2:32, Jason Gunthorpe wrote:
> > On Thu, Sep 02, 2021 at 04:46:36PM +0800, Xiao Yang wrote:
> >> 1) post_one_send() always processes kernel's send queue.
> >> 2) rxe_poll_cq() always processes kernel's completion queue.
> >>
> >> Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
> >> Signed-off-by: Xiao Yang<yangx.jy@fujitsu.com>
> >> drivers/infiniband/sw/rxe/rxe_verbs.c | 29 ++++++---------------------
> >> 1 file changed, 6 insertions(+), 23 deletions(-)
> >>
> >> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> >> index c223959ac174..cdded9f64910 100644
> >> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> >> @@ -632,7 +632,6 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
> >> struct rxe_sq *sq =&qp->sq;
> >> struct rxe_send_wqe *send_wqe;
> >> unsigned long flags;
> >> - int full;
> >>
> >> err = validate_send_wr(qp, ibwr, mask, length);
> >> if (err)
> >> @@ -640,27 +639,16 @@ static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
> >>
> >> spin_lock_irqsave(&qp->sq.sq_lock, flags);
> >>
> >> - if (qp->is_user)
> >> - full = queue_full(sq->queue, QUEUE_TYPE_FROM_USER);
> >> - else
> >> - full = queue_full(sq->queue, QUEUE_TYPE_KERNEL);
> >> -
> >> - if (unlikely(full)) {
> >> + if (unlikely(queue_full(sq->queue, QUEUE_TYPE_KERNEL))) {
> >> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
> >> return -ENOMEM;
> >> }
> >>
> >> - if (qp->is_user)
> >> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_FROM_USER);
> >> - else
> >> - send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
> >> + send_wqe = producer_addr(sq->queue, QUEUE_TYPE_KERNEL);
> >>
> >> init_send_wqe(qp, ibwr, mask, length, send_wqe);
> >>
> >> - if (qp->is_user)
> >> - advance_producer(sq->queue, QUEUE_TYPE_FROM_USER);
> >> - else
> >> - advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
> >> + advance_producer(sq->queue, QUEUE_TYPE_KERNEL);
> >>
> >> spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
> > This bit looks OK
> >
> >> @@ -852,18 +840,13 @@ static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
> >>
> >> spin_lock_irqsave(&cq->cq_lock, flags);
> >> for (i = 0; i< num_entries; i++) {
> >> - if (cq->is_user)
> >> - cqe = queue_head(cq->queue, QUEUE_TYPE_TO_USER);
> >> - else
> >> - cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
> >> + cqe = queue_head(cq->queue, QUEUE_TYPE_KERNEL);
> >> if (!cqe)
> >> break;
> >>
> >> memcpy(wc++,&cqe->ibwc, sizeof(*wc));
> >> - if (cq->is_user)
> >> - advance_consumer(cq->queue, QUEUE_TYPE_TO_USER);
> >> - else
> >> - advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
> >> +
> >> + advance_consumer(cq->queue, QUEUE_TYPE_KERNEL);
> >> }
> > But why is this OK?
> >
> > It is used here:
> >
> > .poll_cq = rxe_poll_cq,
> >
> > Which is part of:
> >
> > static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
> > [..]
> >
> > ret = ib_poll_cq(cq, 1,&wc);
> >
> > That is used called?
> Hi Jason,
>
> ib_uverbs_poll_cq() is called by ibv_cmd_poll_cq() in userspace but rxe
> uses its own rxe_poll_cq() instead.
> See the following code in rdma-core:
Yes, but rdma-core doesn't matter.
The question is why is this safe and the reason is rxe doesn't set
IB_USER_VERBS_CMD_POLL_CQ in uverbs_cmd_mask.
I'd be a bit happier seeing this fixed so we have a poll_kernel_cq
poll_user_cq op and this isn't so tricky.
Jason
^ permalink raw reply [flat|nested] 12+ messages in thread* Re: [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user
2021-09-16 13:22 ` Jason Gunthorpe
@ 2021-09-17 6:00 ` yangx.jy
0 siblings, 0 replies; 12+ messages in thread
From: yangx.jy @ 2021-09-17 6:00 UTC (permalink / raw)
To: Jason Gunthorpe; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On 2021/9/16 21:22, Jason Gunthorpe wrote:
> Yes, but rdma-core doesn't matter.
>
> The question is why is this safe and the reason is rxe doesn't set
> IB_USER_VERBS_CMD_POLL_CQ in uverbs_cmd_mask.
>
> I'd be a bit happier seeing this fixed so we have a poll_kernel_cq
> poll_user_cq op and this isn't so tricky.
Hi Jason,
I saw you removed IB_USER_VERBS_CMD_POLL_CQ from uverbs_cmd_mask by
commit 628c02bf38aa ("RDMA: Remove uverbs cmds from drivers that don't
use them")
I think I don't need to update the patch, right? :-)
Best Regards,
Xiao Yang
> Jason
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH v2 2/5] RDMA/rxe: Remove the common is_user member of struct rxe_qp
2021-09-02 8:46 [PATCH v2 0/5] RDMA/rxe: Do some cleanup Xiao Yang
2021-09-02 8:46 ` [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user Xiao Yang
@ 2021-09-02 8:46 ` Xiao Yang
2021-09-02 8:46 ` [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool Xiao Yang
` (2 subsequent siblings)
4 siblings, 0 replies; 12+ messages in thread
From: Xiao Yang @ 2021-09-02 8:46 UTC (permalink / raw)
To: linux-rdma; +Cc: rpearsonhpe, zyjzyj2000, jgg, leon, Xiao Yang
The following commit introduced separate is_user members for struct
rxe_sq/rxe_rq/rxe_srq but no code uses the separate is_user member
of struct rxe_sq and lots of code still use the common is_user member
of struct rxe_qp. So it is clear to make all code use separate is_user
members uniformly and remove the common is_user member.
Fixes: 5bcf5a59c41e ("RDMA/rxe: Protext kernel index from user space")
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
drivers/infiniband/sw/rxe/rxe_comp.c | 6 +++---
drivers/infiniband/sw/rxe/rxe_qp.c | 5 +++--
drivers/infiniband/sw/rxe/rxe_req.c | 4 ++--
drivers/infiniband/sw/rxe/rxe_resp.c | 10 +++++-----
drivers/infiniband/sw/rxe/rxe_verbs.c | 13 ++++---------
drivers/infiniband/sw/rxe/rxe_verbs.h | 1 -
6 files changed, 17 insertions(+), 22 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 58ad9c2644f3..fb7741ec5cd3 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -142,7 +142,7 @@ static inline enum comp_state get_wqe(struct rxe_qp *qp,
/* we come here whether or not we found a response packet to see if
* there are any posted WQEs
*/
- if (qp->is_user)
+ if (qp->sq.is_user)
wqe = queue_head(qp->sq.queue, QUEUE_TYPE_FROM_USER);
else
wqe = queue_head(qp->sq.queue, QUEUE_TYPE_KERNEL);
@@ -385,7 +385,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
{
memset(cqe, 0, sizeof(*cqe));
- if (!qp->is_user) {
+ if (!qp->sq.is_user) {
struct ib_wc *wc = &cqe->ibwc;
wc->wr_id = wqe->wr.wr_id;
@@ -432,7 +432,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
if (post)
make_send_cqe(qp, wqe, &cqe);
- if (qp->is_user)
+ if (qp->sq.is_user)
advance_consumer(qp->sq.queue, QUEUE_TYPE_FROM_USER);
else
advance_consumer(qp->sq.queue, QUEUE_TYPE_KERNEL);
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 1ab6af7ddb25..cfa04e92286a 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -248,7 +248,8 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
return err;
}
- if (qp->is_user)
+ qp->sq.is_user = uresp ? true : false;
+ if (qp->sq.is_user)
qp->req.wqe_index = producer_index(qp->sq.queue,
QUEUE_TYPE_FROM_USER);
else
@@ -313,7 +314,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
spin_lock_init(&qp->rq.producer_lock);
spin_lock_init(&qp->rq.consumer_lock);
- qp->rq.is_user = qp->is_user;
+ qp->rq.is_user = uresp ? true : false;
skb_queue_head_init(&qp->resp_pkts);
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index c57699cc6578..2886709e1823 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -49,7 +49,7 @@ static void req_retry(struct rxe_qp *qp)
unsigned int cons;
unsigned int prod;
- if (qp->is_user) {
+ if (qp->sq.is_user) {
cons = consumer_index(q, QUEUE_TYPE_FROM_USER);
prod = producer_index(q, QUEUE_TYPE_FROM_USER);
} else {
@@ -121,7 +121,7 @@ static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
unsigned int cons;
unsigned int prod;
- if (qp->is_user) {
+ if (qp->sq.is_user) {
wqe = queue_head(q, QUEUE_TYPE_FROM_USER);
cons = consumer_index(q, QUEUE_TYPE_FROM_USER);
prod = producer_index(q, QUEUE_TYPE_FROM_USER);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 360ec67cb9e1..71406a49fca3 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -303,7 +303,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
spin_lock_bh(&srq->rq.consumer_lock);
- if (qp->is_user)
+ if (srq->is_user)
wqe = queue_head(q, QUEUE_TYPE_FROM_USER);
else
wqe = queue_head(q, QUEUE_TYPE_KERNEL);
@@ -322,7 +322,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
memcpy(&qp->resp.srq_wqe, wqe, size);
qp->resp.wqe = &qp->resp.srq_wqe.wqe;
- if (qp->is_user) {
+ if (srq->is_user) {
advance_consumer(q, QUEUE_TYPE_FROM_USER);
count = queue_count(q, QUEUE_TYPE_FROM_USER);
} else {
@@ -357,7 +357,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
qp->resp.status = IB_WC_WR_FLUSH_ERR;
return RESPST_COMPLETE;
} else if (!srq) {
- if (qp->is_user)
+ if (qp->rq.is_user)
qp->resp.wqe = queue_head(qp->rq.queue,
QUEUE_TYPE_FROM_USER);
else
@@ -389,7 +389,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
if (srq)
return get_srq_wqe(qp);
- if (qp->is_user)
+ if (qp->rq.is_user)
qp->resp.wqe = queue_head(qp->rq.queue,
QUEUE_TYPE_FROM_USER);
else
@@ -954,7 +954,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
/* have copy for srq and reference for !srq */
if (!qp->srq) {
- if (qp->is_user)
+ if (qp->rq.is_user)
advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_USER);
else
advance_consumer(qp->rq.queue, QUEUE_TYPE_KERNEL);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index cdded9f64910..4ce4ff6ad80e 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -420,14 +420,9 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd,
goto err1;
}
- if (udata) {
- if (udata->inlen) {
- err = -EINVAL;
- goto err2;
- }
- qp->is_user = true;
- } else {
- qp->is_user = false;
+ if (udata && udata->inlen) {
+ err = -EINVAL;
+ goto err2;
}
rxe_add_index(qp);
@@ -716,7 +711,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
return -EINVAL;
}
- if (qp->is_user) {
+ if (qp->sq.is_user) {
/* Utilize process context to do protocol processing */
rxe_run_task(&qp->req.task, 0);
return 0;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 959a3260fcab..bb5fb157d073 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -215,7 +215,6 @@ struct rxe_qp {
struct ib_qp_attr attr;
unsigned int valid;
unsigned int mtu;
- bool is_user;
struct rxe_pd *pd;
struct rxe_srq *srq;
--
2.23.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool
2021-09-02 8:46 [PATCH v2 0/5] RDMA/rxe: Do some cleanup Xiao Yang
2021-09-02 8:46 ` [PATCH v2 1/5] RDMA/rxe: Remove unnecessary check for qp->is_user/cq->is_user Xiao Yang
2021-09-02 8:46 ` [PATCH v2 2/5] RDMA/rxe: Remove the common is_user member of struct rxe_qp Xiao Yang
@ 2021-09-02 8:46 ` Xiao Yang
2021-09-14 18:33 ` Jason Gunthorpe
2021-09-02 8:46 ` [PATCH v2 4/5] RDMA/rxe: Set partial attributes when completion status != IBV_WC_SUCCESS Xiao Yang
2021-09-02 8:46 ` [PATCH v2 5/5] RDMA/rxe: Remove duplicate settings Xiao Yang
4 siblings, 1 reply; 12+ messages in thread
From: Xiao Yang @ 2021-09-02 8:46 UTC (permalink / raw)
To: linux-rdma; +Cc: rpearsonhpe, zyjzyj2000, jgg, leon, Xiao Yang
Make all is_user members of struct rxe_sq/rxe_cq/rxe_srq/rxe_cq
has the same type.
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
drivers/infiniband/sw/rxe/rxe_cq.c | 3 +--
drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +-
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
index aef288f164fd..fd655e41d621 100644
--- a/drivers/infiniband/sw/rxe/rxe_cq.c
+++ b/drivers/infiniband/sw/rxe/rxe_cq.c
@@ -81,8 +81,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
return err;
}
- if (uresp)
- cq->is_user = 1;
+ cq->is_user = uresp ? true : false;
cq->is_dying = false;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index bb5fb157d073..645eaea564ca 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -64,7 +64,7 @@ struct rxe_cq {
spinlock_t cq_lock;
u8 notify;
bool is_dying;
- int is_user;
+ bool is_user;
struct tasklet_struct comp_task;
};
--
2.23.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* Re: [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool
2021-09-02 8:46 ` [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool Xiao Yang
@ 2021-09-14 18:33 ` Jason Gunthorpe
2021-09-16 9:17 ` yangx.jy
0 siblings, 1 reply; 12+ messages in thread
From: Jason Gunthorpe @ 2021-09-14 18:33 UTC (permalink / raw)
To: Xiao Yang; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On Thu, Sep 02, 2021 at 04:46:38PM +0800, Xiao Yang wrote:
> Make all is_user members of struct rxe_sq/rxe_cq/rxe_srq/rxe_cq
> has the same type.
>
> Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
> drivers/infiniband/sw/rxe/rxe_cq.c | 3 +--
> drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +-
> 2 files changed, 2 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
> index aef288f164fd..fd655e41d621 100644
> +++ b/drivers/infiniband/sw/rxe/rxe_cq.c
> @@ -81,8 +81,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
> return err;
> }
>
> - if (uresp)
> - cq->is_user = 1;
> + cq->is_user = uresp ? true : false;
When you resend this series we don't need any of the ternary
expressions, when things implicitly cast to bool they are converted to
1/0 using the same logic as any other boolean expression. So just
write
cq->is_user = uresp;
Jason
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool
2021-09-14 18:33 ` Jason Gunthorpe
@ 2021-09-16 9:17 ` yangx.jy
0 siblings, 0 replies; 12+ messages in thread
From: yangx.jy @ 2021-09-16 9:17 UTC (permalink / raw)
To: Jason Gunthorpe; +Cc: linux-rdma, rpearsonhpe, zyjzyj2000, leon
On 2021/9/15 2:33, Jason Gunthorpe wrote:
> On Thu, Sep 02, 2021 at 04:46:38PM +0800, Xiao Yang wrote:
>> Make all is_user members of struct rxe_sq/rxe_cq/rxe_srq/rxe_cq
>> has the same type.
>>
>> Signed-off-by: Xiao Yang<yangx.jy@fujitsu.com>
>> drivers/infiniband/sw/rxe/rxe_cq.c | 3 +--
>> drivers/infiniband/sw/rxe/rxe_verbs.h | 2 +-
>> 2 files changed, 2 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
>> index aef288f164fd..fd655e41d621 100644
>> +++ b/drivers/infiniband/sw/rxe/rxe_cq.c
>> @@ -81,8 +81,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
>> return err;
>> }
>>
>> - if (uresp)
>> - cq->is_user = 1;
>> + cq->is_user = uresp ? true : false;
> When you resend this series we don't need any of the ternary
> expressions, when things implicitly cast to bool they are converted to
> 1/0 using the same logic as any other boolean expression. So just
> write
>
> cq->is_user = uresp;
Hi Jason,
Thanks for your suggestion.
I will update it in the next version.
Best Regards,
Xiao Yang
> Jason
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH v2 4/5] RDMA/rxe: Set partial attributes when completion status != IBV_WC_SUCCESS
2021-09-02 8:46 [PATCH v2 0/5] RDMA/rxe: Do some cleanup Xiao Yang
` (2 preceding siblings ...)
2021-09-02 8:46 ` [PATCH v2 3/5] RDMA/rxe: Change the is_user member of struct rxe_cq to bool Xiao Yang
@ 2021-09-02 8:46 ` Xiao Yang
2021-09-02 8:46 ` [PATCH v2 5/5] RDMA/rxe: Remove duplicate settings Xiao Yang
4 siblings, 0 replies; 12+ messages in thread
From: Xiao Yang @ 2021-09-02 8:46 UTC (permalink / raw)
To: linux-rdma; +Cc: rpearsonhpe, zyjzyj2000, jgg, leon, Xiao Yang
As ibv_poll_cq()'s manual said, only partial attributes are valid
when completion status != IBV_WC_SUCCESS.
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
drivers/infiniband/sw/rxe/rxe_comp.c | 45 +++++++++++++++-------------
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index fb7741ec5cd3..1bbce4af1414 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -383,30 +383,35 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,
static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
struct rxe_cqe *cqe)
{
+ struct ib_wc *wc = &cqe->ibwc;
+ struct ib_uverbs_wc *uwc = &cqe->uibwc;
+
memset(cqe, 0, sizeof(*cqe));
if (!qp->sq.is_user) {
- struct ib_wc *wc = &cqe->ibwc;
-
- wc->wr_id = wqe->wr.wr_id;
- wc->status = wqe->status;
- wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
- if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
- wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
- wc->wc_flags = IB_WC_WITH_IMM;
- wc->byte_len = wqe->dma.length;
- wc->qp = &qp->ibqp;
+ wc->wr_id = wqe->wr.wr_id;
+ wc->status = wqe->status;
+ wc->qp = &qp->ibqp;
} else {
- struct ib_uverbs_wc *uwc = &cqe->uibwc;
-
- uwc->wr_id = wqe->wr.wr_id;
- uwc->status = wqe->status;
- uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
- if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
- wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
- uwc->wc_flags = IB_WC_WITH_IMM;
- uwc->byte_len = wqe->dma.length;
- uwc->qp_num = qp->ibqp.qp_num;
+ uwc->wr_id = wqe->wr.wr_id;
+ uwc->status = wqe->status;
+ uwc->qp_num = qp->ibqp.qp_num;
+ }
+
+ if (wqe->status == IB_WC_SUCCESS) {
+ if (!qp->sq.is_user) {
+ wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+ if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+ wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+ wc->wc_flags = IB_WC_WITH_IMM;
+ wc->byte_len = wqe->dma.length;
+ } else {
+ uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+ if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+ wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+ uwc->wc_flags = IB_WC_WITH_IMM;
+ uwc->byte_len = wqe->dma.length;
+ }
}
}
--
2.23.0
^ permalink raw reply related [flat|nested] 12+ messages in thread* [PATCH v2 5/5] RDMA/rxe: Remove duplicate settings
2021-09-02 8:46 [PATCH v2 0/5] RDMA/rxe: Do some cleanup Xiao Yang
` (3 preceding siblings ...)
2021-09-02 8:46 ` [PATCH v2 4/5] RDMA/rxe: Set partial attributes when completion status != IBV_WC_SUCCESS Xiao Yang
@ 2021-09-02 8:46 ` Xiao Yang
4 siblings, 0 replies; 12+ messages in thread
From: Xiao Yang @ 2021-09-02 8:46 UTC (permalink / raw)
To: linux-rdma; +Cc: rpearsonhpe, zyjzyj2000, jgg, leon, Xiao Yang
Remove duplicate settings for vendor_err and qp_num.
Signed-off-by: Xiao Yang <yangx.jy@fujitsu.com>
---
drivers/infiniband/sw/rxe/rxe_resp.c | 4 ----
1 file changed, 4 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 71406a49fca3..ce89ade59527 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -893,7 +893,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
wc->opcode = (pkt->mask & RXE_IMMDT_MASK &&
pkt->mask & RXE_WRITE_MASK) ?
IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
- wc->vendor_err = 0;
wc->byte_len = (pkt->mask & RXE_IMMDT_MASK &&
pkt->mask & RXE_WRITE_MASK) ?
qp->resp.length : wqe->dma.length - wqe->dma.resid;
@@ -914,8 +913,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
uwc->ex.invalidate_rkey = ieth_rkey(pkt);
}
- uwc->qp_num = qp->ibqp.qp_num;
-
if (pkt->mask & RXE_DETH_MASK)
uwc->src_qp = deth_sqp(pkt);
@@ -947,7 +944,6 @@ static enum resp_states do_complete(struct rxe_qp *qp,
if (pkt->mask & RXE_DETH_MASK)
wc->src_qp = deth_sqp(pkt);
- wc->qp = &qp->ibqp;
wc->port_num = qp->attr.port_num;
}
}
--
2.23.0
^ permalink raw reply related [flat|nested] 12+ messages in thread