* [PATCH v1 1/2] svcrdma: Remove svc_rdma_wq
2019-07-23 19:20 [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
@ 2019-07-23 19:20 ` Chuck Lever
2019-07-23 19:21 ` [PATCH v1 2/2] svcrdma: Use llist for managing cache of recv_ctxts Chuck Lever
2019-08-06 15:55 ` [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
2 siblings, 0 replies; 6+ messages in thread
From: Chuck Lever @ 2019-07-23 19:20 UTC (permalink / raw)
To: linux-nfs, linux-rdma
Clean up: the system workqueue will work just as well.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
include/linux/sunrpc/svc_rdma.h | 1 -
net/sunrpc/xprtrdma/svc_rdma.c | 7 -------
net/sunrpc/xprtrdma/svc_rdma_transport.c | 3 ++-
3 files changed, 2 insertions(+), 9 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 981f0d726ad4..edb39900fe04 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -200,7 +200,6 @@ extern struct svc_xprt_class svc_rdma_bc_class;
#endif
/* svc_rdma.c */
-extern struct workqueue_struct *svc_rdma_wq;
extern int svc_rdma_init(void);
extern void svc_rdma_cleanup(void);
diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c
index abdb3004a1e3..97bca509a391 100644
--- a/net/sunrpc/xprtrdma/svc_rdma.c
+++ b/net/sunrpc/xprtrdma/svc_rdma.c
@@ -73,8 +73,6 @@ atomic_t rdma_stat_rq_prod;
atomic_t rdma_stat_sq_poll;
atomic_t rdma_stat_sq_prod;
-struct workqueue_struct *svc_rdma_wq;
-
/*
* This function implements reading and resetting an atomic_t stat
* variable through read/write to a proc file. Any write to the file
@@ -230,7 +228,6 @@ static struct ctl_table svcrdma_root_table[] = {
void svc_rdma_cleanup(void)
{
dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
- destroy_workqueue(svc_rdma_wq);
if (svcrdma_table_header) {
unregister_sysctl_table(svcrdma_table_header);
svcrdma_table_header = NULL;
@@ -246,10 +243,6 @@ int svc_rdma_init(void)
dprintk("\tmax_bc_requests : %u\n", svcrdma_max_bc_requests);
dprintk("\tmax_inline : %d\n", svcrdma_max_req_size);
- svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
- if (!svc_rdma_wq)
- return -ENOMEM;
-
if (!svcrdma_table_header)
svcrdma_table_header =
register_sysctl_table(svcrdma_root_table);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 7df6de6e9162..f8e48d6824a0 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -632,8 +632,9 @@ static void svc_rdma_free(struct svc_xprt *xprt)
{
struct svcxprt_rdma *rdma =
container_of(xprt, struct svcxprt_rdma, sc_xprt);
+
INIT_WORK(&rdma->sc_work, __svc_rdma_free);
- queue_work(svc_rdma_wq, &rdma->sc_work);
+ schedule_work(&rdma->sc_work);
}
static int svc_rdma_has_wspace(struct svc_xprt *xprt)
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH v1 2/2] svcrdma: Use llist for managing cache of recv_ctxts
2019-07-23 19:20 [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
2019-07-23 19:20 ` [PATCH v1 1/2] svcrdma: Remove svc_rdma_wq Chuck Lever
@ 2019-07-23 19:21 ` Chuck Lever
2019-08-06 15:55 ` [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
2 siblings, 0 replies; 6+ messages in thread
From: Chuck Lever @ 2019-07-23 19:21 UTC (permalink / raw)
To: linux-nfs, linux-rdma
Use a wait-free mechanism for managing the svc_rdma_recv_ctxts free
list. Subsequently, sc_recv_lock can be eliminated.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
include/linux/sunrpc/svc_rdma.h | 5 +++--
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 24 ++++++++++--------------
net/sunrpc/xprtrdma/svc_rdma_transport.c | 3 +--
3 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index edb39900fe04..40f65888dd38 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -42,6 +42,7 @@
#ifndef SVC_RDMA_H
#define SVC_RDMA_H
+#include <linux/llist.h>
#include <linux/sunrpc/xdr.h>
#include <linux/sunrpc/svcsock.h>
#include <linux/sunrpc/rpc_rdma.h>
@@ -107,8 +108,7 @@ struct svcxprt_rdma {
struct list_head sc_read_complete_q;
struct work_struct sc_work;
- spinlock_t sc_recv_lock;
- struct list_head sc_recv_ctxts;
+ struct llist_head sc_recv_ctxts;
};
/* sc_flags */
#define RDMAXPRT_CONN_PENDING 3
@@ -125,6 +125,7 @@ enum {
#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
struct svc_rdma_recv_ctxt {
+ struct llist_node rc_node;
struct list_head rc_list;
struct ib_recv_wr rc_recv_wr;
struct ib_cqe rc_cqe;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 65e2fb9aac65..96bccd398469 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -172,9 +172,10 @@ static void svc_rdma_recv_ctxt_destroy(struct svcxprt_rdma *rdma,
void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma)
{
struct svc_rdma_recv_ctxt *ctxt;
+ struct llist_node *node;
- while ((ctxt = svc_rdma_next_recv_ctxt(&rdma->sc_recv_ctxts))) {
- list_del(&ctxt->rc_list);
+ while ((node = llist_del_first(&rdma->sc_recv_ctxts))) {
+ ctxt = llist_entry(node, struct svc_rdma_recv_ctxt, rc_node);
svc_rdma_recv_ctxt_destroy(rdma, ctxt);
}
}
@@ -183,21 +184,18 @@ static struct svc_rdma_recv_ctxt *
svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma)
{
struct svc_rdma_recv_ctxt *ctxt;
+ struct llist_node *node;
- spin_lock(&rdma->sc_recv_lock);
- ctxt = svc_rdma_next_recv_ctxt(&rdma->sc_recv_ctxts);
- if (!ctxt)
+ node = llist_del_first(&rdma->sc_recv_ctxts);
+ if (!node)
goto out_empty;
- list_del(&ctxt->rc_list);
- spin_unlock(&rdma->sc_recv_lock);
+ ctxt = llist_entry(node, struct svc_rdma_recv_ctxt, rc_node);
out:
ctxt->rc_page_count = 0;
return ctxt;
out_empty:
- spin_unlock(&rdma->sc_recv_lock);
-
ctxt = svc_rdma_recv_ctxt_alloc(rdma);
if (!ctxt)
return NULL;
@@ -218,11 +216,9 @@ void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
for (i = 0; i < ctxt->rc_page_count; i++)
put_page(ctxt->rc_pages[i]);
- if (!ctxt->rc_temp) {
- spin_lock(&rdma->sc_recv_lock);
- list_add(&ctxt->rc_list, &rdma->sc_recv_ctxts);
- spin_unlock(&rdma->sc_recv_lock);
- } else
+ if (!ctxt->rc_temp)
+ llist_add(&ctxt->rc_node, &rdma->sc_recv_ctxts);
+ else
svc_rdma_recv_ctxt_destroy(rdma, ctxt);
}
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index f8e48d6824a0..935a4a23dff7 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -140,14 +140,13 @@ static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
INIT_LIST_HEAD(&cma_xprt->sc_rq_dto_q);
INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
INIT_LIST_HEAD(&cma_xprt->sc_send_ctxts);
- INIT_LIST_HEAD(&cma_xprt->sc_recv_ctxts);
+ init_llist_head(&cma_xprt->sc_recv_ctxts);
INIT_LIST_HEAD(&cma_xprt->sc_rw_ctxts);
init_waitqueue_head(&cma_xprt->sc_send_wait);
spin_lock_init(&cma_xprt->sc_lock);
spin_lock_init(&cma_xprt->sc_rq_dto_lock);
spin_lock_init(&cma_xprt->sc_send_lock);
- spin_lock_init(&cma_xprt->sc_recv_lock);
spin_lock_init(&cma_xprt->sc_rw_ctxt_lock);
/*
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next
2019-07-23 19:20 [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
2019-07-23 19:20 ` [PATCH v1 1/2] svcrdma: Remove svc_rdma_wq Chuck Lever
2019-07-23 19:21 ` [PATCH v1 2/2] svcrdma: Use llist for managing cache of recv_ctxts Chuck Lever
@ 2019-08-06 15:55 ` Chuck Lever
2019-08-06 19:58 ` Bruce Fields
2019-08-16 16:06 ` Bruce Fields
2 siblings, 2 replies; 6+ messages in thread
From: Chuck Lever @ 2019-08-06 15:55 UTC (permalink / raw)
To: Bruce Fields; +Cc: Linux NFS Mailing List, linux-rdma
Hi Bruce-
Would you consider these for v5.4?
> On Jul 23, 2019, at 3:20 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
>
> Hi-
>
> Two server side patches for NFSD. Both are minor.
>
> ---
>
> Chuck Lever (2):
> svcrdma: Remove svc_rdma_wq
> svcrdma: Use llist for managing cache of recv_ctxts
>
>
> include/linux/sunrpc/svc_rdma.h | 6 +++---
> net/sunrpc/xprtrdma/svc_rdma.c | 7 -------
> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 24 ++++++++++--------------
> net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++---
> 4 files changed, 16 insertions(+), 27 deletions(-)
>
> --
> Chuck Lever
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next
2019-08-06 15:55 ` [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
@ 2019-08-06 19:58 ` Bruce Fields
2019-08-16 16:06 ` Bruce Fields
1 sibling, 0 replies; 6+ messages in thread
From: Bruce Fields @ 2019-08-06 19:58 UTC (permalink / raw)
To: Chuck Lever; +Cc: Linux NFS Mailing List, linux-rdma
On Tue, Aug 06, 2019 at 11:55:29AM -0400, Chuck Lever wrote:
> Would you consider these for v5.4?
Yes, definitely.
I have an intermittent failure in my regression tests that it looks like
I introduced in the most recent merge window. I'd like to track that
down before pushing out a 5.4 tree. Hopefully in the next day or so....
--b.
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next
2019-08-06 15:55 ` [PATCH v1 0/2] NFS/RDMA-related NFSD patches for -next Chuck Lever
2019-08-06 19:58 ` Bruce Fields
@ 2019-08-16 16:06 ` Bruce Fields
1 sibling, 0 replies; 6+ messages in thread
From: Bruce Fields @ 2019-08-16 16:06 UTC (permalink / raw)
To: Chuck Lever; +Cc: Linux NFS Mailing List, linux-rdma
On Tue, Aug 06, 2019 at 11:55:29AM -0400, Chuck Lever wrote:
> Would you consider these for v5.4?
Apologies, I can't find these in my mail anywhere, for some
reason--would you mind resending?
--b.
>
>
> > On Jul 23, 2019, at 3:20 PM, Chuck Lever <chuck.lever@oracle.com> wrote:
> >
> > Hi-
> >
> > Two server side patches for NFSD. Both are minor.
> >
> > ---
> >
> > Chuck Lever (2):
> > svcrdma: Remove svc_rdma_wq
> > svcrdma: Use llist for managing cache of recv_ctxts
> >
> >
> > include/linux/sunrpc/svc_rdma.h | 6 +++---
> > net/sunrpc/xprtrdma/svc_rdma.c | 7 -------
> > net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 24 ++++++++++--------------
> > net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++---
> > 4 files changed, 16 insertions(+), 27 deletions(-)
> >
> > --
> > Chuck Lever
^ permalink raw reply [flat|nested] 6+ messages in thread