[RFC 0/39] nfs41 client backchannel for 2.6.31

[RFC 0/39] nfs41 client backchannel for 2.6.31

[Benny Halevy]

Trond,

Please review the following patchset and consider for 2.6.31.

It is also available on
git://linux-nfs.org/~bhalevy/linux-pnfs.git nfs41-for-2.6.31
which is based onto
git://git.linux-nfs.org/~trondmy/nfs-2.6.git nfsv41

Thanks,

Benny

[RFC 01/39] nfs41: Add ability to read RPC call direction on TCP stream.
[RFC 02/39] nfs41: Skip past the RPC call direction
[RFC 03/39] nfs41: Refactor nfs4_{init,destroy}_callback for nfs4.0
[RFC 04/39] nfs41: minorversion support for nfs4_{init,destroy}_callback
[RFC 05/39] nfs41: client callback structures
[RFC 06/39] nfs41: Initialize new rpc_xprt callback related fields
[RFC 07/39] nfs41: New backchannel helper routines
[RFC 08/39] nfs41: New include/linux/sunrpc/bc_xprt.h
[RFC 09/39] nfs41: New xs_tcp_read_data()
[RFC 10/39] nfs41: Add backchannel processing support to RPC state machine
[RFC 11/39] nfs41: Backchannel callback service helper routines
[RFC 12/39] nfs41: Refactor svc_process()
[RFC 13/39] nfs41: Backchannel bc_svc_process()
[RFC 14/39] nfs41: Implement NFSv4.1 callback service process.
[RFC 15/39] nfs41: sunrpc: provide functions to create and destroy a svc_xprt for backchannel use
[RFC 16/39] nfs41: sunrpc: add a struct svc_xprt pointer to struct svc_serv for backchannel use
[RFC 17/39] nfs41: create a svc_xprt for nfs41 callback thread and use for incoming callbacks
[RFC 18/39] nfs41: save svc_serv in nfs_callback_info
[RFC 19/39] nfs41: Allow NFSv4 and NFSv4.1 callback services to coexist
[RFC 20/39] nfs41: Setup the backchannel
[RFC 21/39] nfs41: Client indicates presence of NFSv4.1 callback channel.
[RFC 22/39] nfs41: Get the rpc_xprt * from the rpc_rqst instead of the rpc_clnt.
[RFC 23/39] nfs41: Release backchannel resources associated with session
[RFC 24/39] nfs41: store minorversion in cb_compound_hdr_arg
[RFC 25/39] nfs41: decode minorversion 1 cb_compound header
[RFC 26/39] nfs41: callback numbers definitions
[RFC 27/39] nfs41: consider minorversion in callback_xdr:process_op
[RFC 28/39] nfs41: define CB_NOTIFY_DEVICEID as not supported
[RFC 29/39] nfs41: cb_sequence protocol level data structures
[RFC 30/39] nfs41: cb_sequence proc implementation
[RFC 31/39] nfs41: cb_sequence xdr implementation
[RFC 32/39] nfs41: verify CB_SEQUENCE position in callback compound
[RFC 33/39] nfs41: Rename rq_received to rq_reply_bytes_recvd
[RFC 34/39] nfs41: Backchannel: update cb_sequence args and results
[RFC 35/39] nfs41: Backchannel: Refactor nfs4_reset_slot_table()
[RFC 36/39] nfs41: Backchannel: Refactor nfs4_init_slot_table()
[RFC 37/39] nfs41: Backchannel: Add a backchannel slot table to the session
[RFC 38/39] nfs41: Backchannel: New find_client_with_session()
[RFC 39/39] nfs41: Backchannel: CB_SEQUENCE validation

[RFC 01/39] nfs41: Add ability to read RPC call direction on TCP stream.

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

NFSv4.1 callbacks can arrive over an existing connection. This patch adds
the logic to read the RPC call direction (call or reply). It does this by
updating the state machine to look for the call direction invoking
xs_tcp_read_calldir(...) after reading the XID.

[nfs41: Keep track of RPC call/reply direction with a flag]

As per 11/14/08 review of RFC 53/85.

Add a new flag to track whether the incoming message is an RPC call or an
RPC reply.  TCP_RPC_REPLY is set in the 'struct sock_xprt' tcp_flags in
xs_tcp_read_calldir() if the message is an RPC reply sent on the forechannel.
It is cleared if the message is an RPC request sent on the back channel.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/xprtsock.c |   52 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index d40ff50..692b517 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -270,6 +270,12 @@ struct sock_xprt {
 #define TCP_RCV_COPY_FRAGHDR	(1UL << 1)
 #define TCP_RCV_COPY_XID	(1UL << 2)
 #define TCP_RCV_COPY_DATA	(1UL << 3)
+#define TCP_RCV_COPY_CALLDIR	(1UL << 4)
+
+/*
+ * TCP RPC flags
+ */
+#define TCP_RPC_REPLY		(1UL << 5)
 
 static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
 {
@@ -945,7 +951,7 @@ static inline void xs_tcp_read_fraghdr(struct rpc_xprt *xprt, struct xdr_skb_rea
 	transport->tcp_offset = 0;
 
 	/* Sanity check of the record length */
-	if (unlikely(transport->tcp_reclen < 4)) {
+	if (unlikely(transport->tcp_reclen < 8)) {
 		dprintk("RPC:       invalid TCP record fragment length\n");
 		xprt_force_disconnect(xprt);
 		return;
@@ -980,13 +986,48 @@ static inline void xs_tcp_read_xid(struct sock_xprt *transport, struct xdr_skb_r
 	if (used != len)
 		return;
 	transport->tcp_flags &= ~TCP_RCV_COPY_XID;
-	transport->tcp_flags |= TCP_RCV_COPY_DATA;
+	transport->tcp_flags |= TCP_RCV_COPY_CALLDIR;
 	transport->tcp_copied = 4;
-	dprintk("RPC:       reading reply for XID %08x\n",
+	dprintk("RPC:       reading %s XID %08x\n",
+			(transport->tcp_flags & TCP_RPC_REPLY) ? "reply for"
+							      : "request with",
 			ntohl(transport->tcp_xid));
 	xs_tcp_check_fraghdr(transport);
 }
 
+static inline void xs_tcp_read_calldir(struct sock_xprt *transport,
+				       struct xdr_skb_reader *desc)
+{
+	size_t len, used;
+	u32 offset;
+	__be32	calldir;
+
+	/*
+	 * We want transport->tcp_offset to be 8 at the end of this routine
+	 * (4 bytes for the xid and 4 bytes for the call/reply flag).
+	 * When this function is called for the first time,
+	 * transport->tcp_offset is 4 (after having already read the xid).
+	 */
+	offset = transport->tcp_offset - sizeof(transport->tcp_xid);
+	len = sizeof(calldir) - offset;
+	dprintk("RPC:       reading CALL/REPLY flag (%Zu bytes)\n", len);
+	used = xdr_skb_read_bits(desc, &calldir, len);
+	transport->tcp_offset += used;
+	if (used != len)
+		return;
+	transport->tcp_flags &= ~TCP_RCV_COPY_CALLDIR;
+	transport->tcp_flags |= TCP_RCV_COPY_DATA;
+	transport->tcp_copied += 4;
+	if (ntohl(calldir) == RPC_REPLY)
+		transport->tcp_flags |= TCP_RPC_REPLY;
+	else
+		transport->tcp_flags &= ~TCP_RPC_REPLY;
+	dprintk("RPC:       reading %s CALL/REPLY flag %08x\n",
+			(transport->tcp_flags & TCP_RPC_REPLY) ?
+				"reply for" : "request with", calldir);
+	xs_tcp_check_fraghdr(transport);
+}
+
 static inline void xs_tcp_read_request(struct rpc_xprt *xprt, struct xdr_skb_reader *desc)
 {
 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
@@ -1103,6 +1144,11 @@ static int xs_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, uns
 			xs_tcp_read_xid(transport, &desc);
 			continue;
 		}
+		/* Read in the call/reply flag */
+		if (transport->tcp_flags & TCP_RCV_COPY_CALLDIR) {
+			xs_tcp_read_calldir(transport, &desc);
+			continue;
+		}
 		/* Read in the request data */
 		if (transport->tcp_flags & TCP_RCV_COPY_DATA) {
 			xs_tcp_read_request(xprt, &desc);
-- 
1.6.2.1

[RFC 02/39] nfs41: Skip past the RPC call direction

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

xs_tcp_read_calldir() is now responsible for reading the RPC call
direction and determining whether it is a reply or a callback request.
The call to xdr_skb_read_bits() inside xs_tcp_read_calldir() moves the
xdr_skb_reader offset past the RPC call direction (offset should be
equal to 8).  Therefore xs_tcp_read_common() called from
xs_tcp_ready_reply() should be copying the TCP buffer starting past the
RPC call direction.  It is now necessary to read the RPC call direction
earlier to determine whether to call the reply handler or the callback
handler.

call_verify() should therefore skip past the XID and call/reply flag.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/clnt.c |   13 +++++++------
 1 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index d00e813..76d7d46 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1390,13 +1390,14 @@ rpc_verify_header(struct rpc_task *task)
 	}
 	if ((len -= 3) < 0)
 		goto out_overflow;
-	p += 1;	/* skip XID */
 
-	if ((n = ntohl(*p++)) != RPC_REPLY) {
-		dprintk("RPC: %5u %s: not an RPC reply: %x\n",
-				task->tk_pid, __func__, n);
-		goto out_garbage;
-	}
+	/*
+	 * Skip the XID and call direction.
+	 * The underlying transport has read the XID and RPC call direction
+	 * to determine this is an RPC reply.
+	 */
+	p += 2;
+
 	if ((n = ntohl(*p++)) != RPC_MSG_ACCEPTED) {
 		if (--len < 0)
 			goto out_overflow;
-- 
1.6.2.1

[RFC 03/39] nfs41: Refactor nfs4_{init,destroy}_callback for nfs4.0

[Benny Halevy]

Refactor-out code to bring the callback service up and down.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/client.c |   46 +++++++++++++++++++++++++++++++++++++---------
 1 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 3e232bf..d9657d4 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -47,6 +47,9 @@
 #include "internal.h"
 #include "fscache.h"
 
+static int nfs4_init_callback(struct nfs_client *);
+static void nfs4_destroy_callback(struct nfs_client *);
+
 #define NFSDBG_FACILITY		NFSDBG_CLIENT
 
 static DEFINE_SPINLOCK(nfs_client_lock);
@@ -121,11 +124,8 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 
 	clp->rpc_ops = cl_init->rpc_ops;
 
-	if (cl_init->rpc_ops->version == 4) {
-		if (nfs_callback_up() < 0)
-			goto error_2;
-		__set_bit(NFS_CS_CALLBACK, &clp->cl_res_state);
-	}
+	if (nfs4_init_callback(clp) < 0)
+		goto error_2;
 
 	atomic_set(&clp->cl_count, 1);
 	clp->cl_cons_state = NFS_CS_INITING;
@@ -162,8 +162,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 	return clp;
 
 error_3:
-	if (__test_and_clear_bit(NFS_CS_CALLBACK, &clp->cl_res_state))
-		nfs_callback_down();
+	nfs4_destroy_callback(clp);
 error_2:
 	kfree(clp);
 error_0:
@@ -184,6 +183,17 @@ static void nfs4_shutdown_client(struct nfs_client *clp)
 }
 
 /*
+ * Destroy the NFS4 callback service
+ */
+static void nfs4_destroy_callback(struct nfs_client *clp)
+{
+#ifdef CONFIG_NFS_V4
+	if (__test_and_clear_bit(NFS_CS_CALLBACK, &clp->cl_res_state))
+		nfs_callback_down();
+#endif /* CONFIG_NFS_V4 */
+}
+
+/*
  * Clears/puts all minor version specific parts from an nfs_client struct
  * reverting it to minorversion 0.
  */
@@ -215,8 +225,7 @@ static void nfs_free_client(struct nfs_client *clp)
 	if (!IS_ERR(clp->cl_rpcclient))
 		rpc_shutdown_client(clp->cl_rpcclient);
 
-	if (__test_and_clear_bit(NFS_CS_CALLBACK, &clp->cl_res_state))
-		nfs_callback_down();
+	nfs4_destroy_callback(clp);
 
 	if (clp->cl_machine_cred != NULL)
 		put_rpccred(clp->cl_machine_cred);
@@ -1088,6 +1097,25 @@ error:
 
 #ifdef CONFIG_NFS_V4
 /*
+ * Initialize the NFS4 callback service
+ */
+static int nfs4_init_callback(struct nfs_client *clp)
+{
+	int error;
+
+	if (clp->rpc_ops->version == 4) {
+		error = nfs_callback_up();
+		if (error < 0) {
+			dprintk("%s: failed to start callback. Error = %d\n",
+				__func__, error);
+			return error;
+		}
+		__set_bit(NFS_CS_CALLBACK, &clp->cl_res_state);
+	}
+	return 0;
+}
+
+/*
  * Initialize the minor version specific parts of an NFS4 client record
  */
 static int nfs4_init_client_minor_version(struct nfs_client *clp)
-- 
1.6.2.1

[RFC 04/39] nfs41: minorversion support for nfs4_{init,destroy}_callback

[Benny Halevy]

move nfs4_init_callback into nfs4_init_client_minor_version
and nfs4_destroy_callback into nfs4_clear_client_minor_version

as these need to happen also when auto-negotiating the minorversion
once the callback service for nfs41 becomes different than for nfs4.0

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: Fix checkpatch warning]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.c |   67 ++++++++++++++++++++++++++++++++++++++---------------
 fs/nfs/callback.h |    2 +-
 fs/nfs/client.c   |   21 +++++-----------
 3 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index a886e69..3926046 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -59,7 +59,7 @@ module_param_call(callback_tcpport, param_set_port, param_get_int,
  * This is the callback kernel thread.
  */
 static int
-nfs_callback_svc(void *vrqstp)
+nfs4_callback_svc(void *vrqstp)
 {
 	int err, preverr = 0;
 	struct svc_rqst *rqstp = vrqstp;
@@ -97,20 +97,12 @@ nfs_callback_svc(void *vrqstp)
 }
 
 /*
- * Bring up the callback thread if it is not already up.
+ * Prepare to bring up the NFSv4 callback service
  */
-int nfs_callback_up(void)
+struct svc_rqst *
+nfs4_callback_up(struct svc_serv *serv)
 {
-	struct svc_serv *serv = NULL;
-	int ret = 0;
-
-	mutex_lock(&nfs_callback_mutex);
-	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
-		goto out;
-	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
-	ret = -ENOMEM;
-	if (!serv)
-		goto out_err;
+	int ret;
 
 	ret = svc_create_xprt(serv, "tcp", PF_INET,
 				nfs_callback_set_tcpport, SVC_SOCK_ANONYMOUS);
@@ -131,18 +123,55 @@ int nfs_callback_up(void)
 		goto out_err;
 #endif	/* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */
 
-	nfs_callback_info.rqst = svc_prepare_thread(serv, &serv->sv_pools[0]);
-	if (IS_ERR(nfs_callback_info.rqst)) {
-		ret = PTR_ERR(nfs_callback_info.rqst);
-		nfs_callback_info.rqst = NULL;
+	return svc_prepare_thread(serv, &serv->sv_pools[0]);
+
+out_err:
+	if (ret == 0)
+		ret = -ENOMEM;
+	return ERR_PTR(ret);
+}
+
+/*
+ * Bring up the callback thread if it is not already up.
+ */
+int nfs_callback_up(u32 minorversion, void *args)
+{
+	struct svc_serv *serv = NULL;
+	struct svc_rqst *rqstp;
+	int (*callback_svc)(void *vrqstp);
+	char svc_name[12];
+	int ret = 0;
+
+	mutex_lock(&nfs_callback_mutex);
+	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
+		goto out;
+	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
+	if (!serv) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+
+	/* FIXME: either 4.0 or 4.1 callback service can be up at a time
+	 * need to monitor and control them both */
+	if (!minorversion) {
+		rqstp = nfs4_callback_up(serv);
+		callback_svc = nfs4_callback_svc;
+	} else {
+		BUG();	/* for now */
+	}
+
+	if (IS_ERR(rqstp)) {
+		ret = PTR_ERR(rqstp);
 		goto out_err;
 	}
 
 	svc_sock_update_bufs(serv);
 
-	nfs_callback_info.task = kthread_run(nfs_callback_svc,
+	sprintf(svc_name, "nfsv4.%u-svc", minorversion);
+	nfs_callback_info.rqst = rqstp;
+	nfs_callback_info.task = kthread_run(callback_svc,
 					     nfs_callback_info.rqst,
-					     "nfsv4-svc");
+					     svc_name);
 	if (IS_ERR(nfs_callback_info.task)) {
 		ret = PTR_ERR(nfs_callback_info.task);
 		svc_exit_thread(nfs_callback_info.rqst);
diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index e110e28..7f12e65 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -63,7 +63,7 @@ extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getat
 extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
 
 #ifdef CONFIG_NFS_V4
-extern int nfs_callback_up(void);
+extern int nfs_callback_up(u32 minorversion, void *args);
 extern void nfs_callback_down(void);
 #else
 #define nfs_callback_up()	(0)
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index d9657d4..df2b40d 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -47,9 +47,6 @@
 #include "internal.h"
 #include "fscache.h"
 
-static int nfs4_init_callback(struct nfs_client *);
-static void nfs4_destroy_callback(struct nfs_client *);
-
 #define NFSDBG_FACILITY		NFSDBG_CLIENT
 
 static DEFINE_SPINLOCK(nfs_client_lock);
@@ -124,9 +121,6 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 
 	clp->rpc_ops = cl_init->rpc_ops;
 
-	if (nfs4_init_callback(clp) < 0)
-		goto error_2;
-
 	atomic_set(&clp->cl_count, 1);
 	clp->cl_cons_state = NFS_CS_INITING;
 
@@ -136,7 +130,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 	if (cl_init->hostname) {
 		clp->cl_hostname = kstrdup(cl_init->hostname, GFP_KERNEL);
 		if (!clp->cl_hostname)
-			goto error_3;
+			goto error_cleanup;
 	}
 
 	INIT_LIST_HEAD(&clp->cl_superblocks);
@@ -161,9 +155,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 
 	return clp;
 
-error_3:
-	nfs4_destroy_callback(clp);
-error_2:
+error_cleanup:
 	kfree(clp);
 error_0:
 	return NULL;
@@ -207,6 +199,8 @@ static void nfs4_clear_client_minor_version(struct nfs_client *clp)
 
 	clp->cl_call_sync = _nfs4_call_sync;
 #endif /* CONFIG_NFS_V4_1 */
+
+	nfs4_destroy_callback(clp);
 }
 
 /*
@@ -225,8 +219,6 @@ static void nfs_free_client(struct nfs_client *clp)
 	if (!IS_ERR(clp->cl_rpcclient))
 		rpc_shutdown_client(clp->cl_rpcclient);
 
-	nfs4_destroy_callback(clp);
-
 	if (clp->cl_machine_cred != NULL)
 		put_rpccred(clp->cl_machine_cred);
 
@@ -1104,7 +1096,8 @@ static int nfs4_init_callback(struct nfs_client *clp)
 	int error;
 
 	if (clp->rpc_ops->version == 4) {
-		error = nfs_callback_up();
+		error = nfs_callback_up(clp->cl_minorversion,
+					clp->cl_rpcclient->cl_xprt);
 		if (error < 0) {
 			dprintk("%s: failed to start callback. Error = %d\n",
 				__func__, error);
@@ -1139,7 +1132,7 @@ static int nfs4_init_client_minor_version(struct nfs_client *clp)
 	}
 #endif /* CONFIG_NFS_V4_1 */
 
-	return 0;
+	return nfs4_init_callback(clp);
 }
 
 /*
-- 
1.6.2.1

[RFC 05/39] nfs41: client callback structures

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Adds new list of rpc_xprt structures, and a readers/writers lock to
protect the list.  The list is used to preallocate resources for
the backchannel during backchannel requests.  Callbacks are not
expected to cause significant latency, so only one callback will
be allowed at this time.

It also adds a pointer to the NFS callback service so that
requests can be directed to it for processing.

New callback members added to svc_serv. The NFSv4.1 callback service will
sleep on the svc_serv->svc_cb_waitq until new callback requests arrive.
The request will be queued in svc_serv->svc_cb_list. This patch adds this
list, the sleep queue and spinlock to svc_serv.

[nfs41: NFSv4.1 callback support]
Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/svc.h  |    8 ++++++++
 include/linux/sunrpc/xprt.h |   22 ++++++++++++++++++++++
 2 files changed, 30 insertions(+), 0 deletions(-)

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 2a30775..4a8afbd 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -96,6 +96,14 @@ struct svc_serv {
 	svc_thread_fn		sv_function;	/* main function for threads */
 	unsigned int		sv_drc_max_pages; /* Total pages for DRC */
 	unsigned int		sv_drc_pages_used;/* DRC pages used */
+#if defined(CONFIG_NFS_V4_1)
+	struct list_head	sv_cb_list;	/* queue for callback requests
+						 * that arrive over the same
+						 * connection */
+	spinlock_t		sv_cb_lock;	/* protects the svc_cb_list */
+	wait_queue_head_t	sv_cb_waitq;	/* sleep here if there are no
+						 * entries in the svc_cb_list */
+#endif /* CONFIG_NFS_V4_1 */
 };
 
 /*
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 1758d9f..b0867e4 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -97,6 +97,12 @@ struct rpc_rqst {
 
 	unsigned long		rq_xtime;	/* when transmitted */
 	int			rq_ntrans;
+
+#if defined(CONFIG_NFS_V4_1)
+	struct list_head	rq_bc_list;	/* Callback service list */
+	unsigned long		rq_bc_pa_state;	/* Backchannel prealloc state */
+	struct list_head	rq_bc_pa_list;	/* Backchannel prealloc list */
+#endif /* CONFIG_NFS_V4_1 */
 };
 #define rq_svec			rq_snd_buf.head
 #define rq_slen			rq_snd_buf.len
@@ -174,6 +180,14 @@ struct rpc_xprt {
 	spinlock_t		reserve_lock;	/* lock slot table */
 	u32			xid;		/* Next XID value to use */
 	struct rpc_task *	snd_task;	/* Task blocked in send */
+#if defined(CONFIG_NFS_V4_1)
+	struct svc_serv		*bc_serv;       /* The RPC service which will */
+						/* process the callback */
+	spinlock_t		bc_pa_lock;	/* Protects the preallocated
+						 * items */
+	struct list_head	bc_pa_list;	/* List of preallocated
+						 * backchannel rpc_rqst's */
+#endif /* CONFIG_NFS_V4_1 */
 	struct list_head	recv;
 
 	struct {
@@ -192,6 +206,14 @@ struct rpc_xprt {
 	const char		*address_strings[RPC_DISPLAY_MAX];
 };
 
+#if defined(CONFIG_NFS_V4_1)
+/*
+ * Backchannel flags
+ */
+#define	RPC_BC_PA_IN_USE	0x0001		/* Preallocated backchannel */
+						/* buffer in use */
+#endif /* CONFIG_NFS_V4_1 */
+
 struct xprt_create {
 	int			ident;		/* XPRT_TRANSPORT identifier */
 	struct sockaddr *	srcaddr;	/* optional local address */
-- 
1.6.2.1

[RFC 06/39] nfs41: Initialize new rpc_xprt callback related fields

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/xprt.c |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index a0bfe53..bbaec23 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -1051,6 +1051,11 @@ found:
 
 	INIT_LIST_HEAD(&xprt->free);
 	INIT_LIST_HEAD(&xprt->recv);
+#if defined(CONFIG_NFS_V4_1)
+	spin_lock_init(&xprt->bc_pa_lock);
+	INIT_LIST_HEAD(&xprt->bc_pa_list);
+#endif /* CONFIG_NFS_V4_1 */
+
 	INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
 	setup_timer(&xprt->timer, xprt_init_autodisconnect,
 			(unsigned long)xprt);
-- 
1.6.2.1

[RFC 07/39] nfs41: New backchannel helper routines

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

This patch introduces support to setup the callback xprt on the client side.
It allocates/ destroys the preallocated memory structures used to process
backchannel requests.

At setup time, xprt_setup_backchannel() is invoked to allocate one or
more rpc_rqst structures and substructures.  This ensures that they
are available when an RPC callback arrives.  The rpc_rqst structures
are maintained in a linked list attached to the rpc_xprt structure.
We keep track of the number of allocations so that they can be correctly
removed when the channel is destroyed.

When an RPC callback arrives, xprt_alloc_bc_request() is invoked to
obtain a preallocated rpc_rqst structure.  An rpc_xprt structure is
returned, and its RPC_BC_PREALLOC_IN_USE bit is set in
rpc_xprt->bc_flags.  The structure is removed from the the list
since it is now in use, and it will be later added back when its
user is done with it.

After the RPC callback replies, the rpc_rqst structure is returned
by invoking xprt_free_bc_request().  This clears the
RPC_BC_PREALLOC_IN_USE bit and adds it back to the list, allowing it
to be reused by a subsequent RPC callback request.

To be consistent with the reception of RPC messages, the backchannel requests
should be placed into the 'struct rpc_rqst' rq_rcv_buf, which is then in turn
copied to the 'struct rpc_rqst' rq_private_buf.

[nfs41: Preallocate rpc_rqst receive buffer for handling callbacks]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/xprt.h   |    1 +
 net/sunrpc/Makefile           |    1 +
 net/sunrpc/backchannel_rqst.c |  270 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 272 insertions(+), 0 deletions(-)
 create mode 100644 net/sunrpc/backchannel_rqst.c

diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index b0867e4..6b37724 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -183,6 +183,7 @@ struct rpc_xprt {
 #if defined(CONFIG_NFS_V4_1)
 	struct svc_serv		*bc_serv;       /* The RPC service which will */
 						/* process the callback */
+	unsigned int		bc_alloc_count;	/* Total number of preallocs */
 	spinlock_t		bc_pa_lock;	/* Protects the preallocated
 						 * items */
 	struct list_head	bc_pa_list;	/* List of preallocated
diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
index 5369aa3..4a01f96 100644
--- a/net/sunrpc/Makefile
+++ b/net/sunrpc/Makefile
@@ -13,5 +13,6 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
 	    rpcb_clnt.o timer.o xdr.o \
 	    sunrpc_syms.o cache.o rpc_pipe.o \
 	    svc_xprt.o
+sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o
 sunrpc-$(CONFIG_PROC_FS) += stats.o
 sunrpc-$(CONFIG_SYSCTL) += sysctl.o
diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c
new file mode 100644
index 0000000..7d7708a
--- /dev/null
+++ b/net/sunrpc/backchannel_rqst.c
@@ -0,0 +1,270 @@
+/******************************************************************************
+
+(c) 2007 Network Appliance, Inc.  All Rights Reserved.
+
+Network Appliance provides this source code under the GPL v2 License.
+The GPL v2 license is available at
+http://opensource.org/licenses/gpl-license.php.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+
+#include <linux/tcp.h>
+#include <linux/sunrpc/xprt.h>
+
+#ifdef RPC_DEBUG
+#define RPCDBG_FACILITY	RPCDBG_TRANS
+#endif
+
+#if defined(CONFIG_NFS_V4_1)
+
+/*
+ * Helper routines that track the number of preallocation elements
+ * on the transport.
+ */
+static inline int xprt_need_to_requeue(struct rpc_xprt *xprt)
+{
+	return xprt->bc_alloc_count > 0;
+}
+
+static inline void xprt_inc_alloc_count(struct rpc_xprt *xprt, unsigned int n)
+{
+	xprt->bc_alloc_count += n;
+}
+
+static inline int xprt_dec_alloc_count(struct rpc_xprt *xprt, unsigned int n)
+{
+	return xprt->bc_alloc_count -= n;
+}
+
+/*
+ * Free the preallocated rpc_rqst structure and the memory
+ * buffers hanging off of it.
+ */
+static void xprt_free_allocation(struct rpc_rqst *req)
+{
+	struct xdr_buf *xbufp;
+
+	dprintk("RPC:        free allocations for req= %p\n", req);
+	BUG_ON(test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state));
+	xbufp = &req->rq_private_buf;
+	free_page((unsigned long)xbufp->head[0].iov_base);
+	xbufp = &req->rq_snd_buf;
+	free_page((unsigned long)xbufp->head[0].iov_base);
+	list_del(&req->rq_bc_pa_list);
+	kfree(req);
+}
+
+/*
+ * Preallocate up to min_reqs structures and related buffers for use
+ * by the backchannel.  This function can be called multiple times
+ * when creating new sessions that use the same rpc_xprt.  The
+ * preallocated buffers are added to the pool of resources used by
+ * the rpc_xprt.  Anyone of these resources may be used used by an
+ * incoming callback request.  It's up to the higher levels in the
+ * stack to enforce that the maximum number of session slots is not
+ * being exceeded.
+ */
+int xprt_setup_backchannel(struct rpc_xprt *xprt, unsigned int min_reqs)
+{
+	struct page *page_rcv = NULL, *page_snd = NULL;
+	struct xdr_buf *xbufp = NULL;
+	struct rpc_rqst *req, *tmp;
+	struct list_head tmp_list;
+	int i;
+
+	dprintk("RPC:       setup backchannel transport\n");
+
+	/*
+	 * We use a temporary list to keep track of the preallocated
+	 * buffers.  Once we're done building the list we splice it
+	 * into the backchannel preallocation list off of the rpc_xprt
+	 * struct.  This helps minimize the amount of time the list
+	 * lock is held on the rpc_xprt struct.  It also makes cleanup
+	 * easier in case of memory allocation errors.
+	 */
+	INIT_LIST_HEAD(&tmp_list);
+	for (i = 0; i < min_reqs; i++) {
+		/* Pre-allocate one backchannel rpc_rqst */
+		req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
+		if (req == NULL) {
+			printk(KERN_ERR "Failed to create bc rpc_rqst\n");
+			goto out_free;
+		}
+
+		/* Add the allocated buffer to the tmp list */
+		dprintk("RPC:       adding req= %p\n", req);
+		list_add(&req->rq_bc_pa_list, &tmp_list);
+
+		req->rq_xprt = xprt;
+		INIT_LIST_HEAD(&req->rq_list);
+		INIT_LIST_HEAD(&req->rq_bc_list);
+
+		/* Preallocate one XDR receive buffer */
+		page_rcv = alloc_page(GFP_KERNEL);
+		if (page_rcv == NULL) {
+			printk(KERN_ERR "Failed to create bc receive xbuf\n");
+			goto out_free;
+		}
+		xbufp = &req->rq_rcv_buf;
+		xbufp->head[0].iov_base = page_address(page_rcv);
+		xbufp->head[0].iov_len = PAGE_SIZE;
+		xbufp->tail[0].iov_base = NULL;
+		xbufp->tail[0].iov_len = 0;
+		xbufp->page_len = 0;
+		xbufp->len = PAGE_SIZE;
+		xbufp->buflen = PAGE_SIZE;
+
+		/* Preallocate one XDR send buffer */
+		page_snd = alloc_page(GFP_KERNEL);
+		if (page_snd == NULL) {
+			printk(KERN_ERR "Failed to create bc snd xbuf\n");
+			goto out_free;
+		}
+
+		xbufp = &req->rq_snd_buf;
+		xbufp->head[0].iov_base = page_address(page_snd);
+		xbufp->head[0].iov_len = 0;
+		xbufp->tail[0].iov_base = NULL;
+		xbufp->tail[0].iov_len = 0;
+		xbufp->page_len = 0;
+		xbufp->len = 0;
+		xbufp->buflen = PAGE_SIZE;
+	}
+
+	/*
+	 * Add the temporary list to the backchannel preallocation list
+	 */
+	spin_lock_bh(&xprt->bc_pa_lock);
+	list_splice(&tmp_list, &xprt->bc_pa_list);
+	xprt_inc_alloc_count(xprt, min_reqs);
+	spin_unlock_bh(&xprt->bc_pa_lock);
+
+	dprintk("RPC:       setup backchannel transport done\n");
+	return 0;
+
+out_free:
+	/*
+	 * Memory allocation failed, free the temporary list
+	 */
+	list_for_each_entry_safe(req, tmp, &tmp_list, rq_bc_pa_list)
+		xprt_free_allocation(req);
+
+	dprintk("RPC:       setup backchannel transport failed\n");
+	return -1;
+}
+EXPORT_SYMBOL(xprt_setup_backchannel);
+
+/*
+ * Destroys the backchannel preallocated structures.
+ * Since these structures may have been allocated by multiple calls
+ * to xprt_setup_backchannel, we only destroy up to the maximum number
+ * of reqs specified by the caller.
+ * @xprt:	the transport holding the preallocated strucures
+ * @max_reqs	the maximum number of preallocated structures to destroy
+ */
+void xprt_destroy_backchannel(struct rpc_xprt *xprt, unsigned int max_reqs)
+{
+	struct rpc_rqst *req = NULL, *tmp = NULL;
+
+	dprintk("RPC:        destroy backchannel transport\n");
+
+	BUG_ON(max_reqs == 0);
+	spin_lock_bh(&xprt->bc_pa_lock);
+	xprt_dec_alloc_count(xprt, max_reqs);
+	list_for_each_entry_safe(req, tmp, &xprt->bc_pa_list, rq_bc_pa_list) {
+		dprintk("RPC:        req=%p\n", req);
+		xprt_free_allocation(req);
+		if (--max_reqs == 0)
+			break;
+	}
+	spin_unlock_bh(&xprt->bc_pa_lock);
+
+	dprintk("RPC:        backchannel list empty= %s\n",
+		list_empty(&xprt->bc_pa_list) ? "true" : "false");
+}
+EXPORT_SYMBOL(xprt_destroy_backchannel);
+
+/*
+ * One or more rpc_rqst structure have been preallocated during the
+ * backchannel setup.  Buffer space for the send and private XDR buffers
+ * has been preallocated as well.  Use xprt_alloc_bc_request to allocate
+ * to this request.  Use xprt_free_bc_request to return it.
+ *
+ * Return an available rpc_rqst, otherwise NULL if non are available.
+ */
+struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt)
+{
+	struct rpc_rqst *req;
+
+	dprintk("RPC:       allocate a backchannel request\n");
+	spin_lock_bh(&xprt->bc_pa_lock);
+	if (!list_empty(&xprt->bc_pa_list)) {
+		req = list_first_entry(&xprt->bc_pa_list, struct rpc_rqst,
+				rq_bc_pa_list);
+		list_del(&req->rq_bc_pa_list);
+	} else {
+		req = NULL;
+	}
+	spin_unlock_bh(&xprt->bc_pa_lock);
+
+	if (req != NULL) {
+		set_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
+		req->rq_received = 0;
+		req->rq_bytes_sent = 0;
+		memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
+			sizeof(req->rq_private_buf));
+	}
+	dprintk("RPC:       backchannel req=%p\n", req);
+	return req;
+}
+
+/*
+ * Return the preallocated rpc_rqst structure and XDR buffers
+ * associated with this rpc_task.
+ */
+void xprt_free_bc_request(struct rpc_rqst *req)
+{
+	struct rpc_xprt *xprt = req->rq_xprt;
+
+	dprintk("RPC:       free backchannel req=%p\n", req);
+
+	smp_mb__before_clear_bit();
+	BUG_ON(!test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state));
+	clear_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
+	smp_mb__after_clear_bit();
+
+	if (!xprt_need_to_requeue(xprt)) {
+		/*
+		 * The last remaining session was destroyed while this
+		 * entry was in use.  Free the entry and don't attempt
+		 * to add back to the list because there is no need to
+		 * have anymore preallocated entries.
+		 */
+		dprintk("RPC:       Last session removed req=%p\n", req);
+		xprt_free_allocation(req);
+		return;
+	}
+
+	/*
+	 * Return it to the list of preallocations so that it
+	 * may be reused by a new callback request.
+	 */
+	spin_lock_bh(&xprt->bc_pa_lock);
+	list_add(&req->rq_bc_pa_list, &xprt->bc_pa_list);
+	spin_unlock_bh(&xprt->bc_pa_lock);
+}
+EXPORT_SYMBOL(xprt_free_bc_request);
+
+#endif /* CONFIG_NFS_V4_1 */
-- 
1.6.2.1

[RFC 08/39] nfs41: New include/linux/sunrpc/bc_xprt.h

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Contains prototype for backchannel helper routines.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: xprt_setup_backchannel v4.0 only inline]
    Fix compile error when CONFIG_NFS_V4_1 is not set.
Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/bc_xprt.h |   44 ++++++++++++++++++++++++++++++++++++++++
 1 files changed, 44 insertions(+), 0 deletions(-)
 create mode 100644 include/linux/sunrpc/bc_xprt.h

diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h
new file mode 100644
index 0000000..1c1746a
--- /dev/null
+++ b/include/linux/sunrpc/bc_xprt.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+
+(c) 2008 Network Appliance, Inc.  All Rights Reserved.
+
+Network Appliance provides this source code under the GPL v2 License.
+The GPL v2 license is available at
+http://opensource.org/licenses/gpl-license.php.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+
+/*
+ * Functions to create and manage the backchannel
+ */
+
+#ifndef _LINUX_SUNRPC_BC_XPRT_H
+#define _LINUX_SUNRPC_BC_XPRT_H
+
+#include <linux/sunrpc/svcsock.h>
+#include <linux/sunrpc/xprt.h>
+
+#ifdef CONFIG_NFS_V4_1
+struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt);
+void xprt_free_bc_request(struct rpc_rqst *req);
+int xprt_setup_backchannel(struct rpc_xprt *, unsigned int min_reqs);
+void xprt_destroy_backchannel(struct rpc_xprt *, int max_reqs);
+#else /* CONFIG_NFS_V4_1 */
+static inline int xprt_setup_backchannel(struct rpc_xprt *xprt,
+					 unsigned int min_reqs)
+{ return 0; }
+#endif /* CONFIG_NFS_V4_1 */
+#endif /* _LINUX_SUNRPC_BC_XPRT_H */
+
-- 
1.6.2.1

[RFC 09/39] nfs41: New xs_tcp_read_data()

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Handles RPC replies and backchannel callbacks.  Traditionally the NFS
client has expected only RPC replies on its open connections.  With
NFSv4.1, callbacks can arrive over an existing open connection.

This patch refactors the old xs_tcp_read_request() into an RPC reply handler:
xs_tcp_read_reply(), a new backchannel callback handler: xs_tcp_read_callback(),
and a common routine to read the data off the transport: xs_tcp_read_common().
The new xs_tcp_read_callback() queues callback requests onto a queue where
the callback service (a separate thread) is listening for the processing.

This patch incorporates work and suggestions from Rahul Iyer (iyer@netapp.com)
and Benny Halevy (bhalevy@panasas.com).

[nfs41: Keep track of RPC call/reply direction with a flag]
[nfs41: Preallocate rpc_rqst receive buffer for handling callbacks]
Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/xprtsock.c |  141 ++++++++++++++++++++++++++++++++++++++++++------
 1 files changed, 123 insertions(+), 18 deletions(-)

diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 692b517..403ebda 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -34,6 +34,9 @@
 #include <linux/sunrpc/sched.h>
 #include <linux/sunrpc/xprtsock.h>
 #include <linux/file.h>
+#ifdef CONFIG_NFS_V4_1
+#include <linux/sunrpc/bc_xprt.h>
+#endif
 
 #include <net/sock.h>
 #include <net/checksum.h>
@@ -1028,25 +1031,16 @@ static inline void xs_tcp_read_calldir(struct sock_xprt *transport,
 	xs_tcp_check_fraghdr(transport);
 }
 
-static inline void xs_tcp_read_request(struct rpc_xprt *xprt, struct xdr_skb_reader *desc)
+static inline void xs_tcp_read_common(struct rpc_xprt *xprt,
+				     struct xdr_skb_reader *desc,
+				     struct rpc_rqst *req)
 {
-	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
-	struct rpc_rqst *req;
+	struct sock_xprt *transport =
+				container_of(xprt, struct sock_xprt, xprt);
 	struct xdr_buf *rcvbuf;
 	size_t len;
 	ssize_t r;
 
-	/* Find and lock the request corresponding to this xid */
-	spin_lock(&xprt->transport_lock);
-	req = xprt_lookup_rqst(xprt, transport->tcp_xid);
-	if (!req) {
-		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
-		dprintk("RPC:       XID %08x request not found!\n",
-				ntohl(transport->tcp_xid));
-		spin_unlock(&xprt->transport_lock);
-		return;
-	}
-
 	rcvbuf = &req->rq_private_buf;
 	len = desc->count;
 	if (len > transport->tcp_reclen - transport->tcp_offset) {
@@ -1084,7 +1078,7 @@ static inline void xs_tcp_read_request(struct rpc_xprt *xprt, struct xdr_skb_rea
 				"tcp_offset = %u, tcp_reclen = %u\n",
 				xprt, transport->tcp_copied,
 				transport->tcp_offset, transport->tcp_reclen);
-		goto out;
+		return;
 	}
 
 	dprintk("RPC:       XID %08x read %Zd bytes\n",
@@ -1100,11 +1094,122 @@ static inline void xs_tcp_read_request(struct rpc_xprt *xprt, struct xdr_skb_rea
 			transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
 	}
 
-out:
+	return;
+}
+
+/*
+ * Finds the request corresponding to the RPC xid and invokes the common
+ * tcp read code to read the data.
+ */
+static inline int xs_tcp_read_reply(struct rpc_xprt *xprt,
+				    struct xdr_skb_reader *desc)
+{
+	struct sock_xprt *transport =
+				container_of(xprt, struct sock_xprt, xprt);
+	struct rpc_rqst *req;
+
+	dprintk("RPC:       read reply XID %08x\n", ntohl(transport->tcp_xid));
+
+	/* Find and lock the request corresponding to this xid */
+	spin_lock(&xprt->transport_lock);
+	req = xprt_lookup_rqst(xprt, transport->tcp_xid);
+	if (!req) {
+		dprintk("RPC:       XID %08x request not found!\n",
+				ntohl(transport->tcp_xid));
+		spin_unlock(&xprt->transport_lock);
+		return -1;
+	}
+
+	xs_tcp_read_common(xprt, desc, req);
+
 	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA))
 		xprt_complete_rqst(req->rq_task, transport->tcp_copied);
+
 	spin_unlock(&xprt->transport_lock);
-	xs_tcp_check_fraghdr(transport);
+	return 0;
+}
+
+#if defined(CONFIG_NFS_V4_1)
+/*
+ * Obtains an rpc_rqst previously allocated and invokes the common
+ * tcp read code to read the data.  The result is placed in the callback
+ * queue.
+ * If we're unable to obtain the rpc_rqst we schedule the closing of the
+ * connection and return -1.
+ */
+static inline int xs_tcp_read_callback(struct rpc_xprt *xprt,
+				       struct xdr_skb_reader *desc)
+{
+	struct sock_xprt *transport =
+				container_of(xprt, struct sock_xprt, xprt);
+	struct rpc_rqst *req;
+
+	req = xprt_alloc_bc_request(xprt);
+	if (req == NULL) {
+		/*
+		 * Schedule an autoclose RPC call
+		 */
+		printk(KERN_WARNING "Callback slot table overflowed\n");
+		set_bit(XPRT_CLOSE_WAIT, &xprt->state);
+		if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
+			queue_work(rpciod_workqueue, &xprt->task_cleanup);
+		return -1;
+	}
+
+	req->rq_xid = transport->tcp_xid;
+	dprintk("RPC:       read callback  XID %08x\n", ntohl(req->rq_xid));
+	xs_tcp_read_common(xprt, desc, req);
+
+	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA)) {
+		struct svc_serv *bc_serv = xprt->bc_serv;
+
+		/*
+		 * Add callback request to callback list.  The callback
+		 * service sleeps on the sv_cb_waitq waiting for new
+		 * requests.  Wake it up after adding enqueing the
+		 * request.
+		 */
+		dprintk("RPC:       add callback request to list\n");
+		spin_lock(&bc_serv->sv_cb_lock);
+		list_add(&req->rq_bc_list, &bc_serv->sv_cb_list);
+		spin_unlock(&bc_serv->sv_cb_lock);
+		wake_up(&bc_serv->sv_cb_waitq);
+	}
+
+	req->rq_private_buf.len = transport->tcp_copied;
+
+	return 0;
+}
+#endif /* CONFIG_NFS_V4_1 */
+
+/*
+ * Read data off the transport.  This can be either an RPC_CALL or an
+ * RPC_REPLY.  Relay the processing to helper functions.
+ */
+static void xs_tcp_read_data(struct rpc_xprt *xprt,
+				    struct xdr_skb_reader *desc)
+{
+	struct sock_xprt *transport =
+				container_of(xprt, struct sock_xprt, xprt);
+	int status;
+
+#if defined(CONFIG_NFS_V4_1)
+	status = (transport->tcp_flags & TCP_RPC_REPLY) ?
+		xs_tcp_read_reply(xprt, desc) :
+		xs_tcp_read_callback(xprt, desc);
+#else
+	status = xs_tcp_read_reply(xprt, desc);
+#endif /* CONFIG_NFS_V4_1 */
+
+	if (status == 0)
+		xs_tcp_check_fraghdr(transport);
+	else {
+		/*
+		 * The transport_lock protects the request handling.
+		 * There's no need to hold it to update the tcp_flags.
+		 */
+		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;
+	}
 }
 
 static inline void xs_tcp_read_discard(struct sock_xprt *transport, struct xdr_skb_reader *desc)
@@ -1151,7 +1256,7 @@ static int xs_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, uns
 		}
 		/* Read in the request data */
 		if (transport->tcp_flags & TCP_RCV_COPY_DATA) {
-			xs_tcp_read_request(xprt, &desc);
+			xs_tcp_read_data(xprt, &desc);
 			continue;
 		}
 		/* Skip over any trailing bytes on short reads */
-- 
1.6.2.1

[RFC 10/39] nfs41: Add backchannel processing support to RPC state machine

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Adds rpc_run_bc_task() which is called by the NFS callback service to
process backchannel requests.  It performs similar work to rpc_run_task()
though "schedules" the backchannel task to be executed starting at the
call_trasmit state in the RPC state machine.

It also introduces some miscellaneous updates to the argument validation,
call_transmit, and transport cleanup functions to take into account
that there are now forechannel and backchannel tasks.

Backchannel requests do not carry an RPC message structure, since the
payload has already been XDR encoded using the existing NFSv4 callback
mechanism.

Introduce a new transmit state for the client to reply on to backchannel
requests.  This new state simply reserves the transport and issues the
reply.  In case of a connection related error, disconnects the transport and
drops the reply.  It requires the forechannel to re-establish the connection
and the server to retransmit the request, as stated in NFSv4.1 section
2.9.2 "Client and Server Transport Behavior".

Note: There is no need to loop attempting to reserve the transport.  If EAGAIN
is returned by xprt_prepare_transmit(), return with tk_status == 0,
setting tk_action to call_bc_transmit.  rpc_execute() will invoke it again
after the task is taken off the sleep queue.

[nfs41: rpc_run_bc_task() need not be exported outside RPC module]
[nfs41: New call_bc_transmit RPC state]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: Backchannel: No need to loop in call_bc_transmit()]
Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/sched.h |    2 +
 include/linux/sunrpc/xprt.h  |   12 ++++
 net/sunrpc/clnt.c            |  117 +++++++++++++++++++++++++++++++++++++++++-
 net/sunrpc/stats.c           |    6 ++-
 net/sunrpc/sunrpc.h          |   35 +++++++++++++
 net/sunrpc/xprt.c            |   36 +++++++++++--
 6 files changed, 199 insertions(+), 9 deletions(-)
 create mode 100644 net/sunrpc/sunrpc.h

diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
index 1773768..4010977 100644
--- a/include/linux/sunrpc/sched.h
+++ b/include/linux/sunrpc/sched.h
@@ -210,6 +210,8 @@ struct rpc_wait_queue {
  */
 struct rpc_task *rpc_new_task(const struct rpc_task_setup *);
 struct rpc_task *rpc_run_task(const struct rpc_task_setup *);
+struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
+				const struct rpc_call_ops *ops);
 void		rpc_put_task(struct rpc_task *);
 void		rpc_exit_task(struct rpc_task *);
 void		rpc_release_calldata(const struct rpc_call_ops *, void *);
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 6b37724..1531abe 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -215,6 +215,18 @@ struct rpc_xprt {
 						/* buffer in use */
 #endif /* CONFIG_NFS_V4_1 */
 
+#if defined(CONFIG_NFS_V4_1)
+static inline int bc_prealloc(struct rpc_rqst *req)
+{
+	return test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
+}
+#else
+static inline int bc_prealloc(struct rpc_rqst *req)
+{
+	return 0;
+}
+#endif /* CONFIG_NFS_V4_1 */
+
 struct xprt_create {
 	int			ident;		/* XPRT_TRANSPORT identifier */
 	struct sockaddr *	srcaddr;	/* optional local address */
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 76d7d46..349b4d6 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -36,7 +36,9 @@
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/rpc_pipe_fs.h>
 #include <linux/sunrpc/metrics.h>
+#include <linux/sunrpc/bc_xprt.h>
 
+#include "sunrpc.h"
 
 #ifdef RPC_DEBUG
 # define RPCDBG_FACILITY	RPCDBG_CALL
@@ -63,6 +65,9 @@ static void	call_decode(struct rpc_task *task);
 static void	call_bind(struct rpc_task *task);
 static void	call_bind_status(struct rpc_task *task);
 static void	call_transmit(struct rpc_task *task);
+#if defined(CONFIG_NFS_V4_1)
+static void	call_bc_transmit(struct rpc_task *task);
+#endif /* CONFIG_NFS_V4_1 */
 static void	call_status(struct rpc_task *task);
 static void	call_transmit_status(struct rpc_task *task);
 static void	call_refresh(struct rpc_task *task);
@@ -613,6 +618,50 @@ rpc_call_async(struct rpc_clnt *clnt, const struct rpc_message *msg, int flags,
 }
 EXPORT_SYMBOL_GPL(rpc_call_async);
 
+#if defined(CONFIG_NFS_V4_1)
+/**
+ * rpc_run_bc_task - Allocate a new RPC task for backchannel use, then run
+ * rpc_execute against it
+ * @ops: RPC call ops
+ */
+struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
+					const struct rpc_call_ops *tk_ops)
+{
+	struct rpc_task *task;
+	struct xdr_buf *xbufp = &req->rq_snd_buf;
+	struct rpc_task_setup task_setup_data = {
+		.callback_ops = tk_ops,
+	};
+
+	dprintk("RPC: rpc_run_bc_task req= %p\n", req);
+	/*
+	 * Create an rpc_task to send the data
+	 */
+	task = rpc_new_task(&task_setup_data);
+	if (!task) {
+		xprt_free_bc_request(req);
+		goto out;
+	}
+	task->tk_rqstp = req;
+
+	/*
+	 * Set up the xdr_buf length.
+	 * This also indicates that the buffer is XDR encoded already.
+	 */
+	xbufp->len = xbufp->head[0].iov_len + xbufp->page_len +
+			xbufp->tail[0].iov_len;
+
+	task->tk_action = call_bc_transmit;
+	atomic_inc(&task->tk_count);
+	BUG_ON(atomic_read(&task->tk_count) != 2);
+	rpc_execute(task);
+
+out:
+	dprintk("RPC: rpc_run_bc_task: task= %p\n", task);
+	return task;
+}
+#endif /* CONFIG_NFS_V4_1 */
+
 void
 rpc_call_start(struct rpc_task *task)
 {
@@ -1098,7 +1147,7 @@ call_transmit(struct rpc_task *task)
 	 * in order to allow access to the socket to other RPC requests.
 	 */
 	call_transmit_status(task);
-	if (task->tk_msg.rpc_proc->p_decode != NULL)
+	if (rpc_reply_expected(task))
 		return;
 	task->tk_action = rpc_exit_task;
 	rpc_wake_up_queued_task(&task->tk_xprt->pending, task);
@@ -1133,6 +1182,72 @@ call_transmit_status(struct rpc_task *task)
 	}
 }
 
+#if defined(CONFIG_NFS_V4_1)
+/*
+ * 5b.	Send the backchannel RPC reply.  On error, drop the reply.  In
+ * addition, disconnect on connectivity errors.
+ */
+static void
+call_bc_transmit(struct rpc_task *task)
+{
+	struct rpc_rqst *req = task->tk_rqstp;
+
+	BUG_ON(task->tk_status != 0);
+	task->tk_status = xprt_prepare_transmit(task);
+	if (task->tk_status == -EAGAIN) {
+		/*
+		 * Could not reserve the transport. Try again after the
+		 * transport is released.
+		 */
+		task->tk_status = 0;
+		task->tk_action = call_bc_transmit;
+		return;
+	}
+
+	task->tk_action = rpc_exit_task;
+	if (task->tk_status < 0) {
+		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
+			"error: %d\n", task->tk_status);
+		return;
+	}
+
+	xprt_transmit(task);
+	xprt_end_transmit(task);
+	dprint_status(task);
+	switch (task->tk_status) {
+	case 0:
+		/* Success */
+		break;
+	case -EHOSTDOWN:
+	case -EHOSTUNREACH:
+	case -ENETUNREACH:
+	case -ETIMEDOUT:
+		/*
+		 * Problem reaching the server.  Disconnect and let the
+		 * forechannel reestablish the connection.  The server will
+		 * have to retransmit the backchannel request and we'll
+		 * reprocess it.  Since these ops are idempotent, there's no
+		 * need to cache our reply at this time.
+		 */
+		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
+			"error: %d\n", task->tk_status);
+		xprt_conditional_disconnect(task->tk_xprt,
+			req->rq_connect_cookie);
+		break;
+	default:
+		/*
+		 * We were unable to reply and will have to drop the
+		 * request.  The server should reconnect and retransmit.
+		 */
+		BUG_ON(task->tk_status == -EAGAIN);
+		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
+			"error: %d\n", task->tk_status);
+		break;
+	}
+	rpc_wake_up_queued_task(&req->rq_xprt->pending, task);
+}
+#endif /* CONFIG_NFS_V4_1 */
+
 /*
  * 6.	Sort out the RPC call status
  */
diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 1ef6e46..a0e3d97 100644
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -141,12 +141,14 @@ EXPORT_SYMBOL_GPL(rpc_free_iostats);
 void rpc_count_iostats(struct rpc_task *task)
 {
 	struct rpc_rqst *req = task->tk_rqstp;
-	struct rpc_iostats *stats = task->tk_client->cl_metrics;
+	struct rpc_iostats *stats;
 	struct rpc_iostats *op_metrics;
 	long rtt, execute, queue;
 
-	if (!stats || !req)
+	if (!task->tk_client || task->tk_client->cl_metrics || !req)
 		return;
+
+	stats = task->tk_client->cl_metrics;
 	op_metrics = &stats[task->tk_msg.rpc_proc->p_statidx];
 
 	op_metrics->om_ops++;
diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h
new file mode 100644
index 0000000..b462de4
--- /dev/null
+++ b/net/sunrpc/sunrpc.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+
+(c) 2008 Network Appliance, Inc.  All Rights Reserved.
+
+Network Appliance provides this source code under the GPL v2 License.
+The GPL v2 license is available at
+http://opensource.org/licenses/gpl-license.php.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+
+/*
+ * Functions and macros used internally by RPC
+ */
+
+#ifndef _NET_SUNRPC_SUNRPC_H
+#define _NET_SUNRPC_SUNRPC_H
+
+#define rpc_reply_expected(task) \
+	(((task)->tk_msg.rpc_proc != NULL) && \
+	((task)->tk_msg.rpc_proc->p_decode != NULL))
+
+#endif /* _NET_SUNRPC_SUNRPC_H */
+
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index bbaec23..df65d15 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -12,8 +12,9 @@
  *  -	Next, the caller puts together the RPC message, stuffs it into
  *	the request struct, and calls xprt_transmit().
  *  -	xprt_transmit sends the message and installs the caller on the
- *	transport's wait list. At the same time, it installs a timer that
- *	is run after the packet's timeout has expired.
+ *	transport's wait list. At the same time, if a reply is expected,
+ *	it installs a timer that is run after the packet's timeout has
+ *	expired.
  *  -	When a packet arrives, the data_ready handler walks the list of
  *	pending requests for that transport. If a matching XID is found, the
  *	caller is woken up, and the timer removed.
@@ -46,6 +47,8 @@
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/metrics.h>
 
+#include "sunrpc.h"
+
 /*
  * Local variables
  */
@@ -875,7 +878,10 @@ void xprt_transmit(struct rpc_task *task)
 	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
 
 	if (!req->rq_received) {
-		if (list_empty(&req->rq_list)) {
+		if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
+			/*
+			 * Add to the list only if we're expecting a reply
+			 */
 			spin_lock_bh(&xprt->transport_lock);
 			/* Update the softirq receive buffer */
 			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
@@ -910,8 +916,13 @@ void xprt_transmit(struct rpc_task *task)
 	/* Don't race with disconnect */
 	if (!xprt_connected(xprt))
 		task->tk_status = -ENOTCONN;
-	else if (!req->rq_received)
+	else if (!req->rq_received && rpc_reply_expected(task)) {
+		/*
+		 * Sleep on the pending queue since
+		 * we're expecting a reply.
+		 */
 		rpc_sleep_on(&xprt->pending, task, xprt_timer);
+	}
 	spin_unlock_bh(&xprt->transport_lock);
 }
 
@@ -984,11 +995,15 @@ static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
  */
 void xprt_release(struct rpc_task *task)
 {
-	struct rpc_xprt	*xprt = task->tk_xprt;
+	struct rpc_xprt	*xprt;
 	struct rpc_rqst	*req;
+	int prealloc;
 
+	BUG_ON(atomic_read(&task->tk_count) < 0);
 	if (!(req = task->tk_rqstp))
 		return;
+	prealloc = bc_prealloc(req);	/* Preallocated backchannel request? */
+	xprt = req->rq_xprt;
 	rpc_count_iostats(task);
 	spin_lock_bh(&xprt->transport_lock);
 	xprt->ops->release_xprt(xprt, task);
@@ -1001,10 +1016,19 @@ void xprt_release(struct rpc_task *task)
 		mod_timer(&xprt->timer,
 				xprt->last_used + xprt->idle_timeout);
 	spin_unlock_bh(&xprt->transport_lock);
-	xprt->ops->buf_free(req->rq_buffer);
+	if (!bc_prealloc(req))
+		xprt->ops->buf_free(req->rq_buffer);
 	task->tk_rqstp = NULL;
 	if (req->rq_release_snd_buf)
 		req->rq_release_snd_buf(req);
+
+	/*
+	 * Early exit if this is a backchannel preallocated request.
+	 * There is no need to have it added to the RPC slot list.
+	 */
+	if (prealloc)
+		return;
+
 	memset(req, 0, sizeof(*req));	/* mark unused */
 
 	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
-- 
1.6.2.1

[RFC 11/39] nfs41: Backchannel callback service helper routines

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Executes the backchannel task on the RPC state machine using
the existing open connection previously established by the client.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>

nfs41: Add bc_svc.o to sunrpc Makefile.

[nfs41: bc_send() does not need to be exported outside RPC module]
[nfs41: xprt_free_bc_request() need not be exported outside RPC module]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/bc_xprt.h |    3 +
 net/sunrpc/Makefile            |    2 +-
 net/sunrpc/backchannel_rqst.c  |    1 -
 net/sunrpc/bc_svc.c            |   80 ++++++++++++++++++++++++++++++++++++++++
 net/sunrpc/xprtsock.c          |    3 +
 5 files changed, 87 insertions(+), 2 deletions(-)
 create mode 100644 net/sunrpc/bc_svc.c

diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h
index 1c1746a..3016c00 100644
--- a/include/linux/sunrpc/bc_xprt.h
+++ b/include/linux/sunrpc/bc_xprt.h
@@ -29,12 +29,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <linux/sunrpc/svcsock.h>
 #include <linux/sunrpc/xprt.h>
+#include <linux/sunrpc/sched.h>
 
 #ifdef CONFIG_NFS_V4_1
 struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt);
 void xprt_free_bc_request(struct rpc_rqst *req);
 int xprt_setup_backchannel(struct rpc_xprt *, unsigned int min_reqs);
 void xprt_destroy_backchannel(struct rpc_xprt *, int max_reqs);
+void bc_release_request(struct rpc_task *);
+int bc_send(struct rpc_rqst *req);
 #else /* CONFIG_NFS_V4_1 */
 static inline int xprt_setup_backchannel(struct rpc_xprt *xprt,
 					 unsigned int min_reqs)
diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
index 4a01f96..db73fd2 100644
--- a/net/sunrpc/Makefile
+++ b/net/sunrpc/Makefile
@@ -13,6 +13,6 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
 	    rpcb_clnt.o timer.o xdr.o \
 	    sunrpc_syms.o cache.o rpc_pipe.o \
 	    svc_xprt.o
-sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o
+sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o bc_svc.o
 sunrpc-$(CONFIG_PROC_FS) += stats.o
 sunrpc-$(CONFIG_SYSCTL) += sysctl.o
diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c
index 7d7708a..92fb3bd 100644
--- a/net/sunrpc/backchannel_rqst.c
+++ b/net/sunrpc/backchannel_rqst.c
@@ -265,6 +265,5 @@ void xprt_free_bc_request(struct rpc_rqst *req)
 	list_add(&req->rq_bc_pa_list, &xprt->bc_pa_list);
 	spin_unlock_bh(&xprt->bc_pa_lock);
 }
-EXPORT_SYMBOL(xprt_free_bc_request);
 
 #endif /* CONFIG_NFS_V4_1 */
diff --git a/net/sunrpc/bc_svc.c b/net/sunrpc/bc_svc.c
new file mode 100644
index 0000000..b13f51d
--- /dev/null
+++ b/net/sunrpc/bc_svc.c
@@ -0,0 +1,80 @@
+/******************************************************************************
+
+(c) 2007 Network Appliance, Inc.  All Rights Reserved.
+
+Network Appliance provides this source code under the GPL v2 License.
+The GPL v2 license is available at
+http://opensource.org/licenses/gpl-license.php.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+
+/*
+ * The NFSv4.1 callback service helper routines.
+ * They implement the transport level processing required to send the
+ * reply over an existing open connection previously established by the client.
+ */
+
+#if defined(CONFIG_NFS_V4_1)
+
+#include <linux/module.h>
+
+#include <linux/sunrpc/xprt.h>
+#include <linux/sunrpc/sched.h>
+#include <linux/sunrpc/bc_xprt.h>
+
+#define RPCDBG_FACILITY	RPCDBG_SVCDSP
+
+void bc_release_request(struct rpc_task *task)
+{
+	struct rpc_rqst *req = task->tk_rqstp;
+
+	dprintk("RPC:       bc_release_request: task= %p\n", task);
+
+	/*
+	 * Release this request only if it's a backchannel
+	 * preallocated request
+	 */
+	if (!bc_prealloc(req))
+		return;
+	xprt_free_bc_request(req);
+}
+
+/* Empty callback ops */
+static const struct rpc_call_ops nfs41_callback_ops = {
+};
+
+
+/*
+ * Send the callback reply
+ */
+int bc_send(struct rpc_rqst *req)
+{
+	struct rpc_task *task;
+	int ret;
+
+	dprintk("RPC:       bc_send req= %p\n", req);
+	task = rpc_run_bc_task(req, &nfs41_callback_ops);
+	if (IS_ERR(task))
+		ret = PTR_ERR(task);
+	else {
+		BUG_ON(atomic_read(&task->tk_count) != 1);
+		ret = task->tk_status;
+		rpc_put_task(task);
+	}
+	return ret;
+	dprintk("RPC:       bc_send ret= %d \n", ret);
+}
+
+#endif /* CONFIG_NFS_V4_1 */
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 403ebda..5bf9e66 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -2145,6 +2145,9 @@ static struct rpc_xprt_ops xs_tcp_ops = {
 	.buf_free		= rpc_free,
 	.send_request		= xs_tcp_send_request,
 	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
+#if defined(CONFIG_NFS_V4_1)
+	.release_request	= bc_release_request,
+#endif /* CONFIG_NFS_V4_1 */
 	.close			= xs_tcp_shutdown,
 	.destroy		= xs_destroy,
 	.print_stats		= xs_tcp_print_stats,
-- 
1.6.2.1

[RFC 12/39] nfs41: Refactor svc_process()

[Benny Halevy]

From: Ricardo Labiaga <ricardo.labiaga@netapp.com>

net/sunrpc/svc.c:svc_process() is used by the NFSv4 callback service
to process RPC requests arriving over connections initiated by the
server.  NFSv4.1 supports callbacks over the backchannel on connections
initiated by the client.  This patch refactors svc_process() so that
common code can also be used by the backchannel.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/svc.c |   80 +++++++++++++++++++++++++++++++++--------------------
 1 files changed, 50 insertions(+), 30 deletions(-)

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 8847add..bfda66d 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -970,20 +970,18 @@ svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
 }
 
 /*
- * Process the RPC request.
+ * Common routine for processing the RPC request.
  */
-int
-svc_process(struct svc_rqst *rqstp)
+static int
+svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
 {
 	struct svc_program	*progp;
 	struct svc_version	*versp = NULL;	/* compiler food */
 	struct svc_procedure	*procp = NULL;
-	struct kvec *		argv = &rqstp->rq_arg.head[0];
-	struct kvec *		resv = &rqstp->rq_res.head[0];
 	struct svc_serv		*serv = rqstp->rq_server;
 	kxdrproc_t		xdr;
 	__be32			*statp;
-	u32			dir, prog, vers, proc;
+	u32			prog, vers, proc;
 	__be32			auth_stat, rpc_stat;
 	int			auth_res;
 	__be32			*reply_statp;
@@ -993,19 +991,6 @@ svc_process(struct svc_rqst *rqstp)
 	if (argv->iov_len < 6*4)
 		goto err_short_len;
 
-	/* setup response xdr_buf.
-	 * Initially it has just one page
-	 */
-	rqstp->rq_resused = 1;
-	resv->iov_base = page_address(rqstp->rq_respages[0]);
-	resv->iov_len = 0;
-	rqstp->rq_res.pages = rqstp->rq_respages + 1;
-	rqstp->rq_res.len = 0;
-	rqstp->rq_res.page_base = 0;
-	rqstp->rq_res.page_len = 0;
-	rqstp->rq_res.buflen = PAGE_SIZE;
-	rqstp->rq_res.tail[0].iov_base = NULL;
-	rqstp->rq_res.tail[0].iov_len = 0;
 	/* Will be turned off only in gss privacy case: */
 	rqstp->rq_splice_ok = 1;
 	/* Will be turned off only when NFSv4 Sessions are used */
@@ -1014,17 +999,13 @@ svc_process(struct svc_rqst *rqstp)
 	/* Setup reply header */
 	rqstp->rq_xprt->xpt_ops->xpo_prep_reply_hdr(rqstp);
 
-	rqstp->rq_xid = svc_getu32(argv);
 	svc_putu32(resv, rqstp->rq_xid);
 
-	dir  = svc_getnl(argv);
 	vers = svc_getnl(argv);
 
 	/* First words of reply: */
 	svc_putnl(resv, 1);		/* REPLY */
 
-	if (dir != 0)		/* direction != CALL */
-		goto err_bad_dir;
 	if (vers != 2)		/* RPC version number */
 		goto err_bad_rpc;
 
@@ -1147,7 +1128,7 @@ svc_process(struct svc_rqst *rqstp)
  sendit:
 	if (svc_authorise(rqstp))
 		goto dropit;
-	return svc_send(rqstp);
+	return 1;		/* Caller can now send it */
 
  dropit:
 	svc_authorise(rqstp);	/* doesn't hurt to call this twice */
@@ -1161,12 +1142,6 @@ err_short_len:
 
 	goto dropit;			/* drop request */
 
-err_bad_dir:
-	svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
-
-	serv->sv_stats->rpcbadfmt++;
-	goto dropit;			/* drop request */
-
 err_bad_rpc:
 	serv->sv_stats->rpcbadfmt++;
 	svc_putnl(resv, 1);	/* REJECT */
@@ -1220,6 +1195,51 @@ err_bad:
 EXPORT_SYMBOL_GPL(svc_process);
 
 /*
+ * Process the RPC request.
+ */
+int
+svc_process(struct svc_rqst *rqstp)
+{
+	struct kvec		*argv = &rqstp->rq_arg.head[0];
+	struct kvec		*resv = &rqstp->rq_res.head[0];
+	struct svc_serv		*serv = rqstp->rq_server;
+	u32			dir;
+	int			error;
+
+	/*
+	 * Setup response xdr_buf.
+	 * Initially it has just one page
+	 */
+	rqstp->rq_resused = 1;
+	resv->iov_base = page_address(rqstp->rq_respages[0]);
+	resv->iov_len = 0;
+	rqstp->rq_res.pages = rqstp->rq_respages + 1;
+	rqstp->rq_res.len = 0;
+	rqstp->rq_res.page_base = 0;
+	rqstp->rq_res.page_len = 0;
+	rqstp->rq_res.buflen = PAGE_SIZE;
+	rqstp->rq_res.tail[0].iov_base = NULL;
+	rqstp->rq_res.tail[0].iov_len = 0;
+
+	rqstp->rq_xid = svc_getu32(argv);
+
+	dir  = svc_getnl(argv);
+	if (dir != 0) {
+		/* direction != CALL */
+		svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
+		serv->sv_stats->rpcbadfmt++;
+		svc_drop(rqstp);
+		return 0;
+	}
+
+	error = svc_process_common(rqstp, argv, resv);
+	if (error <= 0)
+		return error;
+
+	return svc_send(rqstp);
+}
+
+/*
  * Return (transport-specific) limit on the rpc payload.
  */
 u32 svc_max_payload(const struct svc_rqst *rqstp)
-- 
1.6.2.1

[RFC 13/39] nfs41: Backchannel bc_svc_process()

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Implement the NFSv4.1 backchannel service.  Invokes the common callback
processing logic svc_process_common() to authenticate the call and
dispatch the appropriate NFSv4.1 XDR decoder and operation procedure.
It then invokes bc_send() to send the reply over the same connection.
bc_send() is implemented in a separate patch.

At this time there is no slot validation or reply cache handling.

[nfs41: Preallocate rpc_rqst receive buffer for handling callbacks]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/svc.c |   49 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 49 insertions(+), 0 deletions(-)

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index bfda66d..06b52e4 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -25,6 +25,7 @@
 #include <linux/sunrpc/stats.h>
 #include <linux/sunrpc/svcsock.h>
 #include <linux/sunrpc/clnt.h>
+#include <linux/sunrpc/bc_xprt.h>
 
 #define RPCDBG_FACILITY	RPCDBG_SVCDSP
 
@@ -1239,6 +1240,54 @@ svc_process(struct svc_rqst *rqstp)
 	return svc_send(rqstp);
 }
 
+#if defined(CONFIG_NFS_V4_1)
+/*
+ * Process a backchannel RPC request that arrived over an existing
+ * outbound connection
+ */
+int
+bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
+	       struct svc_rqst *rqstp)
+{
+	struct kvec	*argv = &rqstp->rq_arg.head[0];
+	struct kvec	*resv = &rqstp->rq_res.head[0];
+	int 		error;
+
+	/* Build the svc_rqst used by the common processing routine */
+	rqstp->rq_xid = req->rq_xid;
+	rqstp->rq_prot = req->rq_xprt->prot;
+	rqstp->rq_server = serv;
+
+	rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
+	memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
+	memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
+	memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
+
+	/* reset result send buffer "put" position */
+	resv->iov_len = 0;
+
+	if (rqstp->rq_prot != IPPROTO_TCP) {
+		printk(KERN_ERR "No support for Non-TCP transports!\n");
+		BUG();
+	}
+
+	/*
+	 * Skip the next two words because they've already been
+	 * processed in the trasport
+	 */
+	svc_getu32(argv);	/* XID */
+	svc_getnl(argv);	/* CALLDIR */
+
+	error = svc_process_common(rqstp, argv, resv);
+	if (error <= 0)
+		return error;
+
+	memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
+	return bc_send(req);
+}
+EXPORT_SYMBOL(bc_svc_process);
+#endif /* CONFIG_NFS_V4_1 */
+
 /*
  * Return (transport-specific) limit on the rpc payload.
  */
-- 
1.6.2.1

[RFC 14/39] nfs41: Implement NFSv4.1 callback service process.

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

nfs41_callback_up() initializes the necessary queues and creates the new
nfs41_callback_svc thread.  This thread executes the callback service which
waits for requests to arrive on the svc_serv->sv_cb_list.

NFS41_BC_MIN_CALLBACKS is set to 1 because we expect callbacks to not
cause substantial latency.

The actual processing of the callback will be implemented as a separate patch.

There is only one NFSv4.1 callback service.  The first caller of
nfs4_callback_up() creates the service, subsequent callers increment a
reference count on the service.  The service is destroyed when the last
caller invokes nfs_callback_down().

The transport needs to hold a reference to the callback service in order
to invoke it during callback processing.  Currently this reference is only
obtained when the service is first created.  This is incorrect, since
subsequent registrations for other transports will leave the xprt->serv
pointer uninitialized, leading to an oops when a callback arrives on
the "unreferenced" transport.

This patch fixes the problem by ensuring that a reference to the service
is saved in xprt->serv, either because the service is created by this
invocation to nfs4_callback_up() or by a prior invocation.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: Add a reference to svc_serv during callback service bring up]
Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.c          |   83 ++++++++++++++++++++++++++++++++++++++++++-
 fs/nfs/callback.h          |    7 ++++
 include/linux/sunrpc/svc.h |    2 +
 3 files changed, 90 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index 3926046..9ee174c 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -17,6 +17,9 @@
 #include <linux/freezer.h>
 #include <linux/kthread.h>
 #include <linux/sunrpc/svcauth_gss.h>
+#if defined(CONFIG_NFS_V4_1)
+#include <linux/sunrpc/bc_xprt.h>
+#endif
 
 #include <net/inet_sock.h>
 
@@ -131,6 +134,69 @@ out_err:
 	return ERR_PTR(ret);
 }
 
+#if defined(CONFIG_NFS_V4_1)
+/*
+ * The callback service for NFSv4.1 callbacks
+ */
+static int
+nfs41_callback_svc(void *vrqstp)
+{
+	struct svc_rqst *rqstp = vrqstp;
+	struct svc_serv *serv = rqstp->rq_server;
+	struct rpc_rqst *req;
+	int error;
+	DEFINE_WAIT(wq);
+
+	set_freezable();
+
+	/*
+	 * FIXME: do we really need to run this under the BKL? If so, please
+	 * add a comment about what it's intended to protect.
+	 */
+	lock_kernel();
+	while (!kthread_should_stop()) {
+		prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_INTERRUPTIBLE);
+		spin_lock_bh(&serv->sv_cb_lock);
+		if (!list_empty(&serv->sv_cb_list)) {
+			req = list_first_entry(&serv->sv_cb_list,
+					struct rpc_rqst, rq_bc_list);
+			list_del(&req->rq_bc_list);
+			spin_unlock_bh(&serv->sv_cb_lock);
+			dprintk("Invoking bc_svc_process()\n");
+			error = bc_svc_process(serv, req, rqstp);
+			dprintk("bc_svc_process() returned w/ error code= %d\n",
+				error);
+		} else {
+			spin_unlock_bh(&serv->sv_cb_lock);
+			schedule();
+		}
+		finish_wait(&serv->sv_cb_waitq, &wq);
+	}
+	unlock_kernel();
+	nfs_callback_info.task = NULL;
+	svc_exit_thread(rqstp);
+	return 0;
+}
+
+/*
+ * Bring up the NFSv4.1 callback service
+ */
+struct svc_rqst *
+nfs41_callback_up(struct svc_serv *serv, struct rpc_xprt *xprt)
+{
+	/*
+	 * Save the svc_serv in the transport so that it can
+	 * be referenced when the session backchannel is initialized
+	 */
+	xprt->bc_serv = serv;
+
+	INIT_LIST_HEAD(&serv->sv_cb_list);
+	spin_lock_init(&serv->sv_cb_lock);
+	init_waitqueue_head(&serv->sv_cb_waitq);
+	return svc_prepare_thread(serv, &serv->sv_pools[0]);
+}
+#endif /* CONFIG_NFS_V4_1 */
+
 /*
  * Bring up the callback thread if it is not already up.
  */
@@ -141,10 +207,18 @@ int nfs_callback_up(u32 minorversion, void *args)
 	int (*callback_svc)(void *vrqstp);
 	char svc_name[12];
 	int ret = 0;
+#if defined(CONFIG_NFS_V4_1)
+	struct rpc_xprt *xprt = (struct rpc_xprt *)args;
+#endif /* CONFIG_NFS_V4_1 */
 
 	mutex_lock(&nfs_callback_mutex);
-	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
+	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL) {
+#if defined(CONFIG_NFS_V4_1)
+		if (minorversion)
+			xprt->bc_serv = nfs_callback_info.serv;
+#endif /* CONFIG_NFS_V4_1 */
 		goto out;
+	}
 	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
 	if (!serv) {
 		ret = -ENOMEM;
@@ -157,7 +231,12 @@ int nfs_callback_up(u32 minorversion, void *args)
 		rqstp = nfs4_callback_up(serv);
 		callback_svc = nfs4_callback_svc;
 	} else {
-		BUG();	/* for now */
+#if defined(CONFIG_NFS_V4_1)
+		rqstp = nfs41_callback_up(serv, xprt);
+		callback_svc = nfs41_callback_svc;
+#else  /* CONFIG_NFS_V4_1 */
+		BUG();
+#endif /* CONFIG_NFS_V4_1 */
 	}
 
 	if (IS_ERR(rqstp)) {
diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index 7f12e65..f2696ba 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -70,6 +70,13 @@ extern void nfs_callback_down(void);
 #define nfs_callback_down()	do {} while(0)
 #endif
 
+/*
+ * nfs41: Callbacks are expected to not cause substantial latency,
+ * so we limit their concurrency to 1 by setting up the maximum number
+ * of slots for the backchannel.
+ */
+#define NFS41_BC_MIN_CALLBACKS 1
+
 extern unsigned int nfs_callback_set_tcpport;
 extern unsigned short nfs_callback_tcpport;
 extern unsigned short nfs_callback_tcpport6;
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 4a8afbd..16043c4 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -419,6 +419,8 @@ int		   svc_set_num_threads(struct svc_serv *, struct svc_pool *, int);
 int		   svc_pool_stats_open(struct svc_serv *serv, struct file *file);
 void		   svc_destroy(struct svc_serv *);
 int		   svc_process(struct svc_rqst *);
+int		   bc_svc_process(struct svc_serv *, struct rpc_rqst *,
+			struct svc_rqst *);
 int		   svc_register(const struct svc_serv *, const int,
 				const unsigned short, const unsigned short);
 
-- 
1.6.2.1

[RFC 15/39] nfs41: sunrpc: provide functions to create and destroy a svc_xprt for backchannel use

[Benny Halevy]

For nfs41 callbacks we need an svc_xprt to process requests coming up the
backchannel socket as rpc_rqst's that are transformed into svc_rqst's that
need a rq_xprt to be processed.

The svc_{udp,tcp}_create methods are too heavy for this job as svc_create_socket
creates an actual socket to listen on while for nfs41 we're "reusing" the
fore channel's socket.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/svcsock.h |    2 ++
 net/sunrpc/svcsock.c           |   39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h
index 483e103..6bb1ec4 100644
--- a/include/linux/sunrpc/svcsock.h
+++ b/include/linux/sunrpc/svcsock.h
@@ -42,6 +42,8 @@ int		svc_sock_names(char *buf, struct svc_serv *serv, char *toclose);
 int		svc_addsock(struct svc_serv *serv, int fd, char *name_return);
 void		svc_init_xprt_sock(void);
 void		svc_cleanup_xprt_sock(void);
+struct svc_xprt *svc_sock_create(struct svc_serv *serv, int prot);
+void		svc_sock_destroy(struct svc_xprt *);
 
 /*
  * svc_makesock socket characteristics
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index af31988..2334546 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -1306,3 +1306,42 @@ static void svc_sock_free(struct svc_xprt *xprt)
 		sock_release(svsk->sk_sock);
 	kfree(svsk);
 }
+
+/*
+ * Create a svc_xprt.
+ *
+ * For internal use only (e.g. nfsv4.1 backchannel).
+ * Callers should typically use the xpo_create() method.
+ */
+struct svc_xprt *svc_sock_create(struct svc_serv *serv, int prot)
+{
+	struct svc_sock *svsk;
+	struct svc_xprt *xprt = NULL;
+
+	dprintk("svc: %s\n", __func__);
+	svsk = kzalloc(sizeof(*svsk), GFP_KERNEL);
+	if (!svsk)
+		goto out;
+
+	xprt = &svsk->sk_xprt;
+	if (prot == IPPROTO_TCP)
+		svc_xprt_init(&svc_tcp_class, xprt, serv);
+	else if (prot == IPPROTO_UDP)
+		svc_xprt_init(&svc_udp_class, xprt, serv);
+	else
+		BUG();
+out:
+	dprintk("svc: %s return %p\n", __func__, xprt);
+	return xprt;
+}
+EXPORT_SYMBOL_GPL(svc_sock_create);
+
+/*
+ * Destroy a svc_sock.
+ */
+void svc_sock_destroy(struct svc_xprt *xprt)
+{
+	if (xprt)
+		kfree(container_of(xprt, struct svc_sock, sk_xprt));
+}
+EXPORT_SYMBOL_GPL(svc_sock_destroy);
-- 
1.6.2.1

[RFC 16/39] nfs41: sunrpc: add a struct svc_xprt pointer to struct svc_serv for backchannel use

[Benny Halevy]

From: Andy Adamson <andros@netapp.com>

This svc_xprt is passed on to the callback service thread to be later used
to processes incoming svc_rqst's

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/svc.h |    1 +
 net/sunrpc/svc.c           |    4 ++++
 2 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 16043c4..ea80096 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -103,6 +103,7 @@ struct svc_serv {
 	spinlock_t		sv_cb_lock;	/* protects the svc_cb_list */
 	wait_queue_head_t	sv_cb_waitq;	/* sleep here if there are no
 						 * entries in the svc_cb_list */
+	struct svc_xprt		*bc_xprt;
 #endif /* CONFIG_NFS_V4_1 */
 };
 
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 06b52e4..b35048f 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -487,6 +487,10 @@ svc_destroy(struct svc_serv *serv)
 	if (svc_serv_is_pooled(serv))
 		svc_pool_map_put();
 
+#if defined(CONFIG_NFS_V4_1)
+	svc_sock_destroy(serv->bc_xprt);
+#endif /* CONFIG_NFS_V4_1 */
+
 	svc_unregister(serv);
 	kfree(serv->sv_pools);
 	kfree(serv);
-- 
1.6.2.1

[RFC 17/39] nfs41: create a svc_xprt for nfs41 callback thread and use for incoming callbacks

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.c |   17 ++++++++++++++++-
 net/sunrpc/svc.c  |    1 +
 2 files changed, 17 insertions(+), 1 deletions(-)

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index 9ee174c..8d7c142 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -184,16 +184,31 @@ nfs41_callback_svc(void *vrqstp)
 struct svc_rqst *
 nfs41_callback_up(struct svc_serv *serv, struct rpc_xprt *xprt)
 {
+	struct svc_xprt *bc_xprt;
+	struct svc_rqst *rqstp = ERR_PTR(-ENOMEM);
+
+	dprintk("--> %s\n", __func__);
+	/* Create a svc_sock for the service */
+	bc_xprt = svc_sock_create(serv, xprt->prot);
+	if (!bc_xprt)
+		goto out;
+
 	/*
 	 * Save the svc_serv in the transport so that it can
 	 * be referenced when the session backchannel is initialized
 	 */
+	serv->bc_xprt = bc_xprt;
 	xprt->bc_serv = serv;
 
 	INIT_LIST_HEAD(&serv->sv_cb_list);
 	spin_lock_init(&serv->sv_cb_lock);
 	init_waitqueue_head(&serv->sv_cb_waitq);
-	return svc_prepare_thread(serv, &serv->sv_pools[0]);
+	rqstp = svc_prepare_thread(serv, &serv->sv_pools[0]);
+	if (IS_ERR(rqstp))
+		svc_sock_destroy(bc_xprt);
+out:
+	dprintk("--> %s return %p\n", __func__, rqstp);
+	return rqstp;
 }
 #endif /* CONFIG_NFS_V4_1 */
 
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index b35048f..6b90ce4 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1258,6 +1258,7 @@ bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
 	int 		error;
 
 	/* Build the svc_rqst used by the common processing routine */
+	rqstp->rq_xprt = serv->bc_xprt;
 	rqstp->rq_xid = req->rq_xid;
 	rqstp->rq_prot = req->rq_xprt->prot;
 	rqstp->rq_server = serv;
-- 
1.6.2.1

[RFC 18/39] nfs41: save svc_serv in nfs_callback_info

[Benny Halevy]

We need to pass the svc_serv* to the xprt when nfs_callback_info.users > 0

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index 8d7c142..4bf60b2 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -31,6 +31,7 @@
 
 struct nfs_callback_data {
 	unsigned int users;
+	struct svc_serv *serv;
 	struct svc_rqst *rqst;
 	struct task_struct *task;
 };
@@ -262,6 +263,7 @@ int nfs_callback_up(u32 minorversion, void *args)
 	svc_sock_update_bufs(serv);
 
 	sprintf(svc_name, "nfsv4.%u-svc", minorversion);
+	nfs_callback_info.serv = serv;
 	nfs_callback_info.rqst = rqstp;
 	nfs_callback_info.task = kthread_run(callback_svc,
 					     nfs_callback_info.rqst,
@@ -269,6 +271,7 @@ int nfs_callback_up(u32 minorversion, void *args)
 	if (IS_ERR(nfs_callback_info.task)) {
 		ret = PTR_ERR(nfs_callback_info.task);
 		svc_exit_thread(nfs_callback_info.rqst);
+		nfs_callback_info.serv = NULL;
 		nfs_callback_info.rqst = NULL;
 		nfs_callback_info.task = NULL;
 		goto out_err;
@@ -301,6 +304,7 @@ void nfs_callback_down(void)
 	if (nfs_callback_info.users == 0 && nfs_callback_info.task != NULL) {
 		kthread_stop(nfs_callback_info.task);
 		svc_exit_thread(nfs_callback_info.rqst);
+		nfs_callback_info.serv = NULL;
 		nfs_callback_info.rqst = NULL;
 		nfs_callback_info.task = NULL;
 	}
-- 
1.6.2.1

[RFC 19/39] nfs41: Allow NFSv4 and NFSv4.1 callback services to coexist

[Benny Halevy]

From: Andy Adamson <andros@netapp.com>

Tracks the nfs_callback_info for both versions, enabling the callback
service for v4 and v4.1 to run concurrently and be stopped independently
of each other.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.c |   50 ++++++++++++++++++++++++--------------------------
 fs/nfs/callback.h |    7 ++-----
 fs/nfs/client.c   |    2 +-
 3 files changed, 27 insertions(+), 32 deletions(-)

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index 4bf60b2..972e38b 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -36,7 +36,7 @@ struct nfs_callback_data {
 	struct task_struct *task;
 };
 
-static struct nfs_callback_data nfs_callback_info;
+static struct nfs_callback_data nfs_callback_info[NFS4_MAX_MINOR_VERSION + 1];
 static DEFINE_MUTEX(nfs_callback_mutex);
 static struct svc_program nfs4_callback_program;
 
@@ -60,7 +60,7 @@ module_param_call(callback_tcpport, param_set_port, param_get_int,
 		 &nfs_callback_set_tcpport, 0644);
 
 /*
- * This is the callback kernel thread.
+ * This is the NFSv4 callback kernel thread.
  */
 static int
 nfs4_callback_svc(void *vrqstp)
@@ -174,8 +174,6 @@ nfs41_callback_svc(void *vrqstp)
 		finish_wait(&serv->sv_cb_waitq, &wq);
 	}
 	unlock_kernel();
-	nfs_callback_info.task = NULL;
-	svc_exit_thread(rqstp);
 	return 0;
 }
 
@@ -221,6 +219,7 @@ int nfs_callback_up(u32 minorversion, void *args)
 	struct svc_serv *serv = NULL;
 	struct svc_rqst *rqstp;
 	int (*callback_svc)(void *vrqstp);
+	struct nfs_callback_data *cb_info = &nfs_callback_info[minorversion];
 	char svc_name[12];
 	int ret = 0;
 #if defined(CONFIG_NFS_V4_1)
@@ -228,10 +227,10 @@ int nfs_callback_up(u32 minorversion, void *args)
 #endif /* CONFIG_NFS_V4_1 */
 
 	mutex_lock(&nfs_callback_mutex);
-	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL) {
+	if (cb_info->users++ || cb_info->task != NULL) {
 #if defined(CONFIG_NFS_V4_1)
 		if (minorversion)
-			xprt->bc_serv = nfs_callback_info.serv;
+			xprt->bc_serv = cb_info->serv;
 #endif /* CONFIG_NFS_V4_1 */
 		goto out;
 	}
@@ -263,17 +262,14 @@ int nfs_callback_up(u32 minorversion, void *args)
 	svc_sock_update_bufs(serv);
 
 	sprintf(svc_name, "nfsv4.%u-svc", minorversion);
-	nfs_callback_info.serv = serv;
-	nfs_callback_info.rqst = rqstp;
-	nfs_callback_info.task = kthread_run(callback_svc,
-					     nfs_callback_info.rqst,
-					     svc_name);
-	if (IS_ERR(nfs_callback_info.task)) {
-		ret = PTR_ERR(nfs_callback_info.task);
-		svc_exit_thread(nfs_callback_info.rqst);
-		nfs_callback_info.serv = NULL;
-		nfs_callback_info.rqst = NULL;
-		nfs_callback_info.task = NULL;
+	cb_info->serv = serv;
+	cb_info->rqst = rqstp;
+	cb_info->task = kthread_run(callback_svc, cb_info->rqst, svc_name);
+	if (IS_ERR(cb_info->task)) {
+		ret = PTR_ERR(cb_info->task);
+		svc_exit_thread(cb_info->rqst);
+		cb_info->rqst = NULL;
+		cb_info->task = NULL;
 		goto out_err;
 	}
 out:
@@ -290,23 +286,25 @@ out:
 out_err:
 	dprintk("NFS: Couldn't create callback socket or server thread; "
 		"err = %d\n", ret);
-	nfs_callback_info.users--;
+	cb_info->users--;
 	goto out;
 }
 
 /*
  * Kill the callback thread if it's no longer being used.
  */
-void nfs_callback_down(void)
+void nfs_callback_down(int minorversion)
 {
+	struct nfs_callback_data *cb_info = &nfs_callback_info[minorversion];
+
 	mutex_lock(&nfs_callback_mutex);
-	nfs_callback_info.users--;
-	if (nfs_callback_info.users == 0 && nfs_callback_info.task != NULL) {
-		kthread_stop(nfs_callback_info.task);
-		svc_exit_thread(nfs_callback_info.rqst);
-		nfs_callback_info.serv = NULL;
-		nfs_callback_info.rqst = NULL;
-		nfs_callback_info.task = NULL;
+	cb_info->users--;
+	if (cb_info->users == 0 && cb_info->task != NULL) {
+		kthread_stop(cb_info->task);
+		svc_exit_thread(cb_info->rqst);
+		cb_info->serv = NULL;
+		cb_info->rqst = NULL;
+		cb_info->task = NULL;
 	}
 	mutex_unlock(&nfs_callback_mutex);
 }
diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index f2696ba..ed52e06 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -64,11 +64,8 @@ extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
 
 #ifdef CONFIG_NFS_V4
 extern int nfs_callback_up(u32 minorversion, void *args);
-extern void nfs_callback_down(void);
-#else
-#define nfs_callback_up()	(0)
-#define nfs_callback_down()	do {} while(0)
-#endif
+extern void nfs_callback_down(int minorversion);
+#endif /* CONFIG_NFS_V4 */
 
 /*
  * nfs41: Callbacks are expected to not cause substantial latency,
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index df2b40d..ac6575c 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -181,7 +181,7 @@ static void nfs4_destroy_callback(struct nfs_client *clp)
 {
 #ifdef CONFIG_NFS_V4
 	if (__test_and_clear_bit(NFS_CS_CALLBACK, &clp->cl_res_state))
-		nfs_callback_down();
+		nfs_callback_down(clp->cl_minorversion);
 #endif /* CONFIG_NFS_V4 */
 }
 
-- 
1.6.2.1

[RFC 20/39] nfs41: Setup the backchannel

[Benny Halevy]

From: Andy Adamson <andros@netapp.com>

The NFS v4.1 callback service has already been setup, and
rpc_xprt->serv points to the svc_serv structure describing it.
Invoke the xprt_setup_backchannel() initialization to pre-
allocate the necessary backchannel structures.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: change nfs4_put_session(nfs4_session**) to nfs4_destroy_session(nfs_session*)]
Signed-off-by: Alexandros Batsakis <Alexandros.Batsakis@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[moved xprt_setup_backchannel from nfs4_init_session to nfs4_init_backchannel]
Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/client.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index ac6575c..4f75ec5 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -37,6 +37,7 @@
 #include <linux/in6.h>
 #include <net/ipv6.h>
 #include <linux/nfs_xdr.h>
+#include <linux/sunrpc/bc_xprt.h>
 
 #include <asm/system.h>
 
@@ -1096,6 +1097,14 @@ static int nfs4_init_callback(struct nfs_client *clp)
 	int error;
 
 	if (clp->rpc_ops->version == 4) {
+		if (nfs4_has_session(clp)) {
+			error = xprt_setup_backchannel(
+						clp->cl_rpcclient->cl_xprt,
+						NFS41_BC_MIN_CALLBACKS);
+			if (error < 0)
+				return error;
+		}
+
 		error = nfs_callback_up(clp->cl_minorversion,
 					clp->cl_rpcclient->cl_xprt);
 		if (error < 0) {
-- 
1.6.2.1

[RFC 21/39] nfs41: Client indicates presence of NFSv4.1 callback channel.

[Benny Halevy]

From: Andy Adamson <andros@netapp.com>

Set the SESSION4_BACK_CHAN flag to indicate the client supports a backchannel.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 530ebb1..aab69f0 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4601,7 +4601,7 @@ static int _nfs4_proc_create_session(struct nfs_client *clp)
 	int status;
 
 	nfs4_init_channel_attrs(&args);
-	args.flags = (SESSION4_PERSIST);
+	args.flags = (SESSION4_PERSIST | SESSION4_BACK_CHAN);
 
 	status = rpc_call_sync(session->clp->cl_rpcclient, &msg, 0);
 
-- 
1.6.2.1

[RFC 22/39] nfs41: Get the rpc_xprt * from the rpc_rqst instead of the rpc_clnt.

[Benny Halevy]

From: Rahul Iyer <iyer@netapp.com>

Obtain the rpc_xprt from the rpc_rqst so that calls and callback replies
can both use the same code path.  A client needs the rpc_xprt in order
to reply to a callback.

Signed-off-by: Rahul Iyer <iyer@netapp.com>
Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 net/sunrpc/xprt.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index df65d15..c73b31f 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -195,8 +195,8 @@ EXPORT_SYMBOL_GPL(xprt_load_transport);
  */
 int xprt_reserve_xprt(struct rpc_task *task)
 {
-	struct rpc_xprt	*xprt = task->tk_xprt;
 	struct rpc_rqst *req = task->tk_rqstp;
+	struct rpc_xprt	*xprt = req->rq_xprt;
 
 	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
 		if (task == xprt->snd_task)
@@ -860,7 +860,7 @@ out_unlock:
 
 void xprt_end_transmit(struct rpc_task *task)
 {
-	xprt_release_write(task->tk_xprt, task);
+	xprt_release_write(task->tk_rqstp->rq_xprt, task);
 }
 
 /**
-- 
1.6.2.1

[RFC 23/39] nfs41: Release backchannel resources associated with session

[Benny Halevy]

From: Andy Adamson <andros@netapp.com>

Frees the preallocated backchannel resources that are associated with
this session when the session is destroyed.

A backchannel is currently created once per session. Destroy the backchannel
only when the session is destroyed.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Andy Adamson<andros@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c |    6 ++++++
 fs/nfs/super.c    |    1 -
 2 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index aab69f0..485c279 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -49,6 +49,7 @@
 #include <linux/namei.h>
 #include <linux/mount.h>
 #include <linux/module.h>
+#include <linux/sunrpc/bc_xprt.h>
 
 #include "nfs4_fs.h"
 #include "delegation.h"
@@ -4484,6 +4485,11 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
 
 void nfs4_destroy_session(struct nfs4_session *session)
 {
+	nfs4_proc_destroy_session(session);
+	dprintk("%s Destroy backchannel for xprt %p\n",
+		__func__, session->clp->cl_rpcclient->cl_xprt);
+	xprt_destroy_backchannel(session->clp->cl_rpcclient->cl_xprt,
+				NFS41_BC_MIN_CALLBACKS);
 	nfs4_destroy_slot_table(session);
 	kfree(session);
 }
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index 892215c..923b28f 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -2486,7 +2486,6 @@ static void nfs4_kill_super(struct super_block *sb)
 	dprintk("--> %s\n", __func__);
 	nfs_super_return_all_delegations(sb);
 	kill_anon_super(sb);
-
 	nfs4_renewd_prepare_shutdown(server);
 	nfs_fscache_release_super_cookie(sb);
 	nfs_free_server(server);
-- 
1.6.2.1

[RFC 24/39] nfs41: store minorversion in cb_compound_hdr_arg

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h     |    1 +
 fs/nfs/callback_xdr.c |   12 +++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index ed52e06..21881d9 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -26,6 +26,7 @@ enum nfs4_callback_opnum {
 struct cb_compound_hdr_arg {
 	unsigned int taglen;
 	const char *tag;
+	unsigned int minorversion;
 	unsigned int callback_ident;
 	unsigned nops;
 };
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index dd0ef34..91f6f74 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -132,7 +132,6 @@ static __be32 decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid)
 static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound_hdr_arg *hdr)
 {
 	__be32 *p;
-	unsigned int minor_version;
 	__be32 status;
 
 	status = decode_string(xdr, &hdr->taglen, &hdr->tag);
@@ -147,15 +146,18 @@ static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound
 	p = read_buf(xdr, 12);
 	if (unlikely(p == NULL))
 		return htonl(NFS4ERR_RESOURCE);
-	minor_version = ntohl(*p++);
+	hdr->minorversion = ntohl(*p++);
 	/* Check minor version is zero. */
-	if (minor_version != 0) {
-		printk(KERN_WARNING "%s: NFSv4 server callback with illegal minor version %u!\n",
-				__func__, minor_version);
+	if (hdr->minorversion != 0) {
+		printk(KERN_WARNING "%s: NFSv4 server callback with "
+			"illegal minor version %u!\n",
+			__func__, hdr->minorversion);
 		return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
 	}
 	hdr->callback_ident = ntohl(*p++);
 	hdr->nops = ntohl(*p);
+	dprintk("%s: minorversion %d nops %d\n", __func__,
+		hdr->minorversion, hdr->nops);
 	return 0;
 }
 
-- 
1.6.2.1

[RFC 25/39] nfs41: decode minorversion 1 cb_compound header

[Benny Halevy]

decode cb_compound header conforming to
http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion1-26

Get rid of cb_compound_hdr_arg.callback_ident

callback_ident is not used anywhere so we shouldn't waste any memory to
store it.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h     |    1 -
 fs/nfs/callback_xdr.c |   12 ++++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index 21881d9..e0f4e6b 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -27,7 +27,6 @@ struct cb_compound_hdr_arg {
 	unsigned int taglen;
 	const char *tag;
 	unsigned int minorversion;
-	unsigned int callback_ident;
 	unsigned nops;
 };
 
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 91f6f74..e141a7e 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -143,18 +143,22 @@ static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound
 				__func__, hdr->taglen);
 		return htonl(NFS4ERR_RESOURCE);
 	}
-	p = read_buf(xdr, 12);
+	p = read_buf(xdr, 4);
 	if (unlikely(p == NULL))
 		return htonl(NFS4ERR_RESOURCE);
 	hdr->minorversion = ntohl(*p++);
-	/* Check minor version is zero. */
-	if (hdr->minorversion != 0) {
+	/* Check minor version is zero or one. */
+	if (hdr->minorversion <= 1) {
+		p = read_buf(xdr, 8);
+		if (unlikely(p == NULL))
+			return htonl(NFS4ERR_RESOURCE);
+		p++;	/* skip callback_ident */
+	} else {
 		printk(KERN_WARNING "%s: NFSv4 server callback with "
 			"illegal minor version %u!\n",
 			__func__, hdr->minorversion);
 		return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
 	}
-	hdr->callback_ident = ntohl(*p++);
 	hdr->nops = ntohl(*p);
 	dprintk("%s: minorversion %d nops %d\n", __func__,
 		hdr->minorversion, hdr->nops);
-- 
1.6.2.1

[RFC 26/39] nfs41: callback numbers definitions

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index e0f4e6b..3d1a324 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -20,6 +20,16 @@ enum nfs4_callback_procnum {
 enum nfs4_callback_opnum {
 	OP_CB_GETATTR = 3,
 	OP_CB_RECALL  = 4,
+/* Callback operations new to NFSv4.1 */
+	OP_CB_LAYOUTRECALL  = 5,
+	OP_CB_NOTIFY        = 6,
+	OP_CB_PUSH_DELEG    = 7,
+	OP_CB_RECALL_ANY    = 8,
+	OP_CB_RECALLABLE_OBJ_AVAIL = 9,
+	OP_CB_RECALL_SLOT   = 10,
+	OP_CB_SEQUENCE      = 11,
+	OP_CB_WANTS_CANCELLED = 12,
+	OP_CB_NOTIFY_LOCK   = 13,
 	OP_CB_ILLEGAL = 10044,
 };
 
-- 
1.6.2.1

[RFC 27/39] nfs41: consider minorversion in callback_xdr:process_op

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_xdr.c |   63 ++++++++++++++++++++++++++++++++++++++----------
 1 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index e141a7e..dd25794 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -359,31 +359,61 @@ out:
 	return status;
 }
 
-static __be32 process_op(struct svc_rqst *rqstp,
+static __be32 process_op(uint32_t minorversion, int nop,
+		struct svc_rqst *rqstp,
 		struct xdr_stream *xdr_in, void *argp,
 		struct xdr_stream *xdr_out, void *resp)
 {
 	struct callback_op *op = &callback_ops[0];
 	unsigned int op_nr = OP_CB_ILLEGAL;
-	__be32 status = 0;
+	__be32 status;
 	long maxlen;
 	__be32 res;
 
 	dprintk("%s: start\n", __func__);
 	status = decode_op_hdr(xdr_in, &op_nr);
-	if (likely(status == 0)) {
+	if (unlikely(status))
+		goto out_illegal;
+
+	dprintk("%s: minorversion=%d nop=%d op_nr=%u\n",
+		__func__, minorversion, nop, op_nr);
+#if defined(CONFIG_NFS_V4_1)
+	if (minorversion == 1) {
 		switch (op_nr) {
-			case OP_CB_GETATTR:
-			case OP_CB_RECALL:
-				op = &callback_ops[op_nr];
-				break;
-			default:
-				op_nr = OP_CB_ILLEGAL;
-				op = &callback_ops[0];
-				status = htonl(NFS4ERR_OP_ILLEGAL);
+		case OP_CB_GETATTR:
+		case OP_CB_RECALL:
+			op = &callback_ops[op_nr];
+			break;
+
+		case OP_CB_LAYOUTRECALL:
+		case OP_CB_NOTIFY:
+		case OP_CB_PUSH_DELEG:
+		case OP_CB_RECALL_ANY:
+		case OP_CB_RECALLABLE_OBJ_AVAIL:
+		case OP_CB_RECALL_SLOT:
+		case OP_CB_SEQUENCE:
+		case OP_CB_WANTS_CANCELLED:
+		case OP_CB_NOTIFY_LOCK:
+			op = &callback_ops[0];
+			status = htonl(NFS4ERR_NOTSUPP);
+			break;
+		default:
+			goto out_illegal;
 		}
-	}
 
+		goto out;
+	}
+#endif /* defined(CONFIG_NFS_V4_1) */
+
+	switch (op_nr) {
+	case OP_CB_GETATTR:
+	case OP_CB_RECALL:
+		op = &callback_ops[op_nr];
+		break;
+	default:
+		goto out_illegal;
+	}
+out:
 	maxlen = xdr_out->end - xdr_out->p;
 	if (maxlen > 0 && maxlen < PAGE_SIZE) {
 		if (likely(status == 0 && op->decode_args != NULL))
@@ -400,6 +430,12 @@ static __be32 process_op(struct svc_rqst *rqstp,
 		status = op->encode_res(rqstp, xdr_out, resp);
 	dprintk("%s: done, status = %d\n", __func__, ntohl(status));
 	return status;
+
+out_illegal:
+	op_nr = OP_CB_ILLEGAL;
+	op = &callback_ops[0];
+	status = htonl(NFS4ERR_OP_ILLEGAL);
+	goto out;
 }
 
 /*
@@ -431,7 +467,8 @@ static __be32 nfs4_callback_compound(struct svc_rqst *rqstp, void *argp, void *r
 		return rpc_system_err;
 
 	while (status == 0 && nops != hdr_arg.nops) {
-		status = process_op(rqstp, &xdr_in, argp, &xdr_out, resp);
+		status = process_op(hdr_arg.minorversion, nops,
+				    rqstp, &xdr_in, argp, &xdr_out, resp);
 		nops++;
 	}
 
-- 
1.6.2.1

[RFC 28/39] nfs41: define CB_NOTIFY_DEVICEID as not supported

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Need to return NFS4ERR_NOTSUPP for CB_NOTIFY_DEVICEID rather
than NFS4ERR_OP_ILLEGAL.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h     |    1 +
 fs/nfs/callback_xdr.c |    1 +
 2 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index 3d1a324..b2aea14 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -30,6 +30,7 @@ enum nfs4_callback_opnum {
 	OP_CB_SEQUENCE      = 11,
 	OP_CB_WANTS_CANCELLED = 12,
 	OP_CB_NOTIFY_LOCK   = 13,
+	OP_CB_NOTIFY_DEVICEID = 14,
 	OP_CB_ILLEGAL = 10044,
 };
 
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index dd25794..9db88ce 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -394,6 +394,7 @@ static __be32 process_op(uint32_t minorversion, int nop,
 		case OP_CB_SEQUENCE:
 		case OP_CB_WANTS_CANCELLED:
 		case OP_CB_NOTIFY_LOCK:
+		case OP_CB_NOTIFY_DEVICEID:
 			op = &callback_ops[0];
 			status = htonl(NFS4ERR_NOTSUPP);
 			break;
-- 
1.6.2.1

[RFC 29/39] nfs41: cb_sequence protocol level data structures

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h |   35 +++++++++++++++++++++++++++++++++++
 1 files changed, 35 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index b2aea14..1b046ec 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -70,6 +70,41 @@ struct cb_recallargs {
 	uint32_t truncate;
 };
 
+#if defined(CONFIG_NFS_V4_1)
+
+struct referring_call {
+	uint32_t			rc_sequenceid;
+	uint32_t			rc_slotid;
+};
+
+struct referring_call_list {
+	struct nfs4_sessionid		rcl_sessionid;
+	uint32_t			rcl_nrefcalls;
+	struct referring_call 		*rcl_refcalls;
+};
+
+struct cb_sequenceargs {
+	struct sockaddr_in		*csa_addr;
+	struct nfs4_sessionid		csa_sessionid;
+	uint32_t			csa_sequenceid;
+	uint32_t			csa_slotid;
+	uint32_t			csa_highestslotid;
+	uint32_t			csa_cachethis;
+	uint32_t			csa_nrclists;
+	struct referring_call_list	*csa_rclists;
+};
+
+struct cb_sequenceres {
+	uint32_t			csr_status;
+	struct nfs4_sessionid		csr_sessionid;
+	uint32_t			csr_sequenceid;
+	uint32_t			csr_slotid;
+	uint32_t			csr_highestslotid;
+	uint32_t			csr_target_highestslotid;
+};
+
+#endif /* CONFIG_NFS_V4_1 */
+
 extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res);
 extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
 
-- 
1.6.2.1

[RFC 30/39] nfs41: cb_sequence proc implementation

[Benny Halevy]

Currently, just free up any referring calls information.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
[nfs41: fix csr_{,target}highestslotid]
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h      |    4 ++++
 fs/nfs/callback_proc.c |   28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index 1b046ec..0316a06 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -103,6 +103,9 @@ struct cb_sequenceres {
 	uint32_t			csr_target_highestslotid;
 };
 
+extern unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
+				       struct cb_sequenceres *res);
+
 #endif /* CONFIG_NFS_V4_1 */
 
 extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res);
@@ -119,6 +122,7 @@ extern void nfs_callback_down(int minorversion);
  * of slots for the backchannel.
  */
 #define NFS41_BC_MIN_CALLBACKS 1
+#define NFS41_BC_MAX_CALLBACKS 1
 
 extern unsigned int nfs_callback_set_tcpport;
 extern unsigned short nfs_callback_tcpport;
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index f7e83e2..f731bbe 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -101,3 +101,31 @@ out:
 	dprintk("%s: exit with status = %d\n", __func__, ntohl(res));
 	return res;
 }
+
+#if defined(CONFIG_NFS_V4_1)
+
+/* FIXME: validate args->cbs_{sequence,slot}id */
+/* FIXME: referring calls should be processed */
+unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
+				struct cb_sequenceres *res)
+{
+	int i;
+	unsigned status = 0;
+
+	for (i = 0; i < args->csa_nrclists; i++)
+		kfree(args->csa_rclists[i].rcl_refcalls);
+	kfree(args->csa_rclists);
+
+	memcpy(&res->csr_sessionid, &args->csa_sessionid,
+	       sizeof(res->csr_sessionid));
+	res->csr_sequenceid = args->csa_sequenceid;
+	res->csr_slotid = args->csa_slotid;
+	res->csr_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
+	res->csr_target_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
+
+	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
+	res->csr_status = status;
+	return status;
+}
+
+#endif /* CONFIG_NFS_V4_1 */
-- 
1.6.2.1

[RFC 31/39] nfs41: cb_sequence xdr implementation

[Benny Halevy]

[nfs41: get rid of READMEM and COPYMEM for callback_xdr.c]
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_xdr.c |  181 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 179 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 9db88ce..066273c 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -20,8 +20,18 @@
 				2 + 2 + 3 + 3)
 #define CB_OP_RECALL_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ)
 
+#if defined(CONFIG_NFS_V4_1)
+#define CB_OP_SEQUENCE_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ + \
+					4 + 1 + 3)
+#endif /* CONFIG_NFS_V4_1 */
+
 #define NFSDBG_FACILITY NFSDBG_CALLBACK
 
+#define READ64(x)         do {			\
+	(x) = (u64)ntohl(*p++) << 32;		\
+	(x) |= ntohl(*p++);			\
+} while (0)
+
 typedef __be32 (*callback_process_op_t)(void *, void *);
 typedef __be32 (*callback_decode_arg_t)(struct svc_rqst *, struct xdr_stream *, void *);
 typedef __be32 (*callback_encode_res_t)(struct svc_rqst *, struct xdr_stream *, void *);
@@ -210,6 +220,122 @@ out:
 	return status;
 }
 
+#if defined(CONFIG_NFS_V4_1)
+
+static unsigned decode_sessionid(struct xdr_stream *xdr,
+				 struct nfs4_sessionid *sid)
+{
+	uint32_t *p;
+	int len = NFS4_MAX_SESSIONID_LEN;
+
+	p = read_buf(xdr, len);
+	if (unlikely(p == NULL))
+		return htonl(NFS4ERR_RESOURCE);;
+
+	memcpy(sid->data, p, len);
+	return 0;
+}
+
+static unsigned decode_rc_list(struct xdr_stream *xdr,
+			       struct referring_call_list *rc_list)
+{
+	uint32_t *p;
+	int i;
+	unsigned status;
+
+	status = decode_sessionid(xdr, &rc_list->rcl_sessionid);
+	if (status)
+		goto out;
+
+	status = htonl(NFS4ERR_RESOURCE);
+	p = read_buf(xdr, sizeof(uint32_t));
+	if (unlikely(p == NULL))
+		goto out;
+
+	rc_list->rcl_nrefcalls = ntohl(*p++);
+	if (rc_list->rcl_nrefcalls) {
+		p = read_buf(xdr,
+			     rc_list->rcl_nrefcalls * 2 * sizeof(uint32_t));
+		if (unlikely(p == NULL))
+			goto out;
+		rc_list->rcl_refcalls = kmalloc(rc_list->rcl_nrefcalls *
+						sizeof(*rc_list->rcl_refcalls),
+						GFP_KERNEL);
+		if (unlikely(rc_list->rcl_refcalls == NULL))
+			goto out;
+		for (i = 0; i < rc_list->rcl_nrefcalls; i++) {
+			rc_list->rcl_refcalls[i].rc_sequenceid = ntohl(*p++);
+			rc_list->rcl_refcalls[i].rc_slotid = ntohl(*p++);
+		}
+	}
+	status = 0;
+
+out:
+	return status;
+}
+
+static unsigned decode_cb_sequence_args(struct svc_rqst *rqstp,
+					struct xdr_stream *xdr,
+					struct cb_sequenceargs *args)
+{
+	uint32_t *p;
+	int i;
+	unsigned status;
+
+	status = decode_sessionid(xdr, &args->csa_sessionid);
+	if (status)
+		goto out;
+
+	status = htonl(NFS4ERR_RESOURCE);
+	p = read_buf(xdr, 5 * sizeof(uint32_t));
+	if (unlikely(p == NULL))
+		goto out;
+
+	args->csa_addr = svc_addr_in(rqstp);
+	args->csa_sequenceid = ntohl(*p++);
+	args->csa_slotid = ntohl(*p++);
+	args->csa_highestslotid = ntohl(*p++);
+	args->csa_cachethis = ntohl(*p++);
+	args->csa_nrclists = ntohl(*p++);
+	args->csa_rclists = NULL;
+	if (args->csa_nrclists) {
+		args->csa_rclists = kmalloc(args->csa_nrclists *
+					    sizeof(*args->csa_rclists),
+					    GFP_KERNEL);
+		if (unlikely(args->csa_rclists == NULL))
+			goto out;
+
+		for (i = 0; i < args->csa_nrclists; i++) {
+			status = decode_rc_list(xdr, &args->csa_rclists[i]);
+			if (status)
+				goto out_free;
+		}
+	}
+	status = 0;
+
+	dprintk("%s: sessionid %x:%x:%x:%x sequenceid %u slotid %u "
+		"highestslotid %u cachethis %d nrclists %u\n",
+		__func__,
+		((u32 *)&args->csa_sessionid)[0],
+		((u32 *)&args->csa_sessionid)[1],
+		((u32 *)&args->csa_sessionid)[2],
+		((u32 *)&args->csa_sessionid)[3],
+		args->csa_sequenceid, args->csa_slotid,
+		args->csa_highestslotid, args->csa_cachethis,
+		args->csa_nrclists);
+out:
+	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
+	return status;
+
+out_free:
+	for (i = 0; i < args->csa_nrclists; i++)
+		kfree(args->csa_rclists[i].rcl_refcalls);
+	kfree(args->csa_rclists);
+	goto out;
+}
+
+#endif /* CONFIG_NFS_V4_1 */
+
 static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char *str)
 {
 	__be32 *p;
@@ -359,6 +485,49 @@ out:
 	return status;
 }
 
+#if defined(CONFIG_NFS_V4_1)
+
+static unsigned encode_sessionid(struct xdr_stream *xdr,
+				 const struct nfs4_sessionid *sid)
+{
+	uint32_t *p;
+	int len = NFS4_MAX_SESSIONID_LEN;
+
+	p = xdr_reserve_space(xdr, len);
+	if (unlikely(p == NULL))
+		return htonl(NFS4ERR_RESOURCE);
+
+	memcpy(p, sid, len);
+	return 0;
+}
+
+static unsigned encode_cb_sequence_res(struct svc_rqst *rqstp,
+				       struct xdr_stream *xdr,
+				       const struct cb_sequenceres *res)
+{
+	uint32_t *p;
+	unsigned status = res->csr_status;
+
+	if (unlikely(status != 0))
+		goto out;
+
+	encode_sessionid(xdr, &res->csr_sessionid);
+
+	p = xdr_reserve_space(xdr, 4 * sizeof(uint32_t));
+	if (unlikely(p == NULL))
+		return htonl(NFS4ERR_RESOURCE);
+
+	*p++ = htonl(res->csr_sequenceid);
+	*p++ = htonl(res->csr_slotid);
+	*p++ = htonl(res->csr_highestslotid);
+	*p++ = htonl(res->csr_target_highestslotid);
+out:
+	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
+	return status;
+}
+
+#endif /* CONFIG_NFS_V4_1 */
+
 static __be32 process_op(uint32_t minorversion, int nop,
 		struct svc_rqst *rqstp,
 		struct xdr_stream *xdr_in, void *argp,
@@ -382,6 +551,7 @@ static __be32 process_op(uint32_t minorversion, int nop,
 		switch (op_nr) {
 		case OP_CB_GETATTR:
 		case OP_CB_RECALL:
+		case OP_CB_SEQUENCE:
 			op = &callback_ops[op_nr];
 			break;
 
@@ -391,7 +561,6 @@ static __be32 process_op(uint32_t minorversion, int nop,
 		case OP_CB_RECALL_ANY:
 		case OP_CB_RECALLABLE_OBJ_AVAIL:
 		case OP_CB_RECALL_SLOT:
-		case OP_CB_SEQUENCE:
 		case OP_CB_WANTS_CANCELLED:
 		case OP_CB_NOTIFY_LOCK:
 		case OP_CB_NOTIFY_DEVICEID:
@@ -496,7 +665,15 @@ static struct callback_op callback_ops[] = {
 		.process_op = (callback_process_op_t)nfs4_callback_recall,
 		.decode_args = (callback_decode_arg_t)decode_recall_args,
 		.res_maxsize = CB_OP_RECALL_RES_MAXSZ,
-	}
+	},
+#if defined(CONFIG_NFS_V4_1)
+	[OP_CB_SEQUENCE] = {
+		.process_op = (callback_process_op_t)nfs4_callback_sequence,
+		.decode_args = (callback_decode_arg_t)decode_cb_sequence_args,
+		.encode_res = (callback_encode_res_t)encode_cb_sequence_res,
+		.res_maxsize = CB_OP_SEQUENCE_RES_MAXSZ,
+	},
+#endif /* CONFIG_NFS_V4_1 */
 };
 
 /*
-- 
1.6.2.1

[RFC 32/39] nfs41: verify CB_SEQUENCE position in callback compound

[Benny Halevy]

CB_SEQUENCE must appear first in the callback compound RPC.
If it is not the first operation NFS4ERR_SEQUENCE_POS must be returned.
If the first operation ni the CB_COMPOUND is not CB_SEQUENCE then
NFS4ERR_OP_NOT_IN_SESSION must be returned.

Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_xdr.c |    8 ++++++++
 1 files changed, 8 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 066273c..17cf089 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -548,6 +548,14 @@ static __be32 process_op(uint32_t minorversion, int nop,
 		__func__, minorversion, nop, op_nr);
 #if defined(CONFIG_NFS_V4_1)
 	if (minorversion == 1) {
+		if (op_nr == OP_CB_SEQUENCE) {
+			if (nop != 0) {
+				status = htonl(NFS4ERR_SEQUENCE_POS);
+				goto out;
+			}
+		} else if (nop == 0)
+			status = htonl(NFS4ERR_OP_NOT_IN_SESSION);
+
 		switch (op_nr) {
 		case OP_CB_GETATTR:
 		case OP_CB_RECALL:
-- 
1.6.2.1

[RFC 33/39] nfs41: Rename rq_received to rq_reply_bytes_recvd

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

The 'rq_received' member of 'struct rpc_rqst' is used to track when we
have received a reply to our request.  With v4.1, the backchannel
can now accept callback requests over the existing connection.  Rename
this field to make it clear that it is only used for tracking reply bytes
and not all bytes received on the connection.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 include/linux/sunrpc/xprt.h   |    3 ++-
 net/sunrpc/backchannel_rqst.c |    2 +-
 net/sunrpc/clnt.c             |    8 ++++----
 net/sunrpc/stats.c            |    2 +-
 net/sunrpc/xprt.c             |   15 ++++++++-------
 5 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 1531abe..3b710a7 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -67,7 +67,8 @@ struct rpc_rqst {
 	struct rpc_task *	rq_task;	/* RPC task data */
 	__be32			rq_xid;		/* request XID */
 	int			rq_cong;	/* has incremented xprt->cong */
-	int			rq_received;	/* receive completed */
+	int			rq_reply_bytes_recvd;	/* number of reply */
+							/* bytes received */
 	u32			rq_seqno;	/* gss seq no. used on req. */
 	int			rq_enc_pages_num;
 	struct page		**rq_enc_pages;	/* scratch pages for use by
diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c
index 92fb3bd..8f177a5 100644
--- a/net/sunrpc/backchannel_rqst.c
+++ b/net/sunrpc/backchannel_rqst.c
@@ -221,7 +221,7 @@ struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt)
 
 	if (req != NULL) {
 		set_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
-		req->rq_received = 0;
+		req->rq_reply_bytes_recvd = 0;
 		req->rq_bytes_sent = 0;
 		memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
 			sizeof(req->rq_private_buf));
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 349b4d6..2a9f843 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1258,8 +1258,8 @@ call_status(struct rpc_task *task)
 	struct rpc_rqst	*req = task->tk_rqstp;
 	int		status;
 
-	if (req->rq_received > 0 && !req->rq_bytes_sent)
-		task->tk_status = req->rq_received;
+	if (req->rq_reply_bytes_recvd > 0 && !req->rq_bytes_sent)
+		task->tk_status = req->rq_reply_bytes_recvd;
 
 	dprint_status(task);
 
@@ -1376,7 +1376,7 @@ call_decode(struct rpc_task *task)
 
 	/*
 	 * Ensure that we see all writes made by xprt_complete_rqst()
-	 * before it changed req->rq_received.
+	 * before it changed req->rq_reply_bytes_recvd.
 	 */
 	smp_rmb();
 	req->rq_rcv_buf.len = req->rq_private_buf.len;
@@ -1417,7 +1417,7 @@ out_retry:
 	task->tk_status = 0;
 	/* Note: rpc_verify_header() may have freed the RPC slot */
 	if (task->tk_rqstp == req) {
-		req->rq_received = req->rq_rcv_buf.len = 0;
+		req->rq_reply_bytes_recvd = req->rq_rcv_buf.len = 0;
 		if (task->tk_client->cl_discrtry)
 			xprt_conditional_disconnect(task->tk_xprt,
 					req->rq_connect_cookie);
diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index a0e3d97..c1517e2 100644
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -156,7 +156,7 @@ void rpc_count_iostats(struct rpc_task *task)
 	op_metrics->om_timeouts += task->tk_timeouts;
 
 	op_metrics->om_bytes_sent += task->tk_bytes_sent;
-	op_metrics->om_bytes_recv += req->rq_received;
+	op_metrics->om_bytes_recv += req->rq_reply_bytes_recvd;
 
 	queue = (long)req->rq_xtime - task->tk_start;
 	if (queue < 0)
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index c73b31f..463d810 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -808,9 +808,10 @@ void xprt_complete_rqst(struct rpc_task *task, int copied)
 
 	list_del_init(&req->rq_list);
 	req->rq_private_buf.len = copied;
-	/* Ensure all writes are done before we update req->rq_received */
+	/* Ensure all writes are done before we update */
+	/* req->rq_reply_bytes_recvd */
 	smp_wmb();
-	req->rq_received = copied;
+	req->rq_reply_bytes_recvd = copied;
 	rpc_wake_up_queued_task(&xprt->pending, task);
 }
 EXPORT_SYMBOL_GPL(xprt_complete_rqst);
@@ -825,7 +826,7 @@ static void xprt_timer(struct rpc_task *task)
 	dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
 
 	spin_lock_bh(&xprt->transport_lock);
-	if (!req->rq_received) {
+	if (!req->rq_reply_bytes_recvd) {
 		if (xprt->ops->timer)
 			xprt->ops->timer(task);
 	} else
@@ -847,8 +848,8 @@ int xprt_prepare_transmit(struct rpc_task *task)
 	dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
 
 	spin_lock_bh(&xprt->transport_lock);
-	if (req->rq_received && !req->rq_bytes_sent) {
-		err = req->rq_received;
+	if (req->rq_reply_bytes_recvd && !req->rq_bytes_sent) {
+		err = req->rq_reply_bytes_recvd;
 		goto out_unlock;
 	}
 	if (!xprt->ops->reserve_xprt(task))
@@ -877,7 +878,7 @@ void xprt_transmit(struct rpc_task *task)
 
 	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
 
-	if (!req->rq_received) {
+	if (!req->rq_reply_bytes_recvd) {
 		if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
 			/*
 			 * Add to the list only if we're expecting a reply
@@ -916,7 +917,7 @@ void xprt_transmit(struct rpc_task *task)
 	/* Don't race with disconnect */
 	if (!xprt_connected(xprt))
 		task->tk_status = -ENOTCONN;
-	else if (!req->rq_received && rpc_reply_expected(task)) {
+	else if (!req->rq_reply_bytes_recvd && rpc_reply_expected(task)) {
 		/*
 		 * Sleep on the pending queue since
 		 * we're expecting a reply.
-- 
1.6.2.1

[RFC 34/39] nfs41: Backchannel: update cb_sequence args and results

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Change the type of cs_addr and csr_status to 'struct sockaddr' and
'__be32' since the cb_sequence processing function will use existing
functionality that expects these types.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback.h     |    4 ++--
 fs/nfs/callback_xdr.c |    2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
index 0316a06..4bd0daf 100644
--- a/fs/nfs/callback.h
+++ b/fs/nfs/callback.h
@@ -84,7 +84,7 @@ struct referring_call_list {
 };
 
 struct cb_sequenceargs {
-	struct sockaddr_in		*csa_addr;
+	struct sockaddr			*csa_addr;
 	struct nfs4_sessionid		csa_sessionid;
 	uint32_t			csa_sequenceid;
 	uint32_t			csa_slotid;
@@ -95,7 +95,7 @@ struct cb_sequenceargs {
 };
 
 struct cb_sequenceres {
-	uint32_t			csr_status;
+	__be32				csr_status;
 	struct nfs4_sessionid		csr_sessionid;
 	uint32_t			csr_sequenceid;
 	uint32_t			csr_slotid;
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 17cf089..27ac457 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -291,7 +291,7 @@ static unsigned decode_cb_sequence_args(struct svc_rqst *rqstp,
 	if (unlikely(p == NULL))
 		goto out;
 
-	args->csa_addr = svc_addr_in(rqstp);
+	args->csa_addr = svc_addr(rqstp);
 	args->csa_sequenceid = ntohl(*p++);
 	args->csa_slotid = ntohl(*p++);
 	args->csa_highestslotid = ntohl(*p++);
-- 
1.6.2.1

[RFC 35/39] nfs41: Backchannel: Refactor nfs4_reset_slot_table()

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Generalize nfs4_reset_slot_table() so it can be used to reset the
backchannel slot table in addition to the forechannel slot table.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c |   38 +++++++++++++++++++++++++++-----------
 1 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 485c279..7b18787 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4373,19 +4373,21 @@ int nfs4_proc_get_lease_time(struct nfs_client *clp, struct nfs_fsinfo *fsinfo)
 	return status;
 }
 
-/* Reset a slot table */
-static int nfs4_reset_slot_table(struct nfs4_session *session)
+/*
+ * Reset a slot table
+ */
+static int nfs4_reset_slot_table(struct nfs4_slot_table *tbl, int max_slots,
+		int old_max_slots, int ivalue)
 {
-	struct nfs4_slot_table *tbl = &session->fc_slot_table;
-	int i, max_slots = session->fc_attrs.max_reqs;
-	int old_max_slots = session->fc_slot_table.max_slots;
+	int i;
 	int ret = 0;
 
-	dprintk("--> %s: max_reqs=%u, tbl %p\n", __func__,
-		session->fc_attrs.max_reqs, tbl);
+	dprintk("--> %s: max_reqs=%u, tbl %p\n", __func__, max_slots, tbl);
 
-	/* Until we have dynamic slot table adjustment, insist
-	 * upon the same slot table size */
+	/*
+	 * Until we have dynamic slot table adjustment, insist
+	 * upon the same slot table size
+	 */
 	if (max_slots != old_max_slots) {
 		dprintk("%s reset slot table does't match old\n",
 			__func__);
@@ -4394,7 +4396,7 @@ static int nfs4_reset_slot_table(struct nfs4_session *session)
 	}
 	spin_lock(&tbl->slot_tbl_lock);
 	for (i = 0; i < max_slots; ++i)
-		tbl->slots[i].seq_nr = 1;
+		tbl->slots[i].seq_nr = ivalue;
 	tbl->highest_used_slotid = -1;
 	spin_unlock(&tbl->slot_tbl_lock);
 	dprintk("%s: tbl=%p slots=%p max_slots=%d\n", __func__,
@@ -4405,6 +4407,20 @@ out:
 }
 
 /*
+ * Reset the forechannel and backchannel slot tables
+ */
+static int nfs4_reset_slot_tables(struct nfs4_session *session)
+{
+	int status;
+
+	status = nfs4_reset_slot_table(&session->fc_slot_table,
+			session->fc_attrs.max_reqs,
+			session->fc_slot_table.max_slots,
+			1);
+	return status;
+}
+
+/*
  * Initialize slot table
  */
 static int nfs4_init_slot_table(struct nfs4_session *session)
@@ -4642,7 +4658,7 @@ int nfs4_proc_create_session(struct nfs_client *clp, int reset)
 
 	/* Init or reset the fore channel */
 	if (reset)
-		status = nfs4_reset_slot_table(session);
+		status = nfs4_reset_slot_tables(session);
 	else
 		status = nfs4_init_slot_table(session);
 	dprintk("fore channel slot table initialization returned %d\n", status);
-- 
1.6.2.1

[RFC 36/39] nfs41: Backchannel: Refactor nfs4_init_slot_table()

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Generalize nfs4_init_slot_table() so it can be used to initialize the
backchannel slot table in addition to the forechannel slot table.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c |   26 +++++++++++++++++++-------
 1 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 7b18787..da544b8 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4423,23 +4423,22 @@ static int nfs4_reset_slot_tables(struct nfs4_session *session)
 /*
  * Initialize slot table
  */
-static int nfs4_init_slot_table(struct nfs4_session *session)
+static int nfs4_init_slot_table(struct nfs4_slot_table *tbl,
+		int max_slots, int ivalue)
 {
-	struct nfs4_slot_table *tbl = &session->fc_slot_table;
-	int i, max_slots = session->fc_attrs.max_reqs;
+	int i;
 	struct nfs4_slot *slot;
 	int ret = -ENOMEM;
 
 	BUG_ON(max_slots > NFS4_MAX_SLOT_TABLE);
 
-	dprintk("--> %s: max_reqs=%u\n", __func__,
-		session->fc_attrs.max_reqs);
+	dprintk("--> %s: max_reqs=%u\n", __func__, max_slots);
 
 	slot = kcalloc(max_slots, sizeof(struct nfs4_slot), GFP_KERNEL);
 	if (!slot)
 		goto out;
 	for (i = 0; i < max_slots; ++i)
-		slot[i].seq_nr = 1;
+		slot[i].seq_nr = ivalue;
 	ret = 0;
 
 	spin_lock(&tbl->slot_tbl_lock);
@@ -4459,11 +4458,24 @@ static int nfs4_init_slot_table(struct nfs4_session *session)
 out:
 	dprintk("<-- %s: return %d\n", __func__, ret);
 	return ret;
+
 out_free:
 	kfree(slot);
 	goto out;
 }
 
+/*
+ * Initialize the forechannel and backchannel tables
+ */
+static int nfs4_init_slot_tables(struct nfs4_session *session)
+{
+	int status;
+
+	status = nfs4_init_slot_table(&session->fc_slot_table,
+			session->fc_attrs.max_reqs, 1);
+	return status;
+}
+
 /* Destroy the slot table */
 static void nfs4_destroy_slot_table(struct nfs4_session *session)
 {
@@ -4660,7 +4672,7 @@ int nfs4_proc_create_session(struct nfs_client *clp, int reset)
 	if (reset)
 		status = nfs4_reset_slot_tables(session);
 	else
-		status = nfs4_init_slot_table(session);
+		status = nfs4_init_slot_tables(session);
 	dprintk("fore channel slot table initialization returned %d\n", status);
 	if (status)
 		goto out;
-- 
1.6.2.1

[RFC 37/39] nfs41: Backchannel: Add a backchannel slot table to the session

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Defines a new 'struct nfs4_slot_table' in the 'struct nfs4_session'
for use by the backchannel.  Initializes, resets, and destroys the backchannel
slot table in the same manner the forechannel slot table is initialized,
reset, and destroyed.

The sequenceid for each slot in the backchannel slot table is initialized
to 0, whereas the forechannel slotid's sequenceid is set to 1.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c         |   48 +++++++++++++++++++++++++++++++++-----------
 include/linux/nfs_fs_sb.h |    2 +-
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index da544b8..07b5f19 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -4417,9 +4417,30 @@ static int nfs4_reset_slot_tables(struct nfs4_session *session)
 			session->fc_attrs.max_reqs,
 			session->fc_slot_table.max_slots,
 			1);
+	if (status)
+		return status;
+
+	status = nfs4_reset_slot_table(&session->bc_slot_table,
+			session->bc_attrs.max_reqs,
+			session->bc_slot_table.max_slots,
+			0);
 	return status;
 }
 
+/* Destroy the slot table */
+static void nfs4_destroy_slot_tables(struct nfs4_session *session)
+{
+	if (session->fc_slot_table.slots != NULL) {
+		kfree(session->fc_slot_table.slots);
+		session->fc_slot_table.slots = NULL;
+	}
+	if (session->bc_slot_table.slots != NULL) {
+		kfree(session->bc_slot_table.slots);
+		session->bc_slot_table.slots = NULL;
+	}
+	return;
+}
+
 /*
  * Initialize slot table
  */
@@ -4473,17 +4494,15 @@ static int nfs4_init_slot_tables(struct nfs4_session *session)
 
 	status = nfs4_init_slot_table(&session->fc_slot_table,
 			session->fc_attrs.max_reqs, 1);
-	return status;
-}
+	if (status)
+		return status;
 
-/* Destroy the slot table */
-static void nfs4_destroy_slot_table(struct nfs4_session *session)
-{
-	if (session->fc_slot_table.slots == NULL)
-		return;
-	kfree(session->fc_slot_table.slots);
-	session->fc_slot_table.slots = NULL;
-	return;
+	status = nfs4_init_slot_table(&session->bc_slot_table,
+			session->bc_attrs.max_reqs, 0);
+	if (status)
+		nfs4_destroy_slot_tables(session);
+
+	return status;
 }
 
 struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
@@ -4506,7 +4525,12 @@ struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp)
 
 	tbl = &session->fc_slot_table;
 	spin_lock_init(&tbl->slot_tbl_lock);
-	rpc_init_wait_queue(&tbl->slot_tbl_waitq, "Slot table");
+	rpc_init_wait_queue(&tbl->slot_tbl_waitq, "ForeChannel Slot table");
+
+	tbl = &session->bc_slot_table;
+	spin_lock_init(&tbl->slot_tbl_lock);
+	rpc_init_wait_queue(&tbl->slot_tbl_waitq, "BackChannel Slot table");
+
 	session->clp = clp;
 	return session;
 }
@@ -4518,7 +4542,7 @@ void nfs4_destroy_session(struct nfs4_session *session)
 		__func__, session->clp->cl_rpcclient->cl_xprt);
 	xprt_destroy_backchannel(session->clp->cl_rpcclient->cl_xprt,
 				NFS41_BC_MIN_CALLBACKS);
-	nfs4_destroy_slot_table(session);
+	nfs4_destroy_slot_tables(session);
 	kfree(session);
 }
 
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index d0902cc..19fe15d 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -205,7 +205,7 @@ struct nfs4_session {
 	struct nfs4_channel_attrs	fc_attrs;
 	struct nfs4_slot_table		fc_slot_table;
 	struct nfs4_channel_attrs	bc_attrs;
-					/* back channel has one slot */
+	struct nfs4_slot_table		bc_slot_table;
 	struct nfs_client		*clp;
 };
 
-- 
1.6.2.1

[RFC 38/39] nfs41: Backchannel: New find_client_with_session()

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Finds the 'struct nfs_client' that matches the server's address, major
version number, and session ID.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_proc.c |   36 ++++++++++++++++++++++++++++++++++++
 1 files changed, 36 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index f731bbe..6b342e8 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -104,6 +104,42 @@ out:
 
 #if defined(CONFIG_NFS_V4_1)
 
+/*
+ * Returns a pointer to a held 'struct nfs_client' that matches the server's
+ * address, major version number, and session ID.  It is the caller's
+ * responsibility to release the returned reference.
+ *
+ * Returns NULL if there are no connections with sessions, or if no session
+ * matches the one of interest.
+ */
+ static struct nfs_client *find_client_with_session(
+	const struct sockaddr *addr, u32 nfsversion,
+	struct nfs4_sessionid *sessionid)
+{
+	struct nfs_client *clp;
+
+	clp = nfs_find_client(addr, 4);
+	if (clp == NULL)
+		return NULL;
+
+	do {
+		struct nfs_client *prev = clp;
+
+		if (clp->cl_session != NULL) {
+			if (memcmp(clp->cl_session->sess_id.data,
+					sessionid->data,
+					NFS4_MAX_SESSIONID_LEN) == 0) {
+				/* Returns a held reference to clp */
+				return clp;
+			}
+		}
+		clp = nfs_find_client_next(prev);
+		nfs_put_client(prev);
+	} while (clp != NULL);
+
+	return NULL;
+}
+
 /* FIXME: validate args->cbs_{sequence,slot}id */
 /* FIXME: referring calls should be processed */
 unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
-- 
1.6.2.1

[RFC 39/39] nfs41: Backchannel: CB_SEQUENCE validation

[Benny Halevy]

From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>

Validates the callback's sessionID, the slot number, and the sequence ID.
Increments the slot's sequence.

Detects replays, but simply prints a debug message (if debugging is enabled
since we don't yet implement a duplicate request cache for the backchannel.
This should not present a problem, since only idempotent callbacks are
currently implemented.

Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_proc.c |   75 ++++++++++++++++++++++++++++++++++++++++++++----
 1 files changed, 69 insertions(+), 6 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 6b342e8..c85d66f 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -105,6 +105,57 @@ out:
 #if defined(CONFIG_NFS_V4_1)
 
 /*
+ * Validate the sequenceID sent by the server.
+ * Return success if the sequenceID is one more than what we last saw on
+ * this slot, accounting for wraparound.  Increments the slot's sequence.
+ *
+ * We don't yet implement a duplicate request cache, so at this time
+ * we will log replays, and process them as if we had not seen them before,
+ * but we don't bump the sequence in the slot.  Not too worried about it,
+ * since we only currently implement idempotent callbacks anyway.
+ *
+ * We have a single slot backchannel at this time, so we don't bother
+ * checking the used_slots bit array on the table.  The lower layer guarantees
+ * a single outstanding callback request at a time.
+ */
+static int
+validate_seqid(struct nfs4_slot_table *tbl, u32 slotid, u32 seqid)
+{
+	struct nfs4_slot *slot;
+
+	dprintk("%s enter. slotid %d seqid %d\n",
+		__func__, slotid, seqid);
+
+	if (slotid > NFS41_BC_MAX_CALLBACKS)
+		return NFS4ERR_BADSLOT;
+
+	slot = tbl->slots + slotid;
+	dprintk("%s slot table seqid: %d\n", __func__, slot->seq_nr);
+
+	/* Normal */
+	if (likely(seqid == slot->seq_nr + 1)) {
+		slot->seq_nr++;
+		return NFS4_OK;
+	}
+
+	/* Replay */
+	if (seqid == slot->seq_nr) {
+		dprintk("%s seqid %d is a replay - no DRC available\n",
+			__func__, seqid);
+		return NFS4_OK;
+	}
+
+	/* Wraparound */
+	if (seqid == 1 && (slot->seq_nr + 1) == 0) {
+		slot->seq_nr = 1;
+		return NFS4_OK;
+	}
+
+	/* Misordered request */
+	return NFS4ERR_SEQ_MISORDERED;
+}
+
+/*
  * Returns a pointer to a held 'struct nfs_client' that matches the server's
  * address, major version number, and session ID.  It is the caller's
  * responsibility to release the returned reference.
@@ -140,18 +191,27 @@ out:
 	return NULL;
 }
 
-/* FIXME: validate args->cbs_{sequence,slot}id */
 /* FIXME: referring calls should be processed */
 unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
 				struct cb_sequenceres *res)
 {
-	int i;
-	unsigned status = 0;
+	struct nfs_client *clp;
+	int i, status;
 
 	for (i = 0; i < args->csa_nrclists; i++)
 		kfree(args->csa_rclists[i].rcl_refcalls);
 	kfree(args->csa_rclists);
 
+	status = NFS4ERR_BADSESSION;
+	clp = find_client_with_session(args->csa_addr, 4, &args->csa_sessionid);
+	if (clp == NULL)
+		goto out;
+
+	status = validate_seqid(&clp->cl_session->bc_slot_table,
+				args->csa_slotid, args->csa_sequenceid);
+	if (status)
+		goto out_putclient;
+
 	memcpy(&res->csr_sessionid, &args->csa_sessionid,
 	       sizeof(res->csr_sessionid));
 	res->csr_sequenceid = args->csa_sequenceid;
@@ -159,9 +219,12 @@ unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
 	res->csr_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
 	res->csr_target_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
 
-	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
-	res->csr_status = status;
-	return status;
+out_putclient:
+	nfs_put_client(clp);
+out:
+	dprintk("%s: exit with status = %d\n", __func__, status);
+	res->csr_status = htonl(status);
+	return res->csr_status;
 }
 
 #endif /* CONFIG_NFS_V4_1 */
-- 
1.6.2.1

Re: [pnfs] [RFC 02/39] nfs41: Skip past the RPC call direction

[Trond Myklebust]

On Fri, 2009-05-01 at 02:19 +0300, Benny Halevy wrote:
> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> 
> xs_tcp_read_calldir() is now responsible for reading the RPC call
> direction and determining whether it is a reply or a callback request.
> The call to xdr_skb_read_bits() inside xs_tcp_read_calldir() moves the
> xdr_skb_reader offset past the RPC call direction (offset should be
> equal to 8).  Therefore xs_tcp_read_common() called from
> xs_tcp_ready_reply() should be copying the TCP buffer starting past the
> RPC call direction.  It is now necessary to read the RPC call direction
> earlier to determine whether to call the reply handler or the callback
> handler.
> 
> call_verify() should therefore skip past the XID and call/reply flag.
> 
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  net/sunrpc/clnt.c |   13 +++++++------
>  1 files changed, 7 insertions(+), 6 deletions(-)
> 
> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> index d00e813..76d7d46 100644
> --- a/net/sunrpc/clnt.c
> +++ b/net/sunrpc/clnt.c
> @@ -1390,13 +1390,14 @@ rpc_verify_header(struct rpc_task *task)
>  	}
>  	if ((len -= 3) < 0)
>  		goto out_overflow;
> -	p += 1;	/* skip XID */
>  
> -	if ((n = ntohl(*p++)) != RPC_REPLY) {
> -		dprintk("RPC: %5u %s: not an RPC reply: %x\n",
> -				task->tk_pid, __func__, n);
> -		goto out_garbage;
> -	}
> +	/*
> +	 * Skip the XID and call direction.
> +	 * The underlying transport has read the XID and RPC call direction
> +	 * to determine this is an RPC reply.
> +	 */
> +	p += 2;
> +
>  	if ((n = ntohl(*p++)) != RPC_MSG_ACCEPTED) {
>  		if (--len < 0)
>  			goto out_overflow;

Err... No... The above comment is true only for the case of TCP. Why
don't you simply copy the RPC call direction into the buffer?

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

RE: [pnfs] [RFC 02/39] nfs41: Skip past the RPC call direction

[Labiaga, Ricardo]

> -----Original Message-----
> From: Myklebust, Trond
> Sent: Wednesday, June 03, 2009 2:30 PM
> To: Benny Halevy
> Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 02/39] nfs41: Skip past the RPC call
direction
> 
> On Fri, 2009-05-01 at 02:19 +0300, Benny Halevy wrote:
> > From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> >
> > xs_tcp_read_calldir() is now responsible for reading the RPC call
> > direction and determining whether it is a reply or a callback
request.
> > The call to xdr_skb_read_bits() inside xs_tcp_read_calldir() moves
the
> > xdr_skb_reader offset past the RPC call direction (offset should be
> > equal to 8).  Therefore xs_tcp_read_common() called from
> > xs_tcp_ready_reply() should be copying the TCP buffer starting past
the
> > RPC call direction.  It is now necessary to read the RPC call
direction
> > earlier to determine whether to call the reply handler or the
callback
> > handler.
> >
> > call_verify() should therefore skip past the XID and call/reply
flag.
> >
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  net/sunrpc/clnt.c |   13 +++++++------
> >  1 files changed, 7 insertions(+), 6 deletions(-)
> >
> > diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> > index d00e813..76d7d46 100644
> > --- a/net/sunrpc/clnt.c
> > +++ b/net/sunrpc/clnt.c
> > @@ -1390,13 +1390,14 @@ rpc_verify_header(struct rpc_task *task)
> >  	}
> >  	if ((len -= 3) < 0)
> >  		goto out_overflow;
> > -	p += 1;	/* skip XID */
> >
> > -	if ((n = ntohl(*p++)) != RPC_REPLY) {
> > -		dprintk("RPC: %5u %s: not an RPC reply: %x\n",
> > -				task->tk_pid, __func__, n);
> > -		goto out_garbage;
> > -	}
> > +	/*
> > +	 * Skip the XID and call direction.
> > +	 * The underlying transport has read the XID and RPC call
direction
> > +	 * to determine this is an RPC reply.
> > +	 */
> > +	p += 2;
> > +
> >  	if ((n = ntohl(*p++)) != RPC_MSG_ACCEPTED) {
> >  		if (--len < 0)
> >  			goto out_overflow;
> 
> Err... No... The above comment is true only for the case of TCP. Why
> don't you simply copy the RPC call direction into the buffer?

You're right.  We'll follow your suggestion to copy it into the buffer
and leave the processing as it originally was in rpc_verify_header().

- ricardo


> --
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs"
in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [pnfs] [RFC 04/39] nfs41: minorversion support for nfs4_{init, destroy}_callback

[Trond Myklebust]

On Fri, 2009-05-01 at 02:19 +0300, Benny Halevy wrote:
> move nfs4_init_callback into nfs4_init_client_minor_version
> and nfs4_destroy_callback into nfs4_clear_client_minor_version
> 
> as these need to happen also when auto-negotiating the minorversion
> once the callback service for nfs41 becomes different than for nfs4.0
> 
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> [nfs41: Fix checkpatch warning]
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback.c |   67 ++++++++++++++++++++++++++++++++++++++---------------
>  fs/nfs/callback.h |    2 +-
>  fs/nfs/client.c   |   21 +++++-----------
>  3 files changed, 56 insertions(+), 34 deletions(-)
> 
> diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
> index a886e69..3926046 100644
> --- a/fs/nfs/callback.c
> +++ b/fs/nfs/callback.c
> @@ -59,7 +59,7 @@ module_param_call(callback_tcpport, param_set_port, param_get_int,
>   * This is the callback kernel thread.
>   */
>  static int
> -nfs_callback_svc(void *vrqstp)
> +nfs4_callback_svc(void *vrqstp)
>  {
>  	int err, preverr = 0;
>  	struct svc_rqst *rqstp = vrqstp;
> @@ -97,20 +97,12 @@ nfs_callback_svc(void *vrqstp)
>  }
>  
>  /*
> - * Bring up the callback thread if it is not already up.
> + * Prepare to bring up the NFSv4 callback service
>   */
> -int nfs_callback_up(void)
> +struct svc_rqst *
> +nfs4_callback_up(struct svc_serv *serv)
>  {
> -	struct svc_serv *serv = NULL;
> -	int ret = 0;
> -
> -	mutex_lock(&nfs_callback_mutex);
> -	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> -		goto out;
> -	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
> -	ret = -ENOMEM;
> -	if (!serv)
> -		goto out_err;
> +	int ret;
>  
>  	ret = svc_create_xprt(serv, "tcp", PF_INET,
>  				nfs_callback_set_tcpport, SVC_SOCK_ANONYMOUS);
> @@ -131,18 +123,55 @@ int nfs_callback_up(void)
>  		goto out_err;
>  #endif	/* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE) */
>  
> -	nfs_callback_info.rqst = svc_prepare_thread(serv, &serv->sv_pools[0]);
> -	if (IS_ERR(nfs_callback_info.rqst)) {
> -		ret = PTR_ERR(nfs_callback_info.rqst);
> -		nfs_callback_info.rqst = NULL;
> +	return svc_prepare_thread(serv, &serv->sv_pools[0]);
> +
> +out_err:
> +	if (ret == 0)
> +		ret = -ENOMEM;
> +	return ERR_PTR(ret);
> +}
> +
> +/*
> + * Bring up the callback thread if it is not already up.
> + */
> +int nfs_callback_up(u32 minorversion, void *args)
                                         ^^^^^^^^^^
There is no reason not to type check the arguments here. Please fix...

> +{
> +	struct svc_serv *serv = NULL;
> +	struct svc_rqst *rqstp;
> +	int (*callback_svc)(void *vrqstp);
> +	char svc_name[12];
> +	int ret = 0;
> +
> +	mutex_lock(&nfs_callback_mutex);
> +	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> +		goto out;
> +	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
> +	if (!serv) {
> +		ret = -ENOMEM;
> +		goto out_err;
> +	}
> +
> +	/* FIXME: either 4.0 or 4.1 callback service can be up at a time
> +	 * need to monitor and control them both */
> +	if (!minorversion) {
> +		rqstp = nfs4_callback_up(serv);
> +		callback_svc = nfs4_callback_svc;
> +	} else {
> +		BUG();	/* for now */
> +	}
> +
> +	if (IS_ERR(rqstp)) {
> +		ret = PTR_ERR(rqstp);
>  		goto out_err;
>  	}
>  
>  	svc_sock_update_bufs(serv);
>  
> -	nfs_callback_info.task = kthread_run(nfs_callback_svc,
> +	sprintf(svc_name, "nfsv4.%u-svc", minorversion);
> +	nfs_callback_info.rqst = rqstp;
> +	nfs_callback_info.task = kthread_run(callback_svc,
>  					     nfs_callback_info.rqst,
> -					     "nfsv4-svc");
> +					     svc_name);
>  	if (IS_ERR(nfs_callback_info.task)) {
>  		ret = PTR_ERR(nfs_callback_info.task);
>  		svc_exit_thread(nfs_callback_info.rqst);
> diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> index e110e28..7f12e65 100644
> --- a/fs/nfs/callback.h
> +++ b/fs/nfs/callback.h
> @@ -63,7 +63,7 @@ extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getat
>  extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void *dummy);
>  
>  #ifdef CONFIG_NFS_V4
> -extern int nfs_callback_up(void);
> +extern int nfs_callback_up(u32 minorversion, void *args);
>  extern void nfs_callback_down(void);
>  #else
>  #define nfs_callback_up()	(0)
> diff --git a/fs/nfs/client.c b/fs/nfs/client.c
> index d9657d4..df2b40d 100644
> --- a/fs/nfs/client.c
> +++ b/fs/nfs/client.c
> @@ -47,9 +47,6 @@
>  #include "internal.h"
>  #include "fscache.h"
>  
> -static int nfs4_init_callback(struct nfs_client *);
> -static void nfs4_destroy_callback(struct nfs_client *);
> -
>  #define NFSDBG_FACILITY		NFSDBG_CLIENT
>  
>  static DEFINE_SPINLOCK(nfs_client_lock);
> @@ -124,9 +121,6 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
>  
>  	clp->rpc_ops = cl_init->rpc_ops;
>  
> -	if (nfs4_init_callback(clp) < 0)
> -		goto error_2;
> -
>  	atomic_set(&clp->cl_count, 1);
>  	clp->cl_cons_state = NFS_CS_INITING;
>  
> @@ -136,7 +130,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
>  	if (cl_init->hostname) {
>  		clp->cl_hostname = kstrdup(cl_init->hostname, GFP_KERNEL);
>  		if (!clp->cl_hostname)
> -			goto error_3;
> +			goto error_cleanup;
>  	}
>  
>  	INIT_LIST_HEAD(&clp->cl_superblocks);
> @@ -161,9 +155,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
>  
>  	return clp;
>  
> -error_3:
> -	nfs4_destroy_callback(clp);
> -error_2:
> +error_cleanup:
>  	kfree(clp);
>  error_0:
>  	return NULL;
> @@ -207,6 +199,8 @@ static void nfs4_clear_client_minor_version(struct nfs_client *clp)
>  
>  	clp->cl_call_sync = _nfs4_call_sync;
>  #endif /* CONFIG_NFS_V4_1 */
> +
> +	nfs4_destroy_callback(clp);
>  }
>  
>  /*
> @@ -225,8 +219,6 @@ static void nfs_free_client(struct nfs_client *clp)
>  	if (!IS_ERR(clp->cl_rpcclient))
>  		rpc_shutdown_client(clp->cl_rpcclient);
>  
> -	nfs4_destroy_callback(clp);
> -
>  	if (clp->cl_machine_cred != NULL)
>  		put_rpccred(clp->cl_machine_cred);
>  
> @@ -1104,7 +1096,8 @@ static int nfs4_init_callback(struct nfs_client *clp)
>  	int error;
>  
>  	if (clp->rpc_ops->version == 4) {
> -		error = nfs_callback_up();
> +		error = nfs_callback_up(clp->cl_minorversion,
> +					clp->cl_rpcclient->cl_xprt);
>  		if (error < 0) {
>  			dprintk("%s: failed to start callback. Error = %d\n",
>  				__func__, error);
> @@ -1139,7 +1132,7 @@ static int nfs4_init_client_minor_version(struct nfs_client *clp)
>  	}
>  #endif /* CONFIG_NFS_V4_1 */
>  
> -	return 0;
> +	return nfs4_init_callback(clp);
>  }
>  
>  /*



-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

RE: [pnfs] [RFC 04/39] nfs41: minorversion support for nfs4_{init, destroy}_callback

[Labiaga, Ricardo]

> -----Original Message-----
> From: Myklebust, Trond
> Sent: Wednesday, June 03, 2009 7:40 PM
> To: Benny Halevy
> Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 04/39] nfs41: minorversion support for
> nfs4_{init, destroy}_callback
> 
> On Fri, 2009-05-01 at 02:19 +0300, Benny Halevy wrote:
> > move nfs4_init_callback into nfs4_init_client_minor_version
> > and nfs4_destroy_callback into nfs4_clear_client_minor_version
> >
> > as these need to happen also when auto-negotiating the minorversion
> > once the callback service for nfs41 becomes different than for
nfs4.0
> >
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > [nfs41: Fix checkpatch warning]
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  fs/nfs/callback.c |   67
++++++++++++++++++++++++++++++++++++++--------
> -------
> >  fs/nfs/callback.h |    2 +-
> >  fs/nfs/client.c   |   21 +++++-----------
> >  3 files changed, 56 insertions(+), 34 deletions(-)
> >
> > diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
> > index a886e69..3926046 100644
> > --- a/fs/nfs/callback.c
> > +++ b/fs/nfs/callback.c
> > @@ -59,7 +59,7 @@ module_param_call(callback_tcpport,
param_set_port,
> param_get_int,
> >   * This is the callback kernel thread.
> >   */
> >  static int
> > -nfs_callback_svc(void *vrqstp)
> > +nfs4_callback_svc(void *vrqstp)
> >  {
> >  	int err, preverr = 0;
> >  	struct svc_rqst *rqstp = vrqstp;
> > @@ -97,20 +97,12 @@ nfs_callback_svc(void *vrqstp)
> >  }
> >
> >  /*
> > - * Bring up the callback thread if it is not already up.
> > + * Prepare to bring up the NFSv4 callback service
> >   */
> > -int nfs_callback_up(void)
> > +struct svc_rqst *
> > +nfs4_callback_up(struct svc_serv *serv)
> >  {
> > -	struct svc_serv *serv = NULL;
> > -	int ret = 0;
> > -
> > -	mutex_lock(&nfs_callback_mutex);
> > -	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> > -		goto out;
> > -	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE,
> NULL);
> > -	ret = -ENOMEM;
> > -	if (!serv)
> > -		goto out_err;
> > +	int ret;
> >
> >  	ret = svc_create_xprt(serv, "tcp", PF_INET,
> >  				nfs_callback_set_tcpport,
SVC_SOCK_ANONYMOUS);
> > @@ -131,18 +123,55 @@ int nfs_callback_up(void)
> >  		goto out_err;
> >  #endif	/* defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
*/
> >
> > -	nfs_callback_info.rqst = svc_prepare_thread(serv, &serv-
> >sv_pools[0]);
> > -	if (IS_ERR(nfs_callback_info.rqst)) {
> > -		ret = PTR_ERR(nfs_callback_info.rqst);
> > -		nfs_callback_info.rqst = NULL;
> > +	return svc_prepare_thread(serv, &serv->sv_pools[0]);
> > +
> > +out_err:
> > +	if (ret == 0)
> > +		ret = -ENOMEM;
> > +	return ERR_PTR(ret);
> > +}
> > +
> > +/*
> > + * Bring up the callback thread if it is not already up.
> > + */
> > +int nfs_callback_up(u32 minorversion, void *args)
>                                          ^^^^^^^^^^
> There is no reason not to type check the arguments here. Please fix...

Will do.

- ricardo

> > +{
> > +	struct svc_serv *serv = NULL;
> > +	struct svc_rqst *rqstp;
> > +	int (*callback_svc)(void *vrqstp);
> > +	char svc_name[12];
> > +	int ret = 0;
> > +
> > +	mutex_lock(&nfs_callback_mutex);
> > +	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> > +		goto out;
> > +	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE,
> NULL);
> > +	if (!serv) {
> > +		ret = -ENOMEM;
> > +		goto out_err;
> > +	}
> > +
> > +	/* FIXME: either 4.0 or 4.1 callback service can be up at a time
> > +	 * need to monitor and control them both */
> > +	if (!minorversion) {
> > +		rqstp = nfs4_callback_up(serv);
> > +		callback_svc = nfs4_callback_svc;
> > +	} else {
> > +		BUG();	/* for now */
> > +	}
> > +
> > +	if (IS_ERR(rqstp)) {
> > +		ret = PTR_ERR(rqstp);
> >  		goto out_err;
> >  	}
> >
> >  	svc_sock_update_bufs(serv);
> >
> > -	nfs_callback_info.task = kthread_run(nfs_callback_svc,
> > +	sprintf(svc_name, "nfsv4.%u-svc", minorversion);
> > +	nfs_callback_info.rqst = rqstp;
> > +	nfs_callback_info.task = kthread_run(callback_svc,
> >  					     nfs_callback_info.rqst,
> > -					     "nfsv4-svc");
> > +					     svc_name);
> >  	if (IS_ERR(nfs_callback_info.task)) {
> >  		ret = PTR_ERR(nfs_callback_info.task);
> >  		svc_exit_thread(nfs_callback_info.rqst);
> > diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> > index e110e28..7f12e65 100644
> > --- a/fs/nfs/callback.h
> > +++ b/fs/nfs/callback.h
> > @@ -63,7 +63,7 @@ extern __be32 nfs4_callback_getattr(struct
> cb_getattrargs *args, struct cb_getat
> >  extern __be32 nfs4_callback_recall(struct cb_recallargs *args, void
> *dummy);
> >
> >  #ifdef CONFIG_NFS_V4
> > -extern int nfs_callback_up(void);
> > +extern int nfs_callback_up(u32 minorversion, void *args);
> >  extern void nfs_callback_down(void);
> >  #else
> >  #define nfs_callback_up()	(0)
> > diff --git a/fs/nfs/client.c b/fs/nfs/client.c
> > index d9657d4..df2b40d 100644
> > --- a/fs/nfs/client.c
> > +++ b/fs/nfs/client.c
> > @@ -47,9 +47,6 @@
> >  #include "internal.h"
> >  #include "fscache.h"
> >
> > -static int nfs4_init_callback(struct nfs_client *);
> > -static void nfs4_destroy_callback(struct nfs_client *);
> > -
> >  #define NFSDBG_FACILITY		NFSDBG_CLIENT
> >
> >  static DEFINE_SPINLOCK(nfs_client_lock);
> > @@ -124,9 +121,6 @@ static struct nfs_client *nfs_alloc_client(const
> struct nfs_client_initdata *cl_
> >
> >  	clp->rpc_ops = cl_init->rpc_ops;
> >
> > -	if (nfs4_init_callback(clp) < 0)
> > -		goto error_2;
> > -
> >  	atomic_set(&clp->cl_count, 1);
> >  	clp->cl_cons_state = NFS_CS_INITING;
> >
> > @@ -136,7 +130,7 @@ static struct nfs_client *nfs_alloc_client(const
> struct nfs_client_initdata *cl_
> >  	if (cl_init->hostname) {
> >  		clp->cl_hostname = kstrdup(cl_init->hostname,
GFP_KERNEL);
> >  		if (!clp->cl_hostname)
> > -			goto error_3;
> > +			goto error_cleanup;
> >  	}
> >
> >  	INIT_LIST_HEAD(&clp->cl_superblocks);
> > @@ -161,9 +155,7 @@ static struct nfs_client *nfs_alloc_client(const
> struct nfs_client_initdata *cl_
> >
> >  	return clp;
> >
> > -error_3:
> > -	nfs4_destroy_callback(clp);
> > -error_2:
> > +error_cleanup:
> >  	kfree(clp);
> >  error_0:
> >  	return NULL;
> > @@ -207,6 +199,8 @@ static void
nfs4_clear_client_minor_version(struct
> nfs_client *clp)
> >
> >  	clp->cl_call_sync = _nfs4_call_sync;
> >  #endif /* CONFIG_NFS_V4_1 */
> > +
> > +	nfs4_destroy_callback(clp);
> >  }
> >
> >  /*
> > @@ -225,8 +219,6 @@ static void nfs_free_client(struct nfs_client
*clp)
> >  	if (!IS_ERR(clp->cl_rpcclient))
> >  		rpc_shutdown_client(clp->cl_rpcclient);
> >
> > -	nfs4_destroy_callback(clp);
> > -
> >  	if (clp->cl_machine_cred != NULL)
> >  		put_rpccred(clp->cl_machine_cred);
> >
> > @@ -1104,7 +1096,8 @@ static int nfs4_init_callback(struct
nfs_client
> *clp)
> >  	int error;
> >
> >  	if (clp->rpc_ops->version == 4) {
> > -		error = nfs_callback_up();
> > +		error = nfs_callback_up(clp->cl_minorversion,
> > +					clp->cl_rpcclient->cl_xprt);
> >  		if (error < 0) {
> >  			dprintk("%s: failed to start callback. Error =
%d\n",
> >  				__func__, error);
> > @@ -1139,7 +1132,7 @@ static int
nfs4_init_client_minor_version(struct
> nfs_client *clp)
> >  	}
> >  #endif /* CONFIG_NFS_V4_1 */
> >
> > -	return 0;
> > +	return nfs4_init_callback(clp);
> >  }
> >
> >  /*
> 
> 
> 
> --
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

Re: [pnfs] [RFC 10/39] nfs41: Add backchannel processing support to RPC state machine

[Trond Myklebust]

On Fri, 2009-05-01 at 02:20 +0300, Benny Halevy wrote:
> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> 
> Adds rpc_run_bc_task() which is called by the NFS callback service to
> process backchannel requests.  It performs similar work to rpc_run_task()
> though "schedules" the backchannel task to be executed starting at the
> call_trasmit state in the RPC state machine.
> 
> It also introduces some miscellaneous updates to the argument validation,
> call_transmit, and transport cleanup functions to take into account
> that there are now forechannel and backchannel tasks.
> 
> Backchannel requests do not carry an RPC message structure, since the
> payload has already been XDR encoded using the existing NFSv4 callback
> mechanism.
> 
> Introduce a new transmit state for the client to reply on to backchannel
> requests.  This new state simply reserves the transport and issues the
> reply.  In case of a connection related error, disconnects the transport and
> drops the reply.  It requires the forechannel to re-establish the connection
> and the server to retransmit the request, as stated in NFSv4.1 section
> 2.9.2 "Client and Server Transport Behavior".
> 
> Note: There is no need to loop attempting to reserve the transport.  If EAGAIN
> is returned by xprt_prepare_transmit(), return with tk_status == 0,
> setting tk_action to call_bc_transmit.  rpc_execute() will invoke it again
> after the task is taken off the sleep queue.
> 
> [nfs41: rpc_run_bc_task() need not be exported outside RPC module]
> [nfs41: New call_bc_transmit RPC state]
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> [nfs41: Backchannel: No need to loop in call_bc_transmit()]
> Signed-off-by: Andy Adamson <andros@netapp.com>
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  include/linux/sunrpc/sched.h |    2 +
>  include/linux/sunrpc/xprt.h  |   12 ++++
>  net/sunrpc/clnt.c            |  117 +++++++++++++++++++++++++++++++++++++++++-
>  net/sunrpc/stats.c           |    6 ++-
>  net/sunrpc/sunrpc.h          |   35 +++++++++++++
>  net/sunrpc/xprt.c            |   36 +++++++++++--
>  6 files changed, 199 insertions(+), 9 deletions(-)
>  create mode 100644 net/sunrpc/sunrpc.h
> 
> diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
> index 1773768..4010977 100644
> --- a/include/linux/sunrpc/sched.h
> +++ b/include/linux/sunrpc/sched.h
> @@ -210,6 +210,8 @@ struct rpc_wait_queue {
>   */
>  struct rpc_task *rpc_new_task(const struct rpc_task_setup *);
>  struct rpc_task *rpc_run_task(const struct rpc_task_setup *);
> +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> +				const struct rpc_call_ops *ops);
>  void		rpc_put_task(struct rpc_task *);
>  void		rpc_exit_task(struct rpc_task *);
>  void		rpc_release_calldata(const struct rpc_call_ops *, void *);
> diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
> index 6b37724..1531abe 100644
> --- a/include/linux/sunrpc/xprt.h
> +++ b/include/linux/sunrpc/xprt.h
> @@ -215,6 +215,18 @@ struct rpc_xprt {
>  						/* buffer in use */
>  #endif /* CONFIG_NFS_V4_1 */
>  
> +#if defined(CONFIG_NFS_V4_1)
> +static inline int bc_prealloc(struct rpc_rqst *req)
> +{
> +	return test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
> +}
> +#else
> +static inline int bc_prealloc(struct rpc_rqst *req)
> +{
> +	return 0;
> +}
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  struct xprt_create {
>  	int			ident;		/* XPRT_TRANSPORT identifier */
>  	struct sockaddr *	srcaddr;	/* optional local address */
> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> index 76d7d46..349b4d6 100644
> --- a/net/sunrpc/clnt.c
> +++ b/net/sunrpc/clnt.c
> @@ -36,7 +36,9 @@
>  #include <linux/sunrpc/clnt.h>
>  #include <linux/sunrpc/rpc_pipe_fs.h>
>  #include <linux/sunrpc/metrics.h>
> +#include <linux/sunrpc/bc_xprt.h>
>  
> +#include "sunrpc.h"
>  
>  #ifdef RPC_DEBUG
>  # define RPCDBG_FACILITY	RPCDBG_CALL
> @@ -63,6 +65,9 @@ static void	call_decode(struct rpc_task *task);
>  static void	call_bind(struct rpc_task *task);
>  static void	call_bind_status(struct rpc_task *task);
>  static void	call_transmit(struct rpc_task *task);
> +#if defined(CONFIG_NFS_V4_1)
> +static void	call_bc_transmit(struct rpc_task *task);
> +#endif /* CONFIG_NFS_V4_1 */
>  static void	call_status(struct rpc_task *task);
>  static void	call_transmit_status(struct rpc_task *task);
>  static void	call_refresh(struct rpc_task *task);
> @@ -613,6 +618,50 @@ rpc_call_async(struct rpc_clnt *clnt, const struct rpc_message *msg, int flags,
>  }
>  EXPORT_SYMBOL_GPL(rpc_call_async);
>  
> +#if defined(CONFIG_NFS_V4_1)
> +/**
> + * rpc_run_bc_task - Allocate a new RPC task for backchannel use, then run
> + * rpc_execute against it
> + * @ops: RPC call ops
> + */
> +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> +					const struct rpc_call_ops *tk_ops)
> +{
> +	struct rpc_task *task;
> +	struct xdr_buf *xbufp = &req->rq_snd_buf;
> +	struct rpc_task_setup task_setup_data = {
> +		.callback_ops = tk_ops,
> +	};
> +
> +	dprintk("RPC: rpc_run_bc_task req= %p\n", req);
> +	/*
> +	 * Create an rpc_task to send the data
> +	 */
> +	task = rpc_new_task(&task_setup_data);
> +	if (!task) {
> +		xprt_free_bc_request(req);
> +		goto out;
> +	}
> +	task->tk_rqstp = req;
> +
> +	/*
> +	 * Set up the xdr_buf length.
> +	 * This also indicates that the buffer is XDR encoded already.
> +	 */
> +	xbufp->len = xbufp->head[0].iov_len + xbufp->page_len +
> +			xbufp->tail[0].iov_len;
> +
> +	task->tk_action = call_bc_transmit;
> +	atomic_inc(&task->tk_count);
> +	BUG_ON(atomic_read(&task->tk_count) != 2);
> +	rpc_execute(task);
> +
> +out:
> +	dprintk("RPC: rpc_run_bc_task: task= %p\n", task);
> +	return task;
> +}
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  void
>  rpc_call_start(struct rpc_task *task)
>  {
> @@ -1098,7 +1147,7 @@ call_transmit(struct rpc_task *task)
>  	 * in order to allow access to the socket to other RPC requests.
>  	 */
>  	call_transmit_status(task);
> -	if (task->tk_msg.rpc_proc->p_decode != NULL)
> +	if (rpc_reply_expected(task))
>  		return;
>  	task->tk_action = rpc_exit_task;
>  	rpc_wake_up_queued_task(&task->tk_xprt->pending, task);
> @@ -1133,6 +1182,72 @@ call_transmit_status(struct rpc_task *task)
>  	}
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +/*
> + * 5b.	Send the backchannel RPC reply.  On error, drop the reply.  In
> + * addition, disconnect on connectivity errors.
> + */
> +static void
> +call_bc_transmit(struct rpc_task *task)
> +{
> +	struct rpc_rqst *req = task->tk_rqstp;
> +
> +	BUG_ON(task->tk_status != 0);
> +	task->tk_status = xprt_prepare_transmit(task);
> +	if (task->tk_status == -EAGAIN) {
> +		/*
> +		 * Could not reserve the transport. Try again after the
> +		 * transport is released.
> +		 */
> +		task->tk_status = 0;
> +		task->tk_action = call_bc_transmit;
> +		return;
> +	}
> +
> +	task->tk_action = rpc_exit_task;
> +	if (task->tk_status < 0) {
> +		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
> +			"error: %d\n", task->tk_status);
> +		return;
> +	}
> +
> +	xprt_transmit(task);
> +	xprt_end_transmit(task);
> +	dprint_status(task);
> +	switch (task->tk_status) {
> +	case 0:
> +		/* Success */
> +		break;
> +	case -EHOSTDOWN:
> +	case -EHOSTUNREACH:
> +	case -ENETUNREACH:
> +	case -ETIMEDOUT:
> +		/*
> +		 * Problem reaching the server.  Disconnect and let the
> +		 * forechannel reestablish the connection.  The server will
> +		 * have to retransmit the backchannel request and we'll
> +		 * reprocess it.  Since these ops are idempotent, there's no
> +		 * need to cache our reply at this time.
> +		 */
> +		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
> +			"error: %d\n", task->tk_status);
> +		xprt_conditional_disconnect(task->tk_xprt,
> +			req->rq_connect_cookie);
> +		break;
> +	default:
> +		/*
> +		 * We were unable to reply and will have to drop the
> +		 * request.  The server should reconnect and retransmit.
> +		 */
> +		BUG_ON(task->tk_status == -EAGAIN);
> +		printk(KERN_NOTICE "RPC: Could not send backchannel reply "
> +			"error: %d\n", task->tk_status);
> +		break;
> +	}
> +	rpc_wake_up_queued_task(&req->rq_xprt->pending, task);
> +}
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  /*
>   * 6.	Sort out the RPC call status
>   */
> diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
> index 1ef6e46..a0e3d97 100644
> --- a/net/sunrpc/stats.c
> +++ b/net/sunrpc/stats.c
> @@ -141,12 +141,14 @@ EXPORT_SYMBOL_GPL(rpc_free_iostats);
>  void rpc_count_iostats(struct rpc_task *task)
>  {
>  	struct rpc_rqst *req = task->tk_rqstp;
> -	struct rpc_iostats *stats = task->tk_client->cl_metrics;
> +	struct rpc_iostats *stats;
>  	struct rpc_iostats *op_metrics;
>  	long rtt, execute, queue;
>  
> -	if (!stats || !req)
> +	if (!task->tk_client || task->tk_client->cl_metrics || !req)

When are we going to have a tk_client without metrics?

>  		return;
> +
> +	stats = task->tk_client->cl_metrics;
>  	op_metrics = &stats[task->tk_msg.rpc_proc->p_statidx];
>  
>  	op_metrics->om_ops++;
> diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h
> new file mode 100644
> index 0000000..b462de4
> --- /dev/null
> +++ b/net/sunrpc/sunrpc.h
> @@ -0,0 +1,35 @@
> +/******************************************************************************
> +
> +(c) 2008 Network Appliance, Inc.  All Rights Reserved.
> +

Ditto...

> +Network Appliance provides this source code under the GPL v2 License.
> +The GPL v2 license is available at
> +http://opensource.org/licenses/gpl-license.php.
> +
> +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
> +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
> +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
> +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
> +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
> +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
> +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
> +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +******************************************************************************/
> +
> +/*
> + * Functions and macros used internally by RPC
> + */
> +
> +#ifndef _NET_SUNRPC_SUNRPC_H
> +#define _NET_SUNRPC_SUNRPC_H
> +
> +#define rpc_reply_expected(task) \
> +	(((task)->tk_msg.rpc_proc != NULL) && \
> +	((task)->tk_msg.rpc_proc->p_decode != NULL))

Why is this a macro instead of being an inlined function?

> +
> +#endif /* _NET_SUNRPC_SUNRPC_H */
> +
> diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
> index bbaec23..df65d15 100644
> --- a/net/sunrpc/xprt.c
> +++ b/net/sunrpc/xprt.c
> @@ -12,8 +12,9 @@
>   *  -	Next, the caller puts together the RPC message, stuffs it into
>   *	the request struct, and calls xprt_transmit().
>   *  -	xprt_transmit sends the message and installs the caller on the
> - *	transport's wait list. At the same time, it installs a timer that
> - *	is run after the packet's timeout has expired.
> + *	transport's wait list. At the same time, if a reply is expected,
> + *	it installs a timer that is run after the packet's timeout has
> + *	expired.
>   *  -	When a packet arrives, the data_ready handler walks the list of
>   *	pending requests for that transport. If a matching XID is found, the
>   *	caller is woken up, and the timer removed.
> @@ -46,6 +47,8 @@
>  #include <linux/sunrpc/clnt.h>
>  #include <linux/sunrpc/metrics.h>
>  
> +#include "sunrpc.h"
> +
>  /*
>   * Local variables
>   */
> @@ -875,7 +878,10 @@ void xprt_transmit(struct rpc_task *task)
>  	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
>  
>  	if (!req->rq_received) {
> -		if (list_empty(&req->rq_list)) {
> +		if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
> +			/*
> +			 * Add to the list only if we're expecting a reply
> +			 */
>  			spin_lock_bh(&xprt->transport_lock);
>  			/* Update the softirq receive buffer */
>  			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
> @@ -910,8 +916,13 @@ void xprt_transmit(struct rpc_task *task)
>  	/* Don't race with disconnect */
>  	if (!xprt_connected(xprt))
>  		task->tk_status = -ENOTCONN;
> -	else if (!req->rq_received)
> +	else if (!req->rq_received && rpc_reply_expected(task)) {
> +		/*
> +		 * Sleep on the pending queue since
> +		 * we're expecting a reply.
> +		 */
>  		rpc_sleep_on(&xprt->pending, task, xprt_timer);
> +	}
>  	spin_unlock_bh(&xprt->transport_lock);
>  }
>  
> @@ -984,11 +995,15 @@ static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
>   */
>  void xprt_release(struct rpc_task *task)
>  {
> -	struct rpc_xprt	*xprt = task->tk_xprt;
> +	struct rpc_xprt	*xprt;
>  	struct rpc_rqst	*req;
> +	int prealloc;
>  
> +	BUG_ON(atomic_read(&task->tk_count) < 0);

Err?

>  	if (!(req = task->tk_rqstp))
>  		return;
> +	prealloc = bc_prealloc(req);	/* Preallocated backchannel request? */
> +	xprt = req->rq_xprt;
>  	rpc_count_iostats(task);
>  	spin_lock_bh(&xprt->transport_lock);
>  	xprt->ops->release_xprt(xprt, task);
> @@ -1001,10 +1016,19 @@ void xprt_release(struct rpc_task *task)
>  		mod_timer(&xprt->timer,
>  				xprt->last_used + xprt->idle_timeout);
>  	spin_unlock_bh(&xprt->transport_lock);
> -	xprt->ops->buf_free(req->rq_buffer);
> +	if (!bc_prealloc(req))
> +		xprt->ops->buf_free(req->rq_buffer);
>  	task->tk_rqstp = NULL;
>  	if (req->rq_release_snd_buf)
>  		req->rq_release_snd_buf(req);
> +
> +	/*
> +	 * Early exit if this is a backchannel preallocated request.
> +	 * There is no need to have it added to the RPC slot list.
> +	 */
> +	if (prealloc)
> +		return;

Could we change the name of 'prealloc' to something along the lines of
'is_bc_request'?

> +
>  	memset(req, 0, sizeof(*req));	/* mark unused */
>  
>  	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 14/39] nfs41: Implement NFSv4.1 callback service process.

[Trond Myklebust]

On Fri, 2009-05-01 at 02:21 +0300, Benny Halevy wrote:
> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> 
> nfs41_callback_up() initializes the necessary queues and creates the new
> nfs41_callback_svc thread.  This thread executes the callback service which
> waits for requests to arrive on the svc_serv->sv_cb_list.
> 
> NFS41_BC_MIN_CALLBACKS is set to 1 because we expect callbacks to not
> cause substantial latency.
> 
> The actual processing of the callback will be implemented as a separate patch.
> 
> There is only one NFSv4.1 callback service.  The first caller of
> nfs4_callback_up() creates the service, subsequent callers increment a
> reference count on the service.  The service is destroyed when the last
> caller invokes nfs_callback_down().
> 
> The transport needs to hold a reference to the callback service in order
> to invoke it during callback processing.  Currently this reference is only
> obtained when the service is first created.  This is incorrect, since
> subsequent registrations for other transports will leave the xprt->serv
> pointer uninitialized, leading to an oops when a callback arrives on
> the "unreferenced" transport.
> 
> This patch fixes the problem by ensuring that a reference to the service
> is saved in xprt->serv, either because the service is created by this
> invocation to nfs4_callback_up() or by a prior invocation.
> 
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> [nfs41: Add a reference to svc_serv during callback service bring up]
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback.c          |   83 ++++++++++++++++++++++++++++++++++++++++++-
>  fs/nfs/callback.h          |    7 ++++
>  include/linux/sunrpc/svc.h |    2 +
>  3 files changed, 90 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
> index 3926046..9ee174c 100644
> --- a/fs/nfs/callback.c
> +++ b/fs/nfs/callback.c
> @@ -17,6 +17,9 @@
>  #include <linux/freezer.h>
>  #include <linux/kthread.h>
>  #include <linux/sunrpc/svcauth_gss.h>
> +#if defined(CONFIG_NFS_V4_1)
> +#include <linux/sunrpc/bc_xprt.h>
> +#endif
>  
>  #include <net/inet_sock.h>
>  
> @@ -131,6 +134,69 @@ out_err:
>  	return ERR_PTR(ret);
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +/*
> + * The callback service for NFSv4.1 callbacks
> + */
> +static int
> +nfs41_callback_svc(void *vrqstp)
> +{
> +	struct svc_rqst *rqstp = vrqstp;
> +	struct svc_serv *serv = rqstp->rq_server;
> +	struct rpc_rqst *req;
> +	int error;
> +	DEFINE_WAIT(wq);
> +
> +	set_freezable();
> +
> +	/*
> +	 * FIXME: do we really need to run this under the BKL? If so, please
> +	 * add a comment about what it's intended to protect.
> +	 */
> +	lock_kernel();
> +	while (!kthread_should_stop()) {
> +		prepare_to_wait(&serv->sv_cb_waitq, &wq, TASK_INTERRUPTIBLE);
> +		spin_lock_bh(&serv->sv_cb_lock);
> +		if (!list_empty(&serv->sv_cb_list)) {
> +			req = list_first_entry(&serv->sv_cb_list,
> +					struct rpc_rqst, rq_bc_list);
> +			list_del(&req->rq_bc_list);
> +			spin_unlock_bh(&serv->sv_cb_lock);
> +			dprintk("Invoking bc_svc_process()\n");
> +			error = bc_svc_process(serv, req, rqstp);
> +			dprintk("bc_svc_process() returned w/ error code= %d\n",
> +				error);
> +		} else {
> +			spin_unlock_bh(&serv->sv_cb_lock);
> +			schedule();
> +		}
> +		finish_wait(&serv->sv_cb_waitq, &wq);
> +	}
> +	unlock_kernel();
> +	nfs_callback_info.task = NULL;
> +	svc_exit_thread(rqstp);
> +	return 0;
> +}
> +
> +/*
> + * Bring up the NFSv4.1 callback service
> + */
> +struct svc_rqst *
> +nfs41_callback_up(struct svc_serv *serv, struct rpc_xprt *xprt)
> +{
> +	/*
> +	 * Save the svc_serv in the transport so that it can
> +	 * be referenced when the session backchannel is initialized
> +	 */
> +	xprt->bc_serv = serv;
> +
> +	INIT_LIST_HEAD(&serv->sv_cb_list);
> +	spin_lock_init(&serv->sv_cb_lock);
> +	init_waitqueue_head(&serv->sv_cb_waitq);
> +	return svc_prepare_thread(serv, &serv->sv_pools[0]);
> +}
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  /*
>   * Bring up the callback thread if it is not already up.
>   */
> @@ -141,10 +207,18 @@ int nfs_callback_up(u32 minorversion, void *args)
>  	int (*callback_svc)(void *vrqstp);
>  	char svc_name[12];
>  	int ret = 0;
> +#if defined(CONFIG_NFS_V4_1)
> +	struct rpc_xprt *xprt = (struct rpc_xprt *)args;
> +#endif /* CONFIG_NFS_V4_1 */

More ugly embedded ifdefs...

>  
>  	mutex_lock(&nfs_callback_mutex);
> -	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> +	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL) {
> +#if defined(CONFIG_NFS_V4_1)
> +		if (minorversion)
> +			xprt->bc_serv = nfs_callback_info.serv;
> +#endif /* CONFIG_NFS_V4_1 */

Ditto

>  		goto out;
> +	}
>  	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, NULL);
>  	if (!serv) {
>  		ret = -ENOMEM;
> @@ -157,7 +231,12 @@ int nfs_callback_up(u32 minorversion, void *args)
>  		rqstp = nfs4_callback_up(serv);
>  		callback_svc = nfs4_callback_svc;
>  	} else {
> -		BUG();	/* for now */
> +#if defined(CONFIG_NFS_V4_1)
> +		rqstp = nfs41_callback_up(serv, xprt);
> +		callback_svc = nfs41_callback_svc;
> +#else  /* CONFIG_NFS_V4_1 */
> +		BUG();
> +#endif /* CONFIG_NFS_V4_1 */

Ditto

>  	}
>  
>  	if (IS_ERR(rqstp)) {
> diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> index 7f12e65..f2696ba 100644
> --- a/fs/nfs/callback.h
> +++ b/fs/nfs/callback.h
> @@ -70,6 +70,13 @@ extern void nfs_callback_down(void);
>  #define nfs_callback_down()	do {} while(0)
>  #endif
>  
> +/*
> + * nfs41: Callbacks are expected to not cause substantial latency,
> + * so we limit their concurrency to 1 by setting up the maximum number
> + * of slots for the backchannel.
> + */
> +#define NFS41_BC_MIN_CALLBACKS 1
> +
>  extern unsigned int nfs_callback_set_tcpport;
>  extern unsigned short nfs_callback_tcpport;
>  extern unsigned short nfs_callback_tcpport6;
> diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
> index 4a8afbd..16043c4 100644
> --- a/include/linux/sunrpc/svc.h
> +++ b/include/linux/sunrpc/svc.h
> @@ -419,6 +419,8 @@ int		   svc_set_num_threads(struct svc_serv *, struct svc_pool *, int);
>  int		   svc_pool_stats_open(struct svc_serv *serv, struct file *file);
>  void		   svc_destroy(struct svc_serv *);
>  int		   svc_process(struct svc_rqst *);
> +int		   bc_svc_process(struct svc_serv *, struct rpc_rqst *,
> +			struct svc_rqst *);

OK, I give up. Why is the function text and export declaration in one
patch, while the declaration is in this patch?

>  int		   svc_register(const struct svc_serv *, const int,
>  				const unsigned short, const unsigned short);
>  

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 11/39] nfs41: Backchannel callback service helper routines

[Trond Myklebust]

On Fri, 2009-05-01 at 02:20 +0300, Benny Halevy wrote:
> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> 
> Executes the backchannel task on the RPC state machine using
> the existing open connection previously established by the client.
> 
> Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
> 
> nfs41: Add bc_svc.o to sunrpc Makefile.
> 
> [nfs41: bc_send() does not need to be exported outside RPC module]
> [nfs41: xprt_free_bc_request() need not be exported outside RPC module]
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  include/linux/sunrpc/bc_xprt.h |    3 +
>  net/sunrpc/Makefile            |    2 +-
>  net/sunrpc/backchannel_rqst.c  |    1 -
>  net/sunrpc/bc_svc.c            |   80 ++++++++++++++++++++++++++++++++++++++++
>  net/sunrpc/xprtsock.c          |    3 +
>  5 files changed, 87 insertions(+), 2 deletions(-)
>  create mode 100644 net/sunrpc/bc_svc.c
> 
> diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h
> index 1c1746a..3016c00 100644
> --- a/include/linux/sunrpc/bc_xprt.h
> +++ b/include/linux/sunrpc/bc_xprt.h
> @@ -29,12 +29,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>  
>  #include <linux/sunrpc/svcsock.h>
>  #include <linux/sunrpc/xprt.h>
> +#include <linux/sunrpc/sched.h>
>  
>  #ifdef CONFIG_NFS_V4_1
>  struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt);
>  void xprt_free_bc_request(struct rpc_rqst *req);
>  int xprt_setup_backchannel(struct rpc_xprt *, unsigned int min_reqs);
>  void xprt_destroy_backchannel(struct rpc_xprt *, int max_reqs);
> +void bc_release_request(struct rpc_task *);
> +int bc_send(struct rpc_rqst *req);
>  #else /* CONFIG_NFS_V4_1 */
>  static inline int xprt_setup_backchannel(struct rpc_xprt *xprt,
>  					 unsigned int min_reqs)
> diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
> index 4a01f96..db73fd2 100644
> --- a/net/sunrpc/Makefile
> +++ b/net/sunrpc/Makefile
> @@ -13,6 +13,6 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
>  	    rpcb_clnt.o timer.o xdr.o \
>  	    sunrpc_syms.o cache.o rpc_pipe.o \
>  	    svc_xprt.o
> -sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o
> +sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o bc_svc.o
>  sunrpc-$(CONFIG_PROC_FS) += stats.o
>  sunrpc-$(CONFIG_SYSCTL) += sysctl.o
> diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c
> index 7d7708a..92fb3bd 100644
> --- a/net/sunrpc/backchannel_rqst.c
> +++ b/net/sunrpc/backchannel_rqst.c
> @@ -265,6 +265,5 @@ void xprt_free_bc_request(struct rpc_rqst *req)
>  	list_add(&req->rq_bc_pa_list, &xprt->bc_pa_list);
>  	spin_unlock_bh(&xprt->bc_pa_lock);
>  }
> -EXPORT_SYMBOL(xprt_free_bc_request);

Err.... That's random... What is there in this patch that suddenly makes
the export unnecessary?

>  #endif /* CONFIG_NFS_V4_1 */
> diff --git a/net/sunrpc/bc_svc.c b/net/sunrpc/bc_svc.c
> new file mode 100644
> index 0000000..b13f51d
> --- /dev/null
> +++ b/net/sunrpc/bc_svc.c
> @@ -0,0 +1,80 @@
> +/******************************************************************************
> +
> +(c) 2007 Network Appliance, Inc.  All Rights Reserved.

More references to historic entities...

> +
> +Network Appliance provides this source code under the GPL v2 License.
> +The GPL v2 license is available at
> +http://opensource.org/licenses/gpl-license.php.
> +
> +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
> +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
> +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
> +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
> +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
> +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
> +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
> +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +******************************************************************************/
> +
> +/*
> + * The NFSv4.1 callback service helper routines.
> + * They implement the transport level processing required to send the
> + * reply over an existing open connection previously established by the client.
> + */
> +
> +#if defined(CONFIG_NFS_V4_1)
> +
> +#include <linux/module.h>
> +
> +#include <linux/sunrpc/xprt.h>
> +#include <linux/sunrpc/sched.h>
> +#include <linux/sunrpc/bc_xprt.h>
> +
> +#define RPCDBG_FACILITY	RPCDBG_SVCDSP
> +
> +void bc_release_request(struct rpc_task *task)
> +{
> +	struct rpc_rqst *req = task->tk_rqstp;
> +
> +	dprintk("RPC:       bc_release_request: task= %p\n", task);
> +
> +	/*
> +	 * Release this request only if it's a backchannel
> +	 * preallocated request
> +	 */
> +	if (!bc_prealloc(req))
> +		return;
> +	xprt_free_bc_request(req);
> +}
> +
> +/* Empty callback ops */
> +static const struct rpc_call_ops nfs41_callback_ops = {
> +};
> +
> +
> +/*
> + * Send the callback reply
> + */
> +int bc_send(struct rpc_rqst *req)
> +{
> +	struct rpc_task *task;
> +	int ret;
> +
> +	dprintk("RPC:       bc_send req= %p\n", req);
> +	task = rpc_run_bc_task(req, &nfs41_callback_ops);
> +	if (IS_ERR(task))
> +		ret = PTR_ERR(task);
> +	else {
> +		BUG_ON(atomic_read(&task->tk_count) != 1);
> +		ret = task->tk_status;
> +		rpc_put_task(task);
> +	}
> +	return ret;
> +	dprintk("RPC:       bc_send ret= %d \n", ret);
> +}
> +
> +#endif /* CONFIG_NFS_V4_1 */
> diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
> index 403ebda..5bf9e66 100644
> --- a/net/sunrpc/xprtsock.c
> +++ b/net/sunrpc/xprtsock.c
> @@ -2145,6 +2145,9 @@ static struct rpc_xprt_ops xs_tcp_ops = {
>  	.buf_free		= rpc_free,
>  	.send_request		= xs_tcp_send_request,
>  	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
> +#if defined(CONFIG_NFS_V4_1)
> +	.release_request	= bc_release_request,
> +#endif /* CONFIG_NFS_V4_1 */
>  	.close			= xs_tcp_shutdown,
>  	.destroy		= xs_destroy,
>  	.print_stats		= xs_tcp_print_stats,

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

RE: [pnfs] [RFC 10/39] nfs41: Add backchannel processing support to RPC state machine

[Labiaga, Ricardo]

> -----Original Message-----
> From: Myklebust, Trond
> Sent: Thursday, June 04, 2009 12:55 PM
> To: Benny Halevy
> Cc: Adamson, Andy; linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 10/39] nfs41: Add backchannel processing
support
> to RPC state machine
> 
> On Fri, 2009-05-01 at 02:20 +0300, Benny Halevy wrote:
> > From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> >
> > Adds rpc_run_bc_task() which is called by the NFS callback service
to
> > process backchannel requests.  It performs similar work to
> rpc_run_task()
> > though "schedules" the backchannel task to be executed starting at
the
> > call_trasmit state in the RPC state machine.
> >
> > It also introduces some miscellaneous updates to the argument
> validation,
> > call_transmit, and transport cleanup functions to take into account
> > that there are now forechannel and backchannel tasks.
> >
> > Backchannel requests do not carry an RPC message structure, since
the
> > payload has already been XDR encoded using the existing NFSv4
callback
> > mechanism.
> >
> > Introduce a new transmit state for the client to reply on to
backchannel
> > requests.  This new state simply reserves the transport and issues
the
> > reply.  In case of a connection related error, disconnects the
transport
> and
> > drops the reply.  It requires the forechannel to re-establish the
> connection
> > and the server to retransmit the request, as stated in NFSv4.1
section
> > 2.9.2 "Client and Server Transport Behavior".
> >
> > Note: There is no need to loop attempting to reserve the transport.
If
> EAGAIN
> > is returned by xprt_prepare_transmit(), return with tk_status == 0,
> > setting tk_action to call_bc_transmit.  rpc_execute() will invoke it
> again
> > after the task is taken off the sleep queue.
> >
> > [nfs41: rpc_run_bc_task() need not be exported outside RPC module]
> > [nfs41: New call_bc_transmit RPC state]
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > [nfs41: Backchannel: No need to loop in call_bc_transmit()]
> > Signed-off-by: Andy Adamson <andros@netapp.com>
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  include/linux/sunrpc/sched.h |    2 +
> >  include/linux/sunrpc/xprt.h  |   12 ++++
> >  net/sunrpc/clnt.c            |  117
> +++++++++++++++++++++++++++++++++++++++++-
> >  net/sunrpc/stats.c           |    6 ++-
> >  net/sunrpc/sunrpc.h          |   35 +++++++++++++
> >  net/sunrpc/xprt.c            |   36 +++++++++++--
> >  6 files changed, 199 insertions(+), 9 deletions(-)
> >  create mode 100644 net/sunrpc/sunrpc.h
> >
> > diff --git a/include/linux/sunrpc/sched.h
b/include/linux/sunrpc/sched.h
> > index 1773768..4010977 100644
> > --- a/include/linux/sunrpc/sched.h
> > +++ b/include/linux/sunrpc/sched.h
> > @@ -210,6 +210,8 @@ struct rpc_wait_queue {
> >   */
> >  struct rpc_task *rpc_new_task(const struct rpc_task_setup *);
> >  struct rpc_task *rpc_run_task(const struct rpc_task_setup *);
> > +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> > +				const struct rpc_call_ops *ops);
> >  void		rpc_put_task(struct rpc_task *);
> >  void		rpc_exit_task(struct rpc_task *);
> >  void		rpc_release_calldata(const struct rpc_call_ops
*, void *);
> > diff --git a/include/linux/sunrpc/xprt.h
b/include/linux/sunrpc/xprt.h
> > index 6b37724..1531abe 100644
> > --- a/include/linux/sunrpc/xprt.h
> > +++ b/include/linux/sunrpc/xprt.h
> > @@ -215,6 +215,18 @@ struct rpc_xprt {
> >  						/* buffer in use */
> >  #endif /* CONFIG_NFS_V4_1 */
> >
> > +#if defined(CONFIG_NFS_V4_1)
> > +static inline int bc_prealloc(struct rpc_rqst *req)
> > +{
> > +	return test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
> > +}
> > +#else
> > +static inline int bc_prealloc(struct rpc_rqst *req)
> > +{
> > +	return 0;
> > +}
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  struct xprt_create {
> >  	int			ident;		/* XPRT_TRANSPORT
identifier */
> >  	struct sockaddr *	srcaddr;	/* optional local
address */
> > diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> > index 76d7d46..349b4d6 100644
> > --- a/net/sunrpc/clnt.c
> > +++ b/net/sunrpc/clnt.c
> > @@ -36,7 +36,9 @@
> >  #include <linux/sunrpc/clnt.h>
> >  #include <linux/sunrpc/rpc_pipe_fs.h>
> >  #include <linux/sunrpc/metrics.h>
> > +#include <linux/sunrpc/bc_xprt.h>
> >
> > +#include "sunrpc.h"
> >
> >  #ifdef RPC_DEBUG
> >  # define RPCDBG_FACILITY	RPCDBG_CALL
> > @@ -63,6 +65,9 @@ static void	call_decode(struct rpc_task
*task);
> >  static void	call_bind(struct rpc_task *task);
> >  static void	call_bind_status(struct rpc_task *task);
> >  static void	call_transmit(struct rpc_task *task);
> > +#if defined(CONFIG_NFS_V4_1)
> > +static void	call_bc_transmit(struct rpc_task *task);
> > +#endif /* CONFIG_NFS_V4_1 */
> >  static void	call_status(struct rpc_task *task);
> >  static void	call_transmit_status(struct rpc_task *task);
> >  static void	call_refresh(struct rpc_task *task);
> > @@ -613,6 +618,50 @@ rpc_call_async(struct rpc_clnt *clnt, const
struct
> rpc_message *msg, int flags,
> >  }
> >  EXPORT_SYMBOL_GPL(rpc_call_async);
> >
> > +#if defined(CONFIG_NFS_V4_1)
> > +/**
> > + * rpc_run_bc_task - Allocate a new RPC task for backchannel use,
then
> run
> > + * rpc_execute against it
> > + * @ops: RPC call ops
> > + */
> > +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> > +					const struct rpc_call_ops
*tk_ops)
> > +{
> > +	struct rpc_task *task;
> > +	struct xdr_buf *xbufp = &req->rq_snd_buf;
> > +	struct rpc_task_setup task_setup_data = {
> > +		.callback_ops = tk_ops,
> > +	};
> > +
> > +	dprintk("RPC: rpc_run_bc_task req= %p\n", req);
> > +	/*
> > +	 * Create an rpc_task to send the data
> > +	 */
> > +	task = rpc_new_task(&task_setup_data);
> > +	if (!task) {
> > +		xprt_free_bc_request(req);
> > +		goto out;
> > +	}
> > +	task->tk_rqstp = req;
> > +
> > +	/*
> > +	 * Set up the xdr_buf length.
> > +	 * This also indicates that the buffer is XDR encoded already.
> > +	 */
> > +	xbufp->len = xbufp->head[0].iov_len + xbufp->page_len +
> > +			xbufp->tail[0].iov_len;
> > +
> > +	task->tk_action = call_bc_transmit;
> > +	atomic_inc(&task->tk_count);
> > +	BUG_ON(atomic_read(&task->tk_count) != 2);
> > +	rpc_execute(task);
> > +
> > +out:
> > +	dprintk("RPC: rpc_run_bc_task: task= %p\n", task);
> > +	return task;
> > +}
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  void
> >  rpc_call_start(struct rpc_task *task)
> >  {
> > @@ -1098,7 +1147,7 @@ call_transmit(struct rpc_task *task)
> >  	 * in order to allow access to the socket to other RPC requests.
> >  	 */
> >  	call_transmit_status(task);
> > -	if (task->tk_msg.rpc_proc->p_decode != NULL)
> > +	if (rpc_reply_expected(task))
> >  		return;
> >  	task->tk_action = rpc_exit_task;
> >  	rpc_wake_up_queued_task(&task->tk_xprt->pending, task);
> > @@ -1133,6 +1182,72 @@ call_transmit_status(struct rpc_task *task)
> >  	}
> >  }
> >
> > +#if defined(CONFIG_NFS_V4_1)
> > +/*
> > + * 5b.	Send the backchannel RPC reply.  On error, drop the
reply.  In
> > + * addition, disconnect on connectivity errors.
> > + */
> > +static void
> > +call_bc_transmit(struct rpc_task *task)
> > +{
> > +	struct rpc_rqst *req = task->tk_rqstp;
> > +
> > +	BUG_ON(task->tk_status != 0);
> > +	task->tk_status = xprt_prepare_transmit(task);
> > +	if (task->tk_status == -EAGAIN) {
> > +		/*
> > +		 * Could not reserve the transport. Try again after the
> > +		 * transport is released.
> > +		 */
> > +		task->tk_status = 0;
> > +		task->tk_action = call_bc_transmit;
> > +		return;
> > +	}
> > +
> > +	task->tk_action = rpc_exit_task;
> > +	if (task->tk_status < 0) {
> > +		printk(KERN_NOTICE "RPC: Could not send backchannel
reply "
> > +			"error: %d\n", task->tk_status);
> > +		return;
> > +	}
> > +
> > +	xprt_transmit(task);
> > +	xprt_end_transmit(task);
> > +	dprint_status(task);
> > +	switch (task->tk_status) {
> > +	case 0:
> > +		/* Success */
> > +		break;
> > +	case -EHOSTDOWN:
> > +	case -EHOSTUNREACH:
> > +	case -ENETUNREACH:
> > +	case -ETIMEDOUT:
> > +		/*
> > +		 * Problem reaching the server.  Disconnect and let the
> > +		 * forechannel reestablish the connection.  The server
will
> > +		 * have to retransmit the backchannel request and we'll
> > +		 * reprocess it.  Since these ops are idempotent,
there's no
> > +		 * need to cache our reply at this time.
> > +		 */
> > +		printk(KERN_NOTICE "RPC: Could not send backchannel
reply "
> > +			"error: %d\n", task->tk_status);
> > +		xprt_conditional_disconnect(task->tk_xprt,
> > +			req->rq_connect_cookie);
> > +		break;
> > +	default:
> > +		/*
> > +		 * We were unable to reply and will have to drop the
> > +		 * request.  The server should reconnect and retransmit.
> > +		 */
> > +		BUG_ON(task->tk_status == -EAGAIN);
> > +		printk(KERN_NOTICE "RPC: Could not send backchannel
reply "
> > +			"error: %d\n", task->tk_status);
> > +		break;
> > +	}
> > +	rpc_wake_up_queued_task(&req->rq_xprt->pending, task);
> > +}
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  /*
> >   * 6.	Sort out the RPC call status
> >   */
> > diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
> > index 1ef6e46..a0e3d97 100644
> > --- a/net/sunrpc/stats.c
> > +++ b/net/sunrpc/stats.c
> > @@ -141,12 +141,14 @@ EXPORT_SYMBOL_GPL(rpc_free_iostats);
> >  void rpc_count_iostats(struct rpc_task *task)
> >  {
> >  	struct rpc_rqst *req = task->tk_rqstp;
> > -	struct rpc_iostats *stats = task->tk_client->cl_metrics;
> > +	struct rpc_iostats *stats;
> >  	struct rpc_iostats *op_metrics;
> >  	long rtt, execute, queue;
> >
> > -	if (!stats || !req)
> > +	if (!task->tk_client || task->tk_client->cl_metrics || !req)
> 
> When are we going to have a tk_client without metrics?
> 

The original code checked for stats being non-null, so we're doing the
same thing.  Having said that, I don't see when cl_metrics could be null
either, since rpc_new_client() guarantees its allocation.

I can remove it, but it's not strictly due to functionality in this
patch.

> >  		return;
> > +
> > +	stats = task->tk_client->cl_metrics;
> >  	op_metrics = &stats[task->tk_msg.rpc_proc->p_statidx];
> >
> >  	op_metrics->om_ops++;
> > diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h
> > new file mode 100644
> > index 0000000..b462de4
> > --- /dev/null
> > +++ b/net/sunrpc/sunrpc.h
> > @@ -0,0 +1,35 @@
> >
>
+/**********************************************************************
**
> ******
> > +
> > +(c) 2008 Network Appliance, Inc.  All Rights Reserved.
> > +
> 
> Ditto...

Will fix.

> > +Network Appliance provides this source code under the GPL v2
License.
> > +The GPL v2 license is available at
> > +http://opensource.org/licenses/gpl-license.php.
> > +
> > +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> > +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> > +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR
> > +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT
> OWNER OR
> > +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL,
> > +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
> > +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
> > +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
OF
> > +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING
> > +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
> > +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> > +
> >
>
+***********************************************************************
**
> *****/
> > +
> > +/*
> > + * Functions and macros used internally by RPC
> > + */
> > +
> > +#ifndef _NET_SUNRPC_SUNRPC_H
> > +#define _NET_SUNRPC_SUNRPC_H
> > +
> > +#define rpc_reply_expected(task) \
> > +	(((task)->tk_msg.rpc_proc != NULL) && \
> > +	((task)->tk_msg.rpc_proc->p_decode != NULL))
> 
> Why is this a macro instead of being an inlined function?
> 

My fault, I was still getting acquainted with the difference between
macros and inlining.  Will change to inlined function.

> > +
> > +#endif /* _NET_SUNRPC_SUNRPC_H */
> > +
> > diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
> > index bbaec23..df65d15 100644
> > --- a/net/sunrpc/xprt.c
> > +++ b/net/sunrpc/xprt.c
> > @@ -12,8 +12,9 @@
> >   *  -	Next, the caller puts together the RPC message, stuffs
it into
> >   *	the request struct, and calls xprt_transmit().
> >   *  -	xprt_transmit sends the message and installs the caller
on the
> > - *	transport's wait list. At the same time, it installs a timer
> that
> > - *	is run after the packet's timeout has expired.
> > + *	transport's wait list. At the same time, if a reply is expected,
> > + *	it installs a timer that is run after the packet's timeout has
> > + *	expired.
> >   *  -	When a packet arrives, the data_ready handler walks the
list of
> >   *	pending requests for that transport. If a matching XID is found,
> the
> >   *	caller is woken up, and the timer removed.
> > @@ -46,6 +47,8 @@
> >  #include <linux/sunrpc/clnt.h>
> >  #include <linux/sunrpc/metrics.h>
> >
> > +#include "sunrpc.h"
> > +
> >  /*
> >   * Local variables
> >   */
> > @@ -875,7 +878,10 @@ void xprt_transmit(struct rpc_task *task)
> >  	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid,
req->rq_slen);
> >
> >  	if (!req->rq_received) {
> > -		if (list_empty(&req->rq_list)) {
> > +		if (list_empty(&req->rq_list) &&
rpc_reply_expected(task)) {
> > +			/*
> > +			 * Add to the list only if we're expecting a
reply
> > +			 */
> >  			spin_lock_bh(&xprt->transport_lock);
> >  			/* Update the softirq receive buffer */
> >  			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
> > @@ -910,8 +916,13 @@ void xprt_transmit(struct rpc_task *task)
> >  	/* Don't race with disconnect */
> >  	if (!xprt_connected(xprt))
> >  		task->tk_status = -ENOTCONN;
> > -	else if (!req->rq_received)
> > +	else if (!req->rq_received && rpc_reply_expected(task)) {
> > +		/*
> > +		 * Sleep on the pending queue since
> > +		 * we're expecting a reply.
> > +		 */
> >  		rpc_sleep_on(&xprt->pending, task, xprt_timer);
> > +	}
> >  	spin_unlock_bh(&xprt->transport_lock);
> >  }
> >
> > @@ -984,11 +995,15 @@ static void xprt_request_init(struct rpc_task
> *task, struct rpc_xprt *xprt)
> >   */
> >  void xprt_release(struct rpc_task *task)
> >  {
> > -	struct rpc_xprt	*xprt = task->tk_xprt;
> > +	struct rpc_xprt	*xprt;
> >  	struct rpc_rqst	*req;
> > +	int prealloc;
> >
> > +	BUG_ON(atomic_read(&task->tk_count) < 0);
> 
> Err?
>

This was useful during early development.  Never hit it though :-)  So
I'll remove it.

 
> >  	if (!(req = task->tk_rqstp))
> >  		return;
> > +	prealloc = bc_prealloc(req);	/* Preallocated backchannel
request? */
> > +	xprt = req->rq_xprt;
> >  	rpc_count_iostats(task);
> >  	spin_lock_bh(&xprt->transport_lock);
> >  	xprt->ops->release_xprt(xprt, task);
> > @@ -1001,10 +1016,19 @@ void xprt_release(struct rpc_task *task)
> >  		mod_timer(&xprt->timer,
> >  				xprt->last_used + xprt->idle_timeout);
> >  	spin_unlock_bh(&xprt->transport_lock);
> > -	xprt->ops->buf_free(req->rq_buffer);
> > +	if (!bc_prealloc(req))
> > +		xprt->ops->buf_free(req->rq_buffer);
> >  	task->tk_rqstp = NULL;
> >  	if (req->rq_release_snd_buf)
> >  		req->rq_release_snd_buf(req);
> > +
> > +	/*
> > +	 * Early exit if this is a backchannel preallocated request.
> > +	 * There is no need to have it added to the RPC slot list.
> > +	 */
> > +	if (prealloc)
> > +		return;
> 
> Could we change the name of 'prealloc' to something along the lines of
> 'is_bc_request'?
> 

Yes, will do.

- ricardo

> > +
> >  	memset(req, 0, sizeof(*req));	/* mark unused */
> >
> >  	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
> 
> --
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

RE: [pnfs] [RFC 14/39] nfs41: Implement NFSv4.1 callback service process.

[Labiaga, Ricardo]

> -----Original Message-----
> From: Myklebust, Trond
> Sent: Thursday, June 04, 2009 1:18 PM
> To: Benny Halevy
> Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 14/39] nfs41: Implement NFSv4.1 callback
service
> process.
> 
> On Fri, 2009-05-01 at 02:21 +0300, Benny Halevy wrote:
> > From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> >
> > nfs41_callback_up() initializes the necessary queues and creates the
new
> > nfs41_callback_svc thread.  This thread executes the callback
service
> which
> > waits for requests to arrive on the svc_serv->sv_cb_list.
> >
> > NFS41_BC_MIN_CALLBACKS is set to 1 because we expect callbacks to
not
> > cause substantial latency.
> >
> > The actual processing of the callback will be implemented as a
separate
> patch.
> >
> > There is only one NFSv4.1 callback service.  The first caller of
> > nfs4_callback_up() creates the service, subsequent callers increment
a
> > reference count on the service.  The service is destroyed when the
last
> > caller invokes nfs_callback_down().
> >
> > The transport needs to hold a reference to the callback service in
order
> > to invoke it during callback processing.  Currently this reference
is
> only
> > obtained when the service is first created.  This is incorrect,
since
> > subsequent registrations for other transports will leave the
xprt->serv
> > pointer uninitialized, leading to an oops when a callback arrives on
> > the "unreferenced" transport.
> >
> > This patch fixes the problem by ensuring that a reference to the
service
> > is saved in xprt->serv, either because the service is created by
this
> > invocation to nfs4_callback_up() or by a prior invocation.
> >
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > [nfs41: Add a reference to svc_serv during callback service bring
up]
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  fs/nfs/callback.c          |   83
> ++++++++++++++++++++++++++++++++++++++++++-
> >  fs/nfs/callback.h          |    7 ++++
> >  include/linux/sunrpc/svc.h |    2 +
> >  3 files changed, 90 insertions(+), 2 deletions(-)
> >
> > diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
> > index 3926046..9ee174c 100644
> > --- a/fs/nfs/callback.c
> > +++ b/fs/nfs/callback.c
> > @@ -17,6 +17,9 @@
> >  #include <linux/freezer.h>
> >  #include <linux/kthread.h>
> >  #include <linux/sunrpc/svcauth_gss.h>
> > +#if defined(CONFIG_NFS_V4_1)
> > +#include <linux/sunrpc/bc_xprt.h>
> > +#endif
> >
> >  #include <net/inet_sock.h>
> >
> > @@ -131,6 +134,69 @@ out_err:
> >  	return ERR_PTR(ret);
> >  }
> >
> > +#if defined(CONFIG_NFS_V4_1)
> > +/*
> > + * The callback service for NFSv4.1 callbacks
> > + */
> > +static int
> > +nfs41_callback_svc(void *vrqstp)
> > +{
> > +	struct svc_rqst *rqstp = vrqstp;
> > +	struct svc_serv *serv = rqstp->rq_server;
> > +	struct rpc_rqst *req;
> > +	int error;
> > +	DEFINE_WAIT(wq);
> > +
> > +	set_freezable();
> > +
> > +	/*
> > +	 * FIXME: do we really need to run this under the BKL? If so,
please
> > +	 * add a comment about what it's intended to protect.
> > +	 */
> > +	lock_kernel();
> > +	while (!kthread_should_stop()) {
> > +		prepare_to_wait(&serv->sv_cb_waitq, &wq,
TASK_INTERRUPTIBLE);
> > +		spin_lock_bh(&serv->sv_cb_lock);
> > +		if (!list_empty(&serv->sv_cb_list)) {
> > +			req = list_first_entry(&serv->sv_cb_list,
> > +					struct rpc_rqst, rq_bc_list);
> > +			list_del(&req->rq_bc_list);
> > +			spin_unlock_bh(&serv->sv_cb_lock);
> > +			dprintk("Invoking bc_svc_process()\n");
> > +			error = bc_svc_process(serv, req, rqstp);
> > +			dprintk("bc_svc_process() returned w/ error
code= %d\n",
> > +				error);
> > +		} else {
> > +			spin_unlock_bh(&serv->sv_cb_lock);
> > +			schedule();
> > +		}
> > +		finish_wait(&serv->sv_cb_waitq, &wq);
> > +	}
> > +	unlock_kernel();
> > +	nfs_callback_info.task = NULL;
> > +	svc_exit_thread(rqstp);
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Bring up the NFSv4.1 callback service
> > + */
> > +struct svc_rqst *
> > +nfs41_callback_up(struct svc_serv *serv, struct rpc_xprt *xprt)
> > +{
> > +	/*
> > +	 * Save the svc_serv in the transport so that it can
> > +	 * be referenced when the session backchannel is initialized
> > +	 */
> > +	xprt->bc_serv = serv;
> > +
> > +	INIT_LIST_HEAD(&serv->sv_cb_list);
> > +	spin_lock_init(&serv->sv_cb_lock);
> > +	init_waitqueue_head(&serv->sv_cb_waitq);
> > +	return svc_prepare_thread(serv, &serv->sv_pools[0]);
> > +}
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  /*
> >   * Bring up the callback thread if it is not already up.
> >   */
> > @@ -141,10 +207,18 @@ int nfs_callback_up(u32 minorversion, void
*args)
> >  	int (*callback_svc)(void *vrqstp);
> >  	char svc_name[12];
> >  	int ret = 0;
> > +#if defined(CONFIG_NFS_V4_1)
> > +	struct rpc_xprt *xprt = (struct rpc_xprt *)args;
> > +#endif /* CONFIG_NFS_V4_1 */
> 
> More ugly embedded ifdefs...
> 
> >
> >  	mutex_lock(&nfs_callback_mutex);
> > -	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
> > +	if (nfs_callback_info.users++ || nfs_callback_info.task != NULL)
{
> > +#if defined(CONFIG_NFS_V4_1)
> > +		if (minorversion)
> > +			xprt->bc_serv = nfs_callback_info.serv;
> > +#endif /* CONFIG_NFS_V4_1 */
> 
> Ditto
> 
> >  		goto out;
> > +	}
> >  	serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE,
> NULL);
> >  	if (!serv) {
> >  		ret = -ENOMEM;
> > @@ -157,7 +231,12 @@ int nfs_callback_up(u32 minorversion, void
*args)
> >  		rqstp = nfs4_callback_up(serv);
> >  		callback_svc = nfs4_callback_svc;
> >  	} else {
> > -		BUG();	/* for now */
> > +#if defined(CONFIG_NFS_V4_1)
> > +		rqstp = nfs41_callback_up(serv, xprt);
> > +		callback_svc = nfs41_callback_svc;
> > +#else  /* CONFIG_NFS_V4_1 */
> > +		BUG();
> > +#endif /* CONFIG_NFS_V4_1 */
> 
> Ditto
> 
> >  	}
> >
> >  	if (IS_ERR(rqstp)) {
> > diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> > index 7f12e65..f2696ba 100644
> > --- a/fs/nfs/callback.h
> > +++ b/fs/nfs/callback.h
> > @@ -70,6 +70,13 @@ extern void nfs_callback_down(void);
> >  #define nfs_callback_down()	do {} while(0)
> >  #endif
> >
> > +/*
> > + * nfs41: Callbacks are expected to not cause substantial latency,
> > + * so we limit their concurrency to 1 by setting up the maximum
number
> > + * of slots for the backchannel.
> > + */
> > +#define NFS41_BC_MIN_CALLBACKS 1
> > +
> >  extern unsigned int nfs_callback_set_tcpport;
> >  extern unsigned short nfs_callback_tcpport;
> >  extern unsigned short nfs_callback_tcpport6;
> > diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
> > index 4a8afbd..16043c4 100644
> > --- a/include/linux/sunrpc/svc.h
> > +++ b/include/linux/sunrpc/svc.h
> > @@ -419,6 +419,8 @@ int		   svc_set_num_threads(struct
svc_serv
> *, struct svc_pool *, int);
> >  int		   svc_pool_stats_open(struct svc_serv *serv,
struct file
> *file);
> >  void		   svc_destroy(struct svc_serv *);
> >  int		   svc_process(struct svc_rqst *);
> > +int		   bc_svc_process(struct svc_serv *, struct
rpc_rqst *,
> > +			struct svc_rqst *);
> 
> OK, I give up. Why is the function text and export declaration in one
> patch, while the declaration is in this patch?
> 

Will move this declaration to patch 13 along with the function and
export declaration.  Will also address the ifdefs.

- ricardo

> >  int		   svc_register(const struct svc_serv *, const
int,
> >  				const unsigned short, const unsigned
short);
> >
> 
> --
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

RE: [pnfs] [RFC 11/39] nfs41: Backchannel callback service helper routines

[Labiaga, Ricardo]

> -----Original Message-----
> From: Myklebust, Trond
> Sent: Thursday, June 04, 2009 1:21 PM
> To: Benny Halevy
> Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 11/39] nfs41: Backchannel callback service
helper
> routines
> 
> On Fri, 2009-05-01 at 02:20 +0300, Benny Halevy wrote:
> > From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> >
> > Executes the backchannel task on the RPC state machine using
> > the existing open connection previously established by the client.
> >
> > Signed-off-by: Ricardo Labiaga <ricardo.labiaga@netapp.com>
> >
> > nfs41: Add bc_svc.o to sunrpc Makefile.
> >
> > [nfs41: bc_send() does not need to be exported outside RPC module]
> > [nfs41: xprt_free_bc_request() need not be exported outside RPC
module]
> > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  include/linux/sunrpc/bc_xprt.h |    3 +
> >  net/sunrpc/Makefile            |    2 +-
> >  net/sunrpc/backchannel_rqst.c  |    1 -
> >  net/sunrpc/bc_svc.c            |   80
> ++++++++++++++++++++++++++++++++++++++++
> >  net/sunrpc/xprtsock.c          |    3 +
> >  5 files changed, 87 insertions(+), 2 deletions(-)
> >  create mode 100644 net/sunrpc/bc_svc.c
> >
> > diff --git a/include/linux/sunrpc/bc_xprt.h
> b/include/linux/sunrpc/bc_xprt.h
> > index 1c1746a..3016c00 100644
> > --- a/include/linux/sunrpc/bc_xprt.h
> > +++ b/include/linux/sunrpc/bc_xprt.h
> > @@ -29,12 +29,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH
> DAMAGE.
> >
> >  #include <linux/sunrpc/svcsock.h>
> >  #include <linux/sunrpc/xprt.h>
> > +#include <linux/sunrpc/sched.h>
> >
> >  #ifdef CONFIG_NFS_V4_1
> >  struct rpc_rqst *xprt_alloc_bc_request(struct rpc_xprt *xprt);
> >  void xprt_free_bc_request(struct rpc_rqst *req);
> >  int xprt_setup_backchannel(struct rpc_xprt *, unsigned int
min_reqs);
> >  void xprt_destroy_backchannel(struct rpc_xprt *, int max_reqs);
> > +void bc_release_request(struct rpc_task *);
> > +int bc_send(struct rpc_rqst *req);
> >  #else /* CONFIG_NFS_V4_1 */
> >  static inline int xprt_setup_backchannel(struct rpc_xprt *xprt,
> >  					 unsigned int min_reqs)
> > diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
> > index 4a01f96..db73fd2 100644
> > --- a/net/sunrpc/Makefile
> > +++ b/net/sunrpc/Makefile
> > @@ -13,6 +13,6 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o
sched.o
> \
> >  	    rpcb_clnt.o timer.o xdr.o \
> >  	    sunrpc_syms.o cache.o rpc_pipe.o \
> >  	    svc_xprt.o
> > -sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o
> > +sunrpc-$(CONFIG_NFS_V4_1) += backchannel_rqst.o bc_svc.o
> >  sunrpc-$(CONFIG_PROC_FS) += stats.o
> >  sunrpc-$(CONFIG_SYSCTL) += sysctl.o
> > diff --git a/net/sunrpc/backchannel_rqst.c
> b/net/sunrpc/backchannel_rqst.c
> > index 7d7708a..92fb3bd 100644
> > --- a/net/sunrpc/backchannel_rqst.c
> > +++ b/net/sunrpc/backchannel_rqst.c
> > @@ -265,6 +265,5 @@ void xprt_free_bc_request(struct rpc_rqst *req)
> >  	list_add(&req->rq_bc_pa_list, &xprt->bc_pa_list);
> >  	spin_unlock_bh(&xprt->bc_pa_lock);
> >  }
> > -EXPORT_SYMBOL(xprt_free_bc_request);
> 
> Err.... That's random... What is there in this patch that suddenly
makes
> the export unnecessary?
> 

Squashing mistake.  We should have never exported the
'xprt_free_bc_request' symbol since it is not needed outside sunrpc.
We'll update patch 0007 to remove the EXPORT_SYMBOL all together.

> >  #endif /* CONFIG_NFS_V4_1 */
> > diff --git a/net/sunrpc/bc_svc.c b/net/sunrpc/bc_svc.c
> > new file mode 100644
> > index 0000000..b13f51d
> > --- /dev/null
> > +++ b/net/sunrpc/bc_svc.c
> > @@ -0,0 +1,80 @@
> >
>
+/**********************************************************************
**
> ******
> > +
> > +(c) 2007 Network Appliance, Inc.  All Rights Reserved.
> 
> More references to historic entities...
> 

We'll update.

- ricardo

> > +
> > +Network Appliance provides this source code under the GPL v2
License.
> > +The GPL v2 license is available at
> > +http://opensource.org/licenses/gpl-license.php.
> > +
> > +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> > +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> > +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR
> > +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT
> OWNER OR
> > +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL,
> > +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
> > +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
> > +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
OF
> > +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING
> > +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
> > +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> > +
> >
>
+***********************************************************************
**
> *****/
> > +
> > +/*
> > + * The NFSv4.1 callback service helper routines.
> > + * They implement the transport level processing required to send
the
> > + * reply over an existing open connection previously established by
the
> client.
> > + */
> > +
> > +#if defined(CONFIG_NFS_V4_1)
> > +
> > +#include <linux/module.h>
> > +
> > +#include <linux/sunrpc/xprt.h>
> > +#include <linux/sunrpc/sched.h>
> > +#include <linux/sunrpc/bc_xprt.h>
> > +
> > +#define RPCDBG_FACILITY	RPCDBG_SVCDSP
> > +
> > +void bc_release_request(struct rpc_task *task)
> > +{
> > +	struct rpc_rqst *req = task->tk_rqstp;
> > +
> > +	dprintk("RPC:       bc_release_request: task= %p\n", task);
> > +
> > +	/*
> > +	 * Release this request only if it's a backchannel
> > +	 * preallocated request
> > +	 */
> > +	if (!bc_prealloc(req))
> > +		return;
> > +	xprt_free_bc_request(req);
> > +}
> > +
> > +/* Empty callback ops */
> > +static const struct rpc_call_ops nfs41_callback_ops = {
> > +};
> > +
> > +
> > +/*
> > + * Send the callback reply
> > + */
> > +int bc_send(struct rpc_rqst *req)
> > +{
> > +	struct rpc_task *task;
> > +	int ret;
> > +
> > +	dprintk("RPC:       bc_send req= %p\n", req);
> > +	task = rpc_run_bc_task(req, &nfs41_callback_ops);
> > +	if (IS_ERR(task))
> > +		ret = PTR_ERR(task);
> > +	else {
> > +		BUG_ON(atomic_read(&task->tk_count) != 1);
> > +		ret = task->tk_status;
> > +		rpc_put_task(task);
> > +	}
> > +	return ret;
> > +	dprintk("RPC:       bc_send ret= %d \n", ret);
> > +}
> > +
> > +#endif /* CONFIG_NFS_V4_1 */
> > diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
> > index 403ebda..5bf9e66 100644
> > --- a/net/sunrpc/xprtsock.c
> > +++ b/net/sunrpc/xprtsock.c
> > @@ -2145,6 +2145,9 @@ static struct rpc_xprt_ops xs_tcp_ops = {
> >  	.buf_free		= rpc_free,
> >  	.send_request		= xs_tcp_send_request,
> >  	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
> > +#if defined(CONFIG_NFS_V4_1)
> > +	.release_request	= bc_release_request,
> > +#endif /* CONFIG_NFS_V4_1 */
> >  	.close			= xs_tcp_shutdown,
> >  	.destroy		= xs_destroy,
> >  	.print_stats		= xs_tcp_print_stats,
> 
> --
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

RE: [pnfs] [RFC 10/39] nfs41: Add backchannel processing support toRPC state machine

[Labiaga, Ricardo]

> -----Original Message-----
> From: Labiaga, Ricardo
> Sent: Friday, June 05, 2009 10:44 AM
> To: Myklebust, Trond; Benny Halevy
> Cc: linux-nfs@vger.kernel.org; Adamson, Andy; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 10/39] nfs41: Add backchannel processing
support
> toRPC state machine
> 
> > -----Original Message-----
> > From: Myklebust, Trond
> > Sent: Thursday, June 04, 2009 12:55 PM
> > To: Benny Halevy
> > Cc: Adamson, Andy; linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> > Subject: Re: [pnfs] [RFC 10/39] nfs41: Add backchannel processing
> support
> > to RPC state machine
> >
> > On Fri, 2009-05-01 at 02:20 +0300, Benny Halevy wrote:
> > > From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > >
> > > Adds rpc_run_bc_task() which is called by the NFS callback service
> to
> > > process backchannel requests.  It performs similar work to
> > rpc_run_task()
> > > though "schedules" the backchannel task to be executed starting at
> the
> > > call_trasmit state in the RPC state machine.
> > >
> > > It also introduces some miscellaneous updates to the argument
> > validation,
> > > call_transmit, and transport cleanup functions to take into
account
> > > that there are now forechannel and backchannel tasks.
> > >
> > > Backchannel requests do not carry an RPC message structure, since
> the
> > > payload has already been XDR encoded using the existing NFSv4
> callback
> > > mechanism.
> > >
> > > Introduce a new transmit state for the client to reply on to
> backchannel
> > > requests.  This new state simply reserves the transport and issues
> the
> > > reply.  In case of a connection related error, disconnects the
> transport
> > and
> > > drops the reply.  It requires the forechannel to re-establish the
> > connection
> > > and the server to retransmit the request, as stated in NFSv4.1
> section
> > > 2.9.2 "Client and Server Transport Behavior".
> > >
> > > Note: There is no need to loop attempting to reserve the
transport.
> If
> > EAGAIN
> > > is returned by xprt_prepare_transmit(), return with tk_status ==
0,
> > > setting tk_action to call_bc_transmit.  rpc_execute() will invoke
it
> > again
> > > after the task is taken off the sleep queue.
> > >
> > > [nfs41: rpc_run_bc_task() need not be exported outside RPC module]
> > > [nfs41: New call_bc_transmit RPC state]
> > > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > > [nfs41: Backchannel: No need to loop in call_bc_transmit()]
> > > Signed-off-by: Andy Adamson <andros@netapp.com>
> > > Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> > > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > > ---
> > >  include/linux/sunrpc/sched.h |    2 +
> > >  include/linux/sunrpc/xprt.h  |   12 ++++
> > >  net/sunrpc/clnt.c            |  117
> > +++++++++++++++++++++++++++++++++++++++++-
> > >  net/sunrpc/stats.c           |    6 ++-
> > >  net/sunrpc/sunrpc.h          |   35 +++++++++++++
> > >  net/sunrpc/xprt.c            |   36 +++++++++++--
> > >  6 files changed, 199 insertions(+), 9 deletions(-)
> > >  create mode 100644 net/sunrpc/sunrpc.h
> > >
> > > diff --git a/include/linux/sunrpc/sched.h
> b/include/linux/sunrpc/sched.h
> > > index 1773768..4010977 100644
> > > --- a/include/linux/sunrpc/sched.h
> > > +++ b/include/linux/sunrpc/sched.h
> > > @@ -210,6 +210,8 @@ struct rpc_wait_queue {
> > >   */
> > >  struct rpc_task *rpc_new_task(const struct rpc_task_setup *);
> > >  struct rpc_task *rpc_run_task(const struct rpc_task_setup *);
> > > +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> > > +				const struct rpc_call_ops *ops);
> > >  void		rpc_put_task(struct rpc_task *);
> > >  void		rpc_exit_task(struct rpc_task *);
> > >  void		rpc_release_calldata(const struct rpc_call_ops
> *, void *);
> > > diff --git a/include/linux/sunrpc/xprt.h
> b/include/linux/sunrpc/xprt.h
> > > index 6b37724..1531abe 100644
> > > --- a/include/linux/sunrpc/xprt.h
> > > +++ b/include/linux/sunrpc/xprt.h
> > > @@ -215,6 +215,18 @@ struct rpc_xprt {
> > >  						/* buffer in use */
> > >  #endif /* CONFIG_NFS_V4_1 */
> > >
> > > +#if defined(CONFIG_NFS_V4_1)
> > > +static inline int bc_prealloc(struct rpc_rqst *req)
> > > +{
> > > +	return test_bit(RPC_BC_PA_IN_USE, &req->rq_bc_pa_state);
> > > +}
> > > +#else
> > > +static inline int bc_prealloc(struct rpc_rqst *req)
> > > +{
> > > +	return 0;
> > > +}
> > > +#endif /* CONFIG_NFS_V4_1 */
> > > +
> > >  struct xprt_create {
> > >  	int			ident;		/* XPRT_TRANSPORT
> identifier */
> > >  	struct sockaddr *	srcaddr;	/* optional local
> address */
> > > diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
> > > index 76d7d46..349b4d6 100644
> > > --- a/net/sunrpc/clnt.c
> > > +++ b/net/sunrpc/clnt.c
> > > @@ -36,7 +36,9 @@
> > >  #include <linux/sunrpc/clnt.h>
> > >  #include <linux/sunrpc/rpc_pipe_fs.h>
> > >  #include <linux/sunrpc/metrics.h>
> > > +#include <linux/sunrpc/bc_xprt.h>
> > >
> > > +#include "sunrpc.h"
> > >
> > >  #ifdef RPC_DEBUG
> > >  # define RPCDBG_FACILITY	RPCDBG_CALL
> > > @@ -63,6 +65,9 @@ static void	call_decode(struct rpc_task
> *task);
> > >  static void	call_bind(struct rpc_task *task);
> > >  static void	call_bind_status(struct rpc_task *task);
> > >  static void	call_transmit(struct rpc_task *task);
> > > +#if defined(CONFIG_NFS_V4_1)
> > > +static void	call_bc_transmit(struct rpc_task *task);
> > > +#endif /* CONFIG_NFS_V4_1 */
> > >  static void	call_status(struct rpc_task *task);
> > >  static void	call_transmit_status(struct rpc_task *task);
> > >  static void	call_refresh(struct rpc_task *task);
> > > @@ -613,6 +618,50 @@ rpc_call_async(struct rpc_clnt *clnt, const
> struct
> > rpc_message *msg, int flags,
> > >  }
> > >  EXPORT_SYMBOL_GPL(rpc_call_async);
> > >
> > > +#if defined(CONFIG_NFS_V4_1)
> > > +/**
> > > + * rpc_run_bc_task - Allocate a new RPC task for backchannel use,
> then
> > run
> > > + * rpc_execute against it
> > > + * @ops: RPC call ops
> > > + */
> > > +struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req,
> > > +					const struct rpc_call_ops
> *tk_ops)
> > > +{
> > > +	struct rpc_task *task;
> > > +	struct xdr_buf *xbufp = &req->rq_snd_buf;
> > > +	struct rpc_task_setup task_setup_data = {
> > > +		.callback_ops = tk_ops,
> > > +	};
> > > +
> > > +	dprintk("RPC: rpc_run_bc_task req= %p\n", req);
> > > +	/*
> > > +	 * Create an rpc_task to send the data
> > > +	 */
> > > +	task = rpc_new_task(&task_setup_data);
> > > +	if (!task) {
> > > +		xprt_free_bc_request(req);
> > > +		goto out;
> > > +	}
> > > +	task->tk_rqstp = req;
> > > +
> > > +	/*
> > > +	 * Set up the xdr_buf length.
> > > +	 * This also indicates that the buffer is XDR encoded already.
> > > +	 */
> > > +	xbufp->len = xbufp->head[0].iov_len + xbufp->page_len +
> > > +			xbufp->tail[0].iov_len;
> > > +
> > > +	task->tk_action = call_bc_transmit;
> > > +	atomic_inc(&task->tk_count);
> > > +	BUG_ON(atomic_read(&task->tk_count) != 2);
> > > +	rpc_execute(task);
> > > +
> > > +out:
> > > +	dprintk("RPC: rpc_run_bc_task: task= %p\n", task);
> > > +	return task;
> > > +}
> > > +#endif /* CONFIG_NFS_V4_1 */
> > > +
> > >  void
> > >  rpc_call_start(struct rpc_task *task)
> > >  {
> > > @@ -1098,7 +1147,7 @@ call_transmit(struct rpc_task *task)
> > >  	 * in order to allow access to the socket to other RPC requests.
> > >  	 */
> > >  	call_transmit_status(task);
> > > -	if (task->tk_msg.rpc_proc->p_decode != NULL)
> > > +	if (rpc_reply_expected(task))
> > >  		return;
> > >  	task->tk_action = rpc_exit_task;
> > >  	rpc_wake_up_queued_task(&task->tk_xprt->pending, task);
> > > @@ -1133,6 +1182,72 @@ call_transmit_status(struct rpc_task *task)
> > >  	}
> > >  }
> > >
> > > +#if defined(CONFIG_NFS_V4_1)
> > > +/*
> > > + * 5b.	Send the backchannel RPC reply.  On error, drop the
> reply.  In
> > > + * addition, disconnect on connectivity errors.
> > > + */
> > > +static void
> > > +call_bc_transmit(struct rpc_task *task)
> > > +{
> > > +	struct rpc_rqst *req = task->tk_rqstp;
> > > +
> > > +	BUG_ON(task->tk_status != 0);
> > > +	task->tk_status = xprt_prepare_transmit(task);
> > > +	if (task->tk_status == -EAGAIN) {
> > > +		/*
> > > +		 * Could not reserve the transport. Try again after the
> > > +		 * transport is released.
> > > +		 */
> > > +		task->tk_status = 0;
> > > +		task->tk_action = call_bc_transmit;
> > > +		return;
> > > +	}
> > > +
> > > +	task->tk_action = rpc_exit_task;
> > > +	if (task->tk_status < 0) {
> > > +		printk(KERN_NOTICE "RPC: Could not send backchannel
> reply "
> > > +			"error: %d\n", task->tk_status);
> > > +		return;
> > > +	}
> > > +
> > > +	xprt_transmit(task);
> > > +	xprt_end_transmit(task);
> > > +	dprint_status(task);
> > > +	switch (task->tk_status) {
> > > +	case 0:
> > > +		/* Success */
> > > +		break;
> > > +	case -EHOSTDOWN:
> > > +	case -EHOSTUNREACH:
> > > +	case -ENETUNREACH:
> > > +	case -ETIMEDOUT:
> > > +		/*
> > > +		 * Problem reaching the server.  Disconnect and let the
> > > +		 * forechannel reestablish the connection.  The server
> will
> > > +		 * have to retransmit the backchannel request and we'll
> > > +		 * reprocess it.  Since these ops are idempotent,
> there's no
> > > +		 * need to cache our reply at this time.
> > > +		 */
> > > +		printk(KERN_NOTICE "RPC: Could not send backchannel
> reply "
> > > +			"error: %d\n", task->tk_status);
> > > +		xprt_conditional_disconnect(task->tk_xprt,
> > > +			req->rq_connect_cookie);
> > > +		break;
> > > +	default:
> > > +		/*
> > > +		 * We were unable to reply and will have to drop the
> > > +		 * request.  The server should reconnect and retransmit.
> > > +		 */
> > > +		BUG_ON(task->tk_status == -EAGAIN);
> > > +		printk(KERN_NOTICE "RPC: Could not send backchannel
> reply "
> > > +			"error: %d\n", task->tk_status);
> > > +		break;
> > > +	}
> > > +	rpc_wake_up_queued_task(&req->rq_xprt->pending, task);
> > > +}
> > > +#endif /* CONFIG_NFS_V4_1 */
> > > +
> > >  /*
> > >   * 6.	Sort out the RPC call status
> > >   */
> > > diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
> > > index 1ef6e46..a0e3d97 100644
> > > --- a/net/sunrpc/stats.c
> > > +++ b/net/sunrpc/stats.c
> > > @@ -141,12 +141,14 @@ EXPORT_SYMBOL_GPL(rpc_free_iostats);
> > >  void rpc_count_iostats(struct rpc_task *task)
> > >  {
> > >  	struct rpc_rqst *req = task->tk_rqstp;
> > > -	struct rpc_iostats *stats = task->tk_client->cl_metrics;
> > > +	struct rpc_iostats *stats;
> > >  	struct rpc_iostats *op_metrics;
> > >  	long rtt, execute, queue;
> > >
> > > -	if (!stats || !req)
> > > +	if (!task->tk_client || task->tk_client->cl_metrics || !req)
> >
> > When are we going to have a tk_client without metrics?
> >
> 
> The original code checked for stats being non-null, so we're doing the
> same thing.  Having said that, I don't see when cl_metrics could be
null
> either, since rpc_new_client() guarantees its allocation.
> 
> I can remove it, but it's not strictly due to functionality in this
> patch.

I mis-read the patch.  We are indeed doing the wrong thing.  Should be
taking the early exit if !task->tk_client->cl_metrics instead.  I'll fix
it.

- ricardo

> > >  		return;
> > > +
> > > +	stats = task->tk_client->cl_metrics;
> > >  	op_metrics = &stats[task->tk_msg.rpc_proc->p_statidx];
> > >
> > >  	op_metrics->om_ops++;
> > > diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h
> > > new file mode 100644
> > > index 0000000..b462de4
> > > --- /dev/null
> > > +++ b/net/sunrpc/sunrpc.h
> > > @@ -0,0 +1,35 @@
> > >
> >
>
+/**********************************************************************
> **
> > ******
> > > +
> > > +(c) 2008 Network Appliance, Inc.  All Rights Reserved.
> > > +
> >
> > Ditto...
> 
> Will fix.
> 
> > > +Network Appliance provides this source code under the GPL v2
> License.
> > > +The GPL v2 license is available at
> > > +http://opensource.org/licenses/gpl-license.php.
> > > +
> > > +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
CONTRIBUTORS
> > > +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> > > +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
> FOR
> > > +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT
> > OWNER OR
> > > +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> SPECIAL,
> > > +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO,
> > > +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
OR
> > > +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY
> OF
> > > +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> (INCLUDING
> > > +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS
> > > +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> > > +
> > >
> >
>
+***********************************************************************
> **
> > *****/
> > > +
> > > +/*
> > > + * Functions and macros used internally by RPC
> > > + */
> > > +
> > > +#ifndef _NET_SUNRPC_SUNRPC_H
> > > +#define _NET_SUNRPC_SUNRPC_H
> > > +
> > > +#define rpc_reply_expected(task) \
> > > +	(((task)->tk_msg.rpc_proc != NULL) && \
> > > +	((task)->tk_msg.rpc_proc->p_decode != NULL))
> >
> > Why is this a macro instead of being an inlined function?
> >
> 
> My fault, I was still getting acquainted with the difference between
> macros and inlining.  Will change to inlined function.
> 
> > > +
> > > +#endif /* _NET_SUNRPC_SUNRPC_H */
> > > +
> > > diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
> > > index bbaec23..df65d15 100644
> > > --- a/net/sunrpc/xprt.c
> > > +++ b/net/sunrpc/xprt.c
> > > @@ -12,8 +12,9 @@
> > >   *  -	Next, the caller puts together the RPC message, stuffs
> it into
> > >   *	the request struct, and calls xprt_transmit().
> > >   *  -	xprt_transmit sends the message and installs the caller
> on the
> > > - *	transport's wait list. At the same time, it installs a
timer
> > that
> > > - *	is run after the packet's timeout has expired.
> > > + *	transport's wait list. At the same time, if a reply is
expected,
> > > + *	it installs a timer that is run after the packet's
timeout has
> > > + *	expired.
> > >   *  -	When a packet arrives, the data_ready handler walks the
> list of
> > >   *	pending requests for that transport. If a matching XID
is found,
> > the
> > >   *	caller is woken up, and the timer removed.
> > > @@ -46,6 +47,8 @@
> > >  #include <linux/sunrpc/clnt.h>
> > >  #include <linux/sunrpc/metrics.h>
> > >
> > > +#include "sunrpc.h"
> > > +
> > >  /*
> > >   * Local variables
> > >   */
> > > @@ -875,7 +878,10 @@ void xprt_transmit(struct rpc_task *task)
> > >  	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid,
> req->rq_slen);
> > >
> > >  	if (!req->rq_received) {
> > > -		if (list_empty(&req->rq_list)) {
> > > +		if (list_empty(&req->rq_list) &&
> rpc_reply_expected(task)) {
> > > +			/*
> > > +			 * Add to the list only if we're expecting a
> reply
> > > +			 */
> > >  			spin_lock_bh(&xprt->transport_lock);
> > >  			/* Update the softirq receive buffer */
> > >  			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
> > > @@ -910,8 +916,13 @@ void xprt_transmit(struct rpc_task *task)
> > >  	/* Don't race with disconnect */
> > >  	if (!xprt_connected(xprt))
> > >  		task->tk_status = -ENOTCONN;
> > > -	else if (!req->rq_received)
> > > +	else if (!req->rq_received && rpc_reply_expected(task)) {
> > > +		/*
> > > +		 * Sleep on the pending queue since
> > > +		 * we're expecting a reply.
> > > +		 */
> > >  		rpc_sleep_on(&xprt->pending, task, xprt_timer);
> > > +	}
> > >  	spin_unlock_bh(&xprt->transport_lock);
> > >  }
> > >
> > > @@ -984,11 +995,15 @@ static void xprt_request_init(struct
rpc_task
> > *task, struct rpc_xprt *xprt)
> > >   */
> > >  void xprt_release(struct rpc_task *task)
> > >  {
> > > -	struct rpc_xprt	*xprt = task->tk_xprt;
> > > +	struct rpc_xprt	*xprt;
> > >  	struct rpc_rqst	*req;
> > > +	int prealloc;
> > >
> > > +	BUG_ON(atomic_read(&task->tk_count) < 0);
> >
> > Err?
> >
> 
> This was useful during early development.  Never hit it though :-)  So
> I'll remove it.
> 
> 
> > >  	if (!(req = task->tk_rqstp))
> > >  		return;
> > > +	prealloc = bc_prealloc(req);	/* Preallocated backchannel
> request? */
> > > +	xprt = req->rq_xprt;
> > >  	rpc_count_iostats(task);
> > >  	spin_lock_bh(&xprt->transport_lock);
> > >  	xprt->ops->release_xprt(xprt, task);
> > > @@ -1001,10 +1016,19 @@ void xprt_release(struct rpc_task *task)
> > >  		mod_timer(&xprt->timer,
> > >  				xprt->last_used + xprt->idle_timeout);
> > >  	spin_unlock_bh(&xprt->transport_lock);
> > > -	xprt->ops->buf_free(req->rq_buffer);
> > > +	if (!bc_prealloc(req))
> > > +		xprt->ops->buf_free(req->rq_buffer);
> > >  	task->tk_rqstp = NULL;
> > >  	if (req->rq_release_snd_buf)
> > >  		req->rq_release_snd_buf(req);
> > > +
> > > +	/*
> > > +	 * Early exit if this is a backchannel preallocated request.
> > > +	 * There is no need to have it added to the RPC slot list.
> > > +	 */
> > > +	if (prealloc)
> > > +		return;
> >
> > Could we change the name of 'prealloc' to something along the lines
of
> > 'is_bc_request'?
> >
> 
> Yes, will do.
> 

I removed the variable altogether.  Calling bc_prealloc() directly since
it's now an inline function that checks if a bit is set.

- ricardo


> - ricardo
> 
> > > +
> > >  	memset(req, 0, sizeof(*req));	/* mark unused */
> > >
> > >  	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
> >
> > --
> > Trond Myklebust
> > Linux NFS client maintainer
> >
> > NetApp
> > Trond.Myklebust@netapp.com
> > www.netapp.com
> > _______________________________________________
> > pNFS mailing list
> > pNFS@linux-nfs.org
> > http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs

Re: [pnfs] [RFC 25/39] nfs41: decode minorversion 1 cb_compound header

[Trond Myklebust]

On Fri, 2009-05-01 at 02:23 +0300, Benny Halevy wrote:
> decode cb_compound header conforming to
> http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion1-26
> 
> Get rid of cb_compound_hdr_arg.callback_ident
> 
> callback_ident is not used anywhere so we shouldn't waste any memory to
> store it.
> 
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback.h     |    1 -
>  fs/nfs/callback_xdr.c |   12 ++++++++----
>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> index 21881d9..e0f4e6b 100644
> --- a/fs/nfs/callback.h
> +++ b/fs/nfs/callback.h
> @@ -27,7 +27,6 @@ struct cb_compound_hdr_arg {
>  	unsigned int taglen;
>  	const char *tag;
>  	unsigned int minorversion;
> -	unsigned int callback_ident;
>  	unsigned nops;
>  };
>  
> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> index 91f6f74..e141a7e 100644
> --- a/fs/nfs/callback_xdr.c
> +++ b/fs/nfs/callback_xdr.c
> @@ -143,18 +143,22 @@ static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound
>  				__func__, hdr->taglen);
>  		return htonl(NFS4ERR_RESOURCE);
>  	}
> -	p = read_buf(xdr, 12);
> +	p = read_buf(xdr, 4);
>  	if (unlikely(p == NULL))
>  		return htonl(NFS4ERR_RESOURCE);
>  	hdr->minorversion = ntohl(*p++);
> -	/* Check minor version is zero. */
> -	if (hdr->minorversion != 0) {
> +	/* Check minor version is zero or one. */
> +	if (hdr->minorversion <= 1) {
> +		p = read_buf(xdr, 8);

What is the point of splitting the read_buf() here? That seems very
unnecessary...

> +		if (unlikely(p == NULL))
> +			return htonl(NFS4ERR_RESOURCE);
> +		p++;	/* skip callback_ident */
> +	} else {
>  		printk(KERN_WARNING "%s: NFSv4 server callback with "
>  			"illegal minor version %u!\n",
>  			__func__, hdr->minorversion);
>  		return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
>  	}
> -	hdr->callback_ident = ntohl(*p++);
>  	hdr->nops = ntohl(*p);
>  	dprintk("%s: minorversion %d nops %d\n", __func__,
>  		hdr->minorversion, hdr->nops);

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 27/39] nfs41: consider minorversion in callback_xdr:process_op

[Trond Myklebust]

On Fri, 2009-05-01 at 02:23 +0300, Benny Halevy wrote:
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback_xdr.c |   63 ++++++++++++++++++++++++++++++++++++++----------
>  1 files changed, 50 insertions(+), 13 deletions(-)
> 
> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> index e141a7e..dd25794 100644
> --- a/fs/nfs/callback_xdr.c
> +++ b/fs/nfs/callback_xdr.c
> @@ -359,31 +359,61 @@ out:
>  	return status;
>  }
>  
> -static __be32 process_op(struct svc_rqst *rqstp,
> +static __be32 process_op(uint32_t minorversion, int nop,
> +		struct svc_rqst *rqstp,
>  		struct xdr_stream *xdr_in, void *argp,
>  		struct xdr_stream *xdr_out, void *resp)
>  {
>  	struct callback_op *op = &callback_ops[0];
>  	unsigned int op_nr = OP_CB_ILLEGAL;
> -	__be32 status = 0;
> +	__be32 status;
>  	long maxlen;
>  	__be32 res;
>  
>  	dprintk("%s: start\n", __func__);
>  	status = decode_op_hdr(xdr_in, &op_nr);
> -	if (likely(status == 0)) {
> +	if (unlikely(status))
> +		goto out_illegal;
> +
> +	dprintk("%s: minorversion=%d nop=%d op_nr=%u\n",
> +		__func__, minorversion, nop, op_nr);
> +#if defined(CONFIG_NFS_V4_1)
> +	if (minorversion == 1) {
>  		switch (op_nr) {
> -			case OP_CB_GETATTR:
> -			case OP_CB_RECALL:
> -				op = &callback_ops[op_nr];
> -				break;
> -			default:
> -				op_nr = OP_CB_ILLEGAL;
> -				op = &callback_ops[0];
> -				status = htonl(NFS4ERR_OP_ILLEGAL);
> +		case OP_CB_GETATTR:
> +		case OP_CB_RECALL:
> +			op = &callback_ops[op_nr];
> +			break;
> +
> +		case OP_CB_LAYOUTRECALL:
> +		case OP_CB_NOTIFY:
> +		case OP_CB_PUSH_DELEG:
> +		case OP_CB_RECALL_ANY:
> +		case OP_CB_RECALLABLE_OBJ_AVAIL:
> +		case OP_CB_RECALL_SLOT:
> +		case OP_CB_SEQUENCE:
> +		case OP_CB_WANTS_CANCELLED:
> +		case OP_CB_NOTIFY_LOCK:
> +			op = &callback_ops[0];
> +			status = htonl(NFS4ERR_NOTSUPP);
> +			break;
> +		default:
> +			goto out_illegal;
>  		}
> -	}
>  
> +		goto out;
> +	}
> +#endif /* defined(CONFIG_NFS_V4_1) */

Please move into a separate function. 

> +
> +	switch (op_nr) {
> +	case OP_CB_GETATTR:
> +	case OP_CB_RECALL:
> +		op = &callback_ops[op_nr];
> +		break;
> +	default:
> +		goto out_illegal;
> +	}

Ditto...

> +out:
>  	maxlen = xdr_out->end - xdr_out->p;
>  	if (maxlen > 0 && maxlen < PAGE_SIZE) {
>  		if (likely(status == 0 && op->decode_args != NULL))
> @@ -400,6 +430,12 @@ static __be32 process_op(struct svc_rqst *rqstp,
>  		status = op->encode_res(rqstp, xdr_out, resp);
>  	dprintk("%s: done, status = %d\n", __func__, ntohl(status));
>  	return status;
> +
> +out_illegal:
> +	op_nr = OP_CB_ILLEGAL;
> +	op = &callback_ops[0];
> +	status = htonl(NFS4ERR_OP_ILLEGAL);
> +	goto out;
>  }
>  
>  /*
> @@ -431,7 +467,8 @@ static __be32 nfs4_callback_compound(struct svc_rqst *rqstp, void *argp, void *r
>  		return rpc_system_err;
>  
>  	while (status == 0 && nops != hdr_arg.nops) {
> -		status = process_op(rqstp, &xdr_in, argp, &xdr_out, resp);
> +		status = process_op(hdr_arg.minorversion, nops,
> +				    rqstp, &xdr_in, argp, &xdr_out, resp);
>  		nops++;
>  	}
>  

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 31/39] nfs41: cb_sequence xdr implementation

[Trond Myklebust]

On Fri, 2009-05-01 at 02:24 +0300, Benny Halevy wrote:
> [nfs41: get rid of READMEM and COPYMEM for callback_xdr.c]
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback_xdr.c |  181 ++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 files changed, 179 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> index 9db88ce..066273c 100644
> --- a/fs/nfs/callback_xdr.c
> +++ b/fs/nfs/callback_xdr.c
> @@ -20,8 +20,18 @@
>  				2 + 2 + 3 + 3)
>  #define CB_OP_RECALL_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ)
>  
> +#if defined(CONFIG_NFS_V4_1)
> +#define CB_OP_SEQUENCE_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ + \
> +					4 + 1 + 3)
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  #define NFSDBG_FACILITY NFSDBG_CALLBACK
>  
> +#define READ64(x)         do {			\
> +	(x) = (u64)ntohl(*p++) << 32;		\
> +	(x) |= ntohl(*p++);			\
> +} while (0)

This doesn't appear to be used, and in any case, it duplicates the
existing function xdr_duplicate_hyper(). Please remove...

> +
>  typedef __be32 (*callback_process_op_t)(void *, void *);
>  typedef __be32 (*callback_decode_arg_t)(struct svc_rqst *, struct xdr_stream *, void *);
>  typedef __be32 (*callback_encode_res_t)(struct svc_rqst *, struct xdr_stream *, void *);
> @@ -210,6 +220,122 @@ out:
>  	return status;
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +
> +static unsigned decode_sessionid(struct xdr_stream *xdr,
> +				 struct nfs4_sessionid *sid)
> +{
> +	uint32_t *p;
> +	int len = NFS4_MAX_SESSIONID_LEN;
> +
> +	p = read_buf(xdr, len);
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);;
> +
> +	memcpy(sid->data, p, len);
> +	return 0;
> +}
> +
> +static unsigned decode_rc_list(struct xdr_stream *xdr,
> +			       struct referring_call_list *rc_list)
> +{
> +	uint32_t *p;
> +	int i;
> +	unsigned status;
> +
> +	status = decode_sessionid(xdr, &rc_list->rcl_sessionid);
> +	if (status)
> +		goto out;
> +
> +	status = htonl(NFS4ERR_RESOURCE);
> +	p = read_buf(xdr, sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		goto out;
> +
> +	rc_list->rcl_nrefcalls = ntohl(*p++);
> +	if (rc_list->rcl_nrefcalls) {
> +		p = read_buf(xdr,
> +			     rc_list->rcl_nrefcalls * 2 * sizeof(uint32_t));
> +		if (unlikely(p == NULL))
> +			goto out;
> +		rc_list->rcl_refcalls = kmalloc(rc_list->rcl_nrefcalls *
> +						sizeof(*rc_list->rcl_refcalls),
> +						GFP_KERNEL);
> +		if (unlikely(rc_list->rcl_refcalls == NULL))
> +			goto out;
> +		for (i = 0; i < rc_list->rcl_nrefcalls; i++) {
> +			rc_list->rcl_refcalls[i].rc_sequenceid = ntohl(*p++);
> +			rc_list->rcl_refcalls[i].rc_slotid = ntohl(*p++);
> +		}
> +	}
> +	status = 0;
> +
> +out:
> +	return status;
> +}
> +
> +static unsigned decode_cb_sequence_args(struct svc_rqst *rqstp,
> +					struct xdr_stream *xdr,
> +					struct cb_sequenceargs *args)
> +{
> +	uint32_t *p;
> +	int i;
> +	unsigned status;
> +
> +	status = decode_sessionid(xdr, &args->csa_sessionid);
> +	if (status)
> +		goto out;
> +
> +	status = htonl(NFS4ERR_RESOURCE);
> +	p = read_buf(xdr, 5 * sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		goto out;
> +
> +	args->csa_addr = svc_addr_in(rqstp);
> +	args->csa_sequenceid = ntohl(*p++);
> +	args->csa_slotid = ntohl(*p++);
> +	args->csa_highestslotid = ntohl(*p++);
> +	args->csa_cachethis = ntohl(*p++);
> +	args->csa_nrclists = ntohl(*p++);
> +	args->csa_rclists = NULL;
> +	if (args->csa_nrclists) {
> +		args->csa_rclists = kmalloc(args->csa_nrclists *
> +					    sizeof(*args->csa_rclists),
> +					    GFP_KERNEL);
> +		if (unlikely(args->csa_rclists == NULL))
> +			goto out;
> +
> +		for (i = 0; i < args->csa_nrclists; i++) {
> +			status = decode_rc_list(xdr, &args->csa_rclists[i]);
> +			if (status)
> +				goto out_free;
> +		}
> +	}
> +	status = 0;
> +
> +	dprintk("%s: sessionid %x:%x:%x:%x sequenceid %u slotid %u "
> +		"highestslotid %u cachethis %d nrclists %u\n",
> +		__func__,
> +		((u32 *)&args->csa_sessionid)[0],
> +		((u32 *)&args->csa_sessionid)[1],
> +		((u32 *)&args->csa_sessionid)[2],
> +		((u32 *)&args->csa_sessionid)[3],
> +		args->csa_sequenceid, args->csa_slotid,
> +		args->csa_highestslotid, args->csa_cachethis,
> +		args->csa_nrclists);
> +out:
> +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> +	return status;
> +
> +out_free:
> +	for (i = 0; i < args->csa_nrclists; i++)
> +		kfree(args->csa_rclists[i].rcl_refcalls);
> +	kfree(args->csa_rclists);
> +	goto out;
> +}
> +
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char *str)
>  {
>  	__be32 *p;
> @@ -359,6 +485,49 @@ out:
>  	return status;
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +
> +static unsigned encode_sessionid(struct xdr_stream *xdr,
> +				 const struct nfs4_sessionid *sid)
> +{
> +	uint32_t *p;
> +	int len = NFS4_MAX_SESSIONID_LEN;
> +
> +	p = xdr_reserve_space(xdr, len);
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);
> +
> +	memcpy(p, sid, len);
> +	return 0;
> +}
> +
> +static unsigned encode_cb_sequence_res(struct svc_rqst *rqstp,
> +				       struct xdr_stream *xdr,
> +				       const struct cb_sequenceres *res)
> +{
> +	uint32_t *p;
> +	unsigned status = res->csr_status;
> +
> +	if (unlikely(status != 0))
> +		goto out;
> +
> +	encode_sessionid(xdr, &res->csr_sessionid);
> +
> +	p = xdr_reserve_space(xdr, 4 * sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);
> +
> +	*p++ = htonl(res->csr_sequenceid);
> +	*p++ = htonl(res->csr_slotid);
> +	*p++ = htonl(res->csr_highestslotid);
> +	*p++ = htonl(res->csr_target_highestslotid);
> +out:
> +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> +	return status;
> +}
> +
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  static __be32 process_op(uint32_t minorversion, int nop,
>  		struct svc_rqst *rqstp,
>  		struct xdr_stream *xdr_in, void *argp,
> @@ -382,6 +551,7 @@ static __be32 process_op(uint32_t minorversion, int nop,
>  		switch (op_nr) {
>  		case OP_CB_GETATTR:
>  		case OP_CB_RECALL:
> +		case OP_CB_SEQUENCE:
>  			op = &callback_ops[op_nr];
>  			break;
>  
> @@ -391,7 +561,6 @@ static __be32 process_op(uint32_t minorversion, int nop,
>  		case OP_CB_RECALL_ANY:
>  		case OP_CB_RECALLABLE_OBJ_AVAIL:
>  		case OP_CB_RECALL_SLOT:
> -		case OP_CB_SEQUENCE:
>  		case OP_CB_WANTS_CANCELLED:
>  		case OP_CB_NOTIFY_LOCK:
>  		case OP_CB_NOTIFY_DEVICEID:
> @@ -496,7 +665,15 @@ static struct callback_op callback_ops[] = {
>  		.process_op = (callback_process_op_t)nfs4_callback_recall,
>  		.decode_args = (callback_decode_arg_t)decode_recall_args,
>  		.res_maxsize = CB_OP_RECALL_RES_MAXSZ,
> -	}
> +	},
> +#if defined(CONFIG_NFS_V4_1)
> +	[OP_CB_SEQUENCE] = {
> +		.process_op = (callback_process_op_t)nfs4_callback_sequence,
> +		.decode_args = (callback_decode_arg_t)decode_cb_sequence_args,
> +		.encode_res = (callback_encode_res_t)encode_cb_sequence_res,
> +		.res_maxsize = CB_OP_SEQUENCE_RES_MAXSZ,
> +	},
> +#endif /* CONFIG_NFS_V4_1 */
>  };
>  
>  /*

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 39/39] nfs41: Backchannel: CB_SEQUENCE validation

[Trond Myklebust]

On Fri, 2009-05-01 at 02:25 +0300, Benny Halevy wrote:
> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> 
> Validates the callback's sessionID, the slot number, and the sequence ID.
> Increments the slot's sequence.
> 
> Detects replays, but simply prints a debug message (if debugging is enabled
> since we don't yet implement a duplicate request cache for the backchannel.
> This should not present a problem, since only idempotent callbacks are
> currently implemented.
> 
> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback_proc.c |   75 ++++++++++++++++++++++++++++++++++++++++++++----
>  1 files changed, 69 insertions(+), 6 deletions(-)
> 
> diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
> index 6b342e8..c85d66f 100644
> --- a/fs/nfs/callback_proc.c
> +++ b/fs/nfs/callback_proc.c
> @@ -105,6 +105,57 @@ out:
>  #if defined(CONFIG_NFS_V4_1)
>  
>  /*
> + * Validate the sequenceID sent by the server.
> + * Return success if the sequenceID is one more than what we last saw on
> + * this slot, accounting for wraparound.  Increments the slot's sequence.
> + *
> + * We don't yet implement a duplicate request cache, so at this time
> + * we will log replays, and process them as if we had not seen them before,
> + * but we don't bump the sequence in the slot.  Not too worried about it,
> + * since we only currently implement idempotent callbacks anyway.
> + *
> + * We have a single slot backchannel at this time, so we don't bother
> + * checking the used_slots bit array on the table.  The lower layer guarantees
> + * a single outstanding callback request at a time.
> + */
> +static int
> +validate_seqid(struct nfs4_slot_table *tbl, u32 slotid, u32 seqid)
> +{
> +	struct nfs4_slot *slot;
> +
> +	dprintk("%s enter. slotid %d seqid %d\n",
> +		__func__, slotid, seqid);
> +
> +	if (slotid > NFS41_BC_MAX_CALLBACKS)
> +		return NFS4ERR_BADSLOT;

I thought we agreed to always return error values as _negative_? Why the
change here?

> +
> +	slot = tbl->slots + slotid;
> +	dprintk("%s slot table seqid: %d\n", __func__, slot->seq_nr);
> +
> +	/* Normal */
> +	if (likely(seqid == slot->seq_nr + 1)) {
> +		slot->seq_nr++;
> +		return NFS4_OK;
> +	}
> +
> +	/* Replay */
> +	if (seqid == slot->seq_nr) {
> +		dprintk("%s seqid %d is a replay - no DRC available\n",
> +			__func__, seqid);
> +		return NFS4_OK;
> +	}
> +
> +	/* Wraparound */
> +	if (seqid == 1 && (slot->seq_nr + 1) == 0) {
> +		slot->seq_nr = 1;
> +		return NFS4_OK;
> +	}
> +
> +	/* Misordered request */
> +	return NFS4ERR_SEQ_MISORDERED;
> +}
> +
> +/*
>   * Returns a pointer to a held 'struct nfs_client' that matches the server's
>   * address, major version number, and session ID.  It is the caller's
>   * responsibility to release the returned reference.
> @@ -140,18 +191,27 @@ out:
>  	return NULL;
>  }
>  
> -/* FIXME: validate args->cbs_{sequence,slot}id */
>  /* FIXME: referring calls should be processed */
>  unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
>  				struct cb_sequenceres *res)
>  {
> -	int i;
> -	unsigned status = 0;
> +	struct nfs_client *clp;
> +	int i, status;
>  
>  	for (i = 0; i < args->csa_nrclists; i++)
>  		kfree(args->csa_rclists[i].rcl_refcalls);
>  	kfree(args->csa_rclists);
>  
> +	status = NFS4ERR_BADSESSION;
> +	clp = find_client_with_session(args->csa_addr, 4, &args->csa_sessionid);
> +	if (clp == NULL)
> +		goto out;
> +
> +	status = validate_seqid(&clp->cl_session->bc_slot_table,
> +				args->csa_slotid, args->csa_sequenceid);
> +	if (status)
> +		goto out_putclient;
> +
>  	memcpy(&res->csr_sessionid, &args->csa_sessionid,
>  	       sizeof(res->csr_sessionid));
>  	res->csr_sequenceid = args->csa_sequenceid;
> @@ -159,9 +219,12 @@ unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
>  	res->csr_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>  	res->csr_target_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>  
> -	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> -	res->csr_status = status;
> -	return status;
> +out_putclient:
> +	nfs_put_client(clp);
> +out:
> +	dprintk("%s: exit with status = %d\n", __func__, status);
> +	res->csr_status = htonl(status);
> +	return res->csr_status;
>  }
>  
>  #endif /* CONFIG_NFS_V4_1 */

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 39/39] nfs41: Backchannel: CB_SEQUENCE validation

[Labiaga, Ricardo]

On 6/15/09 5:27 PM, "Trond Myklebust" <Trond.Myklebust@netapp.com> wrote:

> On Fri, 2009-05-01 at 02:25 +0300, Benny Halevy wrote:
>> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
>> 
>> Validates the callback's sessionID, the slot number, and the sequence ID.
>> Increments the slot's sequence.
>> 
>> Detects replays, but simply prints a debug message (if debugging is enabled
>> since we don't yet implement a duplicate request cache for the backchannel.
>> This should not present a problem, since only idempotent callbacks are
>> currently implemented.
>> 
>> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
>> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
>> ---
>>  fs/nfs/callback_proc.c |   75
>> ++++++++++++++++++++++++++++++++++++++++++++----
>>  1 files changed, 69 insertions(+), 6 deletions(-)
>> 
>> diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
>> index 6b342e8..c85d66f 100644
>> --- a/fs/nfs/callback_proc.c
>> +++ b/fs/nfs/callback_proc.c
>> @@ -105,6 +105,57 @@ out:
>>  #if defined(CONFIG_NFS_V4_1)
>>  
>>  /*
>> + * Validate the sequenceID sent by the server.
>> + * Return success if the sequenceID is one more than what we last saw on
>> + * this slot, accounting for wraparound.  Increments the slot's sequence.
>> + *
>> + * We don't yet implement a duplicate request cache, so at this time
>> + * we will log replays, and process them as if we had not seen them before,
>> + * but we don't bump the sequence in the slot.  Not too worried about it,
>> + * since we only currently implement idempotent callbacks anyway.
>> + *
>> + * We have a single slot backchannel at this time, so we don't bother
>> + * checking the used_slots bit array on the table.  The lower layer
>> guarantees
>> + * a single outstanding callback request at a time.
>> + */
>> +static int
>> +validate_seqid(struct nfs4_slot_table *tbl, u32 slotid, u32 seqid)
>> +{
>> + struct nfs4_slot *slot;
>> +
>> + dprintk("%s enter. slotid %d seqid %d\n",
>> +  __func__, slotid, seqid);
>> +
>> + if (slotid > NFS41_BC_MAX_CALLBACKS)
>> +  return NFS4ERR_BADSLOT;
> 
> I thought we agreed to always return error values as _negative_? Why the
> change here?
> 

Will change to negative.

- ricardo

>> +
>> + slot = tbl->slots + slotid;
>> + dprintk("%s slot table seqid: %d\n", __func__, slot->seq_nr);
>> +
>> + /* Normal */
>> + if (likely(seqid == slot->seq_nr + 1)) {
>> +  slot->seq_nr++;
>> +  return NFS4_OK;
>> + }
>> +
>> + /* Replay */
>> + if (seqid == slot->seq_nr) {
>> +  dprintk("%s seqid %d is a replay - no DRC available\n",
>> +   __func__, seqid);
>> +  return NFS4_OK;
>> + }
>> +
>> + /* Wraparound */
>> + if (seqid == 1 && (slot->seq_nr + 1) == 0) {
>> +  slot->seq_nr = 1;
>> +  return NFS4_OK;
>> + }
>> +
>> + /* Misordered request */
>> + return NFS4ERR_SEQ_MISORDERED;
>> +}
>> +
>> +/*
>>   * Returns a pointer to a held 'struct nfs_client' that matches the server's
>>   * address, major version number, and session ID.  It is the caller's
>>   * responsibility to release the returned reference.
>> @@ -140,18 +191,27 @@ out:
>> return NULL;
>>  }
>>  
>> -/* FIXME: validate args->cbs_{sequence,slot}id */
>>  /* FIXME: referring calls should be processed */
>>  unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
>> struct cb_sequenceres *res)
>>  {
>> - int i;
>> - unsigned status = 0;
>> + struct nfs_client *clp;
>> + int i, status;
>>  
>> for (i = 0; i < args->csa_nrclists; i++)
>> kfree(args->csa_rclists[i].rcl_refcalls);
>> kfree(args->csa_rclists);
>>  
>> + status = NFS4ERR_BADSESSION;
>> + clp = find_client_with_session(args->csa_addr, 4, &args->csa_sessionid);
>> + if (clp == NULL)
>> +  goto out;
>> +
>> + status = validate_seqid(&clp->cl_session->bc_slot_table,
>> +    args->csa_slotid, args->csa_sequenceid);
>> + if (status)
>> +  goto out_putclient;
>> +
>> memcpy(&res->csr_sessionid, &args->csa_sessionid,
>>       sizeof(res->csr_sessionid));
>> res->csr_sequenceid = args->csa_sequenceid;
>> @@ -159,9 +219,12 @@ unsigned nfs4_callback_sequence(struct cb_sequenceargs
>> *args,
>> res->csr_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>> res->csr_target_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>>  
>> - dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
>> - res->csr_status = status;
>> - return status;
>> +out_putclient:
>> + nfs_put_client(clp);
>> +out:
>> + dprintk("%s: exit with status = %d\n", __func__, status);
>> + res->csr_status = htonl(status);
>> + return res->csr_status;
>>  }
>>  
>>  #endif /* CONFIG_NFS_V4_1 */

RE: [pnfs] [RFC 25/39] nfs41: decode minorversion 1 cb_compound header

[Halevy, Benny]

> What is the point of splitting the read_buf() here? That seems very
> unnecessary...

Agreed.
I'll revert that.

For the record, my original thinking was that you don't want to read
the rest of the header if you don't support its minorversion.
However, minorversion can't change the compound header's format
so its safe to read it all.

Benny

-----Original Message-----
From: Trond Myklebust [mailto:Trond.Myklebust@netapp.com]
Sent: Tue 2009-06-16 03:13
To: Halevy, Benny
Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
Subject: Re: [pnfs] [RFC 25/39] nfs41: decode minorversion 1 cb_compound header
 
On Fri, 2009-05-01 at 02:23 +0300, Benny Halevy wrote:
> decode cb_compound header conforming to
> http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion1-26
> 
> Get rid of cb_compound_hdr_arg.callback_ident
> 
> callback_ident is not used anywhere so we shouldn't waste any memory to
> store it.
> 
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback.h     |    1 -
>  fs/nfs/callback_xdr.c |   12 ++++++++----
>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h
> index 21881d9..e0f4e6b 100644
> --- a/fs/nfs/callback.h
> +++ b/fs/nfs/callback.h
> @@ -27,7 +27,6 @@ struct cb_compound_hdr_arg {
>  	unsigned int taglen;
>  	const char *tag;
>  	unsigned int minorversion;
> -	unsigned int callback_ident;
>  	unsigned nops;
>  };
>  
> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> index 91f6f74..e141a7e 100644
> --- a/fs/nfs/callback_xdr.c
> +++ b/fs/nfs/callback_xdr.c
> @@ -143,18 +143,22 @@ static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound
>  				__func__, hdr->taglen);
>  		return htonl(NFS4ERR_RESOURCE);
>  	}
> -	p = read_buf(xdr, 12);
> +	p = read_buf(xdr, 4);
>  	if (unlikely(p == NULL))
>  		return htonl(NFS4ERR_RESOURCE);
>  	hdr->minorversion = ntohl(*p++);
> -	/* Check minor version is zero. */
> -	if (hdr->minorversion != 0) {
> +	/* Check minor version is zero or one. */
> +	if (hdr->minorversion <= 1) {
> +		p = read_buf(xdr, 8);

What is the point of splitting the read_buf() here? That seems very
unnecessary...

> +		if (unlikely(p == NULL))
> +			return htonl(NFS4ERR_RESOURCE);
> +		p++;	/* skip callback_ident */
> +	} else {
>  		printk(KERN_WARNING "%s: NFSv4 server callback with "
>  			"illegal minor version %u!\n",
>  			__func__, hdr->minorversion);
>  		return htonl(NFS4ERR_MINOR_VERS_MISMATCH);
>  	}
> -	hdr->callback_ident = ntohl(*p++);
>  	hdr->nops = ntohl(*p);
>  	dprintk("%s: minorversion %d nops %d\n", __func__,
>  		hdr->minorversion, hdr->nops);

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

RE: [pnfs] [RFC 31/39] nfs41: cb_sequence xdr implementation

[Halevy, Benny]

> > +#define READ64(x)         do {			\
> > +	(x) = (u64)ntohl(*p++) << 32;		\
> > +	(x) |= ntohl(*p++);			\
> > +} while (0)
> 
> This doesn't appear to be used, and in any case, it duplicates the

Oops, Will do.

> existing function xdr_duplicate_hyper(). Please remove...

xdr_decode_hyper?

Benny

-----Original Message-----
From: Trond Myklebust [mailto:Trond.Myklebust@netapp.com]
Sent: Tue 2009-06-16 03:18
To: Halevy, Benny
Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
Subject: Re: [pnfs] [RFC 31/39] nfs41: cb_sequence xdr implementation
 
On Fri, 2009-05-01 at 02:24 +0300, Benny Halevy wrote:
> [nfs41: get rid of READMEM and COPYMEM for callback_xdr.c]
> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> ---
>  fs/nfs/callback_xdr.c |  181 ++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 files changed, 179 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> index 9db88ce..066273c 100644
> --- a/fs/nfs/callback_xdr.c
> +++ b/fs/nfs/callback_xdr.c
> @@ -20,8 +20,18 @@
>  				2 + 2 + 3 + 3)
>  #define CB_OP_RECALL_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ)
>  
> +#if defined(CONFIG_NFS_V4_1)
> +#define CB_OP_SEQUENCE_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ + \
> +					4 + 1 + 3)
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  #define NFSDBG_FACILITY NFSDBG_CALLBACK
>  
> +#define READ64(x)         do {			\
> +	(x) = (u64)ntohl(*p++) << 32;		\
> +	(x) |= ntohl(*p++);			\
> +} while (0)

This doesn't appear to be used, and in any case, it duplicates the
existing function xdr_duplicate_hyper(). Please remove...

> +
>  typedef __be32 (*callback_process_op_t)(void *, void *);
>  typedef __be32 (*callback_decode_arg_t)(struct svc_rqst *, struct xdr_stream *, void *);
>  typedef __be32 (*callback_encode_res_t)(struct svc_rqst *, struct xdr_stream *, void *);
> @@ -210,6 +220,122 @@ out:
>  	return status;
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +
> +static unsigned decode_sessionid(struct xdr_stream *xdr,
> +				 struct nfs4_sessionid *sid)
> +{
> +	uint32_t *p;
> +	int len = NFS4_MAX_SESSIONID_LEN;
> +
> +	p = read_buf(xdr, len);
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);;
> +
> +	memcpy(sid->data, p, len);
> +	return 0;
> +}
> +
> +static unsigned decode_rc_list(struct xdr_stream *xdr,
> +			       struct referring_call_list *rc_list)
> +{
> +	uint32_t *p;
> +	int i;
> +	unsigned status;
> +
> +	status = decode_sessionid(xdr, &rc_list->rcl_sessionid);
> +	if (status)
> +		goto out;
> +
> +	status = htonl(NFS4ERR_RESOURCE);
> +	p = read_buf(xdr, sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		goto out;
> +
> +	rc_list->rcl_nrefcalls = ntohl(*p++);
> +	if (rc_list->rcl_nrefcalls) {
> +		p = read_buf(xdr,
> +			     rc_list->rcl_nrefcalls * 2 * sizeof(uint32_t));
> +		if (unlikely(p == NULL))
> +			goto out;
> +		rc_list->rcl_refcalls = kmalloc(rc_list->rcl_nrefcalls *
> +						sizeof(*rc_list->rcl_refcalls),
> +						GFP_KERNEL);
> +		if (unlikely(rc_list->rcl_refcalls == NULL))
> +			goto out;
> +		for (i = 0; i < rc_list->rcl_nrefcalls; i++) {
> +			rc_list->rcl_refcalls[i].rc_sequenceid = ntohl(*p++);
> +			rc_list->rcl_refcalls[i].rc_slotid = ntohl(*p++);
> +		}
> +	}
> +	status = 0;
> +
> +out:
> +	return status;
> +}
> +
> +static unsigned decode_cb_sequence_args(struct svc_rqst *rqstp,
> +					struct xdr_stream *xdr,
> +					struct cb_sequenceargs *args)
> +{
> +	uint32_t *p;
> +	int i;
> +	unsigned status;
> +
> +	status = decode_sessionid(xdr, &args->csa_sessionid);
> +	if (status)
> +		goto out;
> +
> +	status = htonl(NFS4ERR_RESOURCE);
> +	p = read_buf(xdr, 5 * sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		goto out;
> +
> +	args->csa_addr = svc_addr_in(rqstp);
> +	args->csa_sequenceid = ntohl(*p++);
> +	args->csa_slotid = ntohl(*p++);
> +	args->csa_highestslotid = ntohl(*p++);
> +	args->csa_cachethis = ntohl(*p++);
> +	args->csa_nrclists = ntohl(*p++);
> +	args->csa_rclists = NULL;
> +	if (args->csa_nrclists) {
> +		args->csa_rclists = kmalloc(args->csa_nrclists *
> +					    sizeof(*args->csa_rclists),
> +					    GFP_KERNEL);
> +		if (unlikely(args->csa_rclists == NULL))
> +			goto out;
> +
> +		for (i = 0; i < args->csa_nrclists; i++) {
> +			status = decode_rc_list(xdr, &args->csa_rclists[i]);
> +			if (status)
> +				goto out_free;
> +		}
> +	}
> +	status = 0;
> +
> +	dprintk("%s: sessionid %x:%x:%x:%x sequenceid %u slotid %u "
> +		"highestslotid %u cachethis %d nrclists %u\n",
> +		__func__,
> +		((u32 *)&args->csa_sessionid)[0],
> +		((u32 *)&args->csa_sessionid)[1],
> +		((u32 *)&args->csa_sessionid)[2],
> +		((u32 *)&args->csa_sessionid)[3],
> +		args->csa_sequenceid, args->csa_slotid,
> +		args->csa_highestslotid, args->csa_cachethis,
> +		args->csa_nrclists);
> +out:
> +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> +	return status;
> +
> +out_free:
> +	for (i = 0; i < args->csa_nrclists; i++)
> +		kfree(args->csa_rclists[i].rcl_refcalls);
> +	kfree(args->csa_rclists);
> +	goto out;
> +}
> +
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char *str)
>  {
>  	__be32 *p;
> @@ -359,6 +485,49 @@ out:
>  	return status;
>  }
>  
> +#if defined(CONFIG_NFS_V4_1)
> +
> +static unsigned encode_sessionid(struct xdr_stream *xdr,
> +				 const struct nfs4_sessionid *sid)
> +{
> +	uint32_t *p;
> +	int len = NFS4_MAX_SESSIONID_LEN;
> +
> +	p = xdr_reserve_space(xdr, len);
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);
> +
> +	memcpy(p, sid, len);
> +	return 0;
> +}
> +
> +static unsigned encode_cb_sequence_res(struct svc_rqst *rqstp,
> +				       struct xdr_stream *xdr,
> +				       const struct cb_sequenceres *res)
> +{
> +	uint32_t *p;
> +	unsigned status = res->csr_status;
> +
> +	if (unlikely(status != 0))
> +		goto out;
> +
> +	encode_sessionid(xdr, &res->csr_sessionid);
> +
> +	p = xdr_reserve_space(xdr, 4 * sizeof(uint32_t));
> +	if (unlikely(p == NULL))
> +		return htonl(NFS4ERR_RESOURCE);
> +
> +	*p++ = htonl(res->csr_sequenceid);
> +	*p++ = htonl(res->csr_slotid);
> +	*p++ = htonl(res->csr_highestslotid);
> +	*p++ = htonl(res->csr_target_highestslotid);
> +out:
> +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> +	return status;
> +}
> +
> +#endif /* CONFIG_NFS_V4_1 */
> +
>  static __be32 process_op(uint32_t minorversion, int nop,
>  		struct svc_rqst *rqstp,
>  		struct xdr_stream *xdr_in, void *argp,
> @@ -382,6 +551,7 @@ static __be32 process_op(uint32_t minorversion, int nop,
>  		switch (op_nr) {
>  		case OP_CB_GETATTR:
>  		case OP_CB_RECALL:
> +		case OP_CB_SEQUENCE:
>  			op = &callback_ops[op_nr];
>  			break;
>  
> @@ -391,7 +561,6 @@ static __be32 process_op(uint32_t minorversion, int nop,
>  		case OP_CB_RECALL_ANY:
>  		case OP_CB_RECALLABLE_OBJ_AVAIL:
>  		case OP_CB_RECALL_SLOT:
> -		case OP_CB_SEQUENCE:
>  		case OP_CB_WANTS_CANCELLED:
>  		case OP_CB_NOTIFY_LOCK:
>  		case OP_CB_NOTIFY_DEVICEID:
> @@ -496,7 +665,15 @@ static struct callback_op callback_ops[] = {
>  		.process_op = (callback_process_op_t)nfs4_callback_recall,
>  		.decode_args = (callback_decode_arg_t)decode_recall_args,
>  		.res_maxsize = CB_OP_RECALL_RES_MAXSZ,
> -	}
> +	},
> +#if defined(CONFIG_NFS_V4_1)
> +	[OP_CB_SEQUENCE] = {
> +		.process_op = (callback_process_op_t)nfs4_callback_sequence,
> +		.decode_args = (callback_decode_arg_t)decode_cb_sequence_args,
> +		.encode_res = (callback_encode_res_t)encode_cb_sequence_res,
> +		.res_maxsize = CB_OP_SEQUENCE_RES_MAXSZ,
> +	},
> +#endif /* CONFIG_NFS_V4_1 */
>  };
>  
>  /*

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com

Re: [pnfs] [RFC 39/39] nfs41: Backchannel: CB_SEQUENCE validation

[Labiaga, Ricardo]

On 6/15/09 5:42 PM, "Ricardo Labiaga" <ricardo.labiaga@netapp.com> wrote:

> On 6/15/09 5:27 PM, "Trond Myklebust" <Trond.Myklebust@netapp.com> wrote:
> 
>> On Fri, 2009-05-01 at 02:25 +0300, Benny Halevy wrote:
>>> From: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
>>> 
>>> Validates the callback's sessionID, the slot number, and the sequence ID.
>>> Increments the slot's sequence.
>>> 
>>> Detects replays, but simply prints a debug message (if debugging is enabled
>>> since we don't yet implement a duplicate request cache for the backchannel.
>>> This should not present a problem, since only idempotent callbacks are
>>> currently implemented.
>>> 
>>> Signed-off-by: Ricardo Labiaga <Ricardo.Labiaga@netapp.com>
>>> Signed-off-by: Benny Halevy <bhalevy@panasas.com>
>>> ---
>>>  fs/nfs/callback_proc.c |   75
>>> ++++++++++++++++++++++++++++++++++++++++++++----
>>>  1 files changed, 69 insertions(+), 6 deletions(-)
>>> 
>>> diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
>>> index 6b342e8..c85d66f 100644
>>> --- a/fs/nfs/callback_proc.c
>>> +++ b/fs/nfs/callback_proc.c
>>> @@ -105,6 +105,57 @@ out:
>>>  #if defined(CONFIG_NFS_V4_1)
>>>  
>>>  /*
>>> + * Validate the sequenceID sent by the server.
>>> + * Return success if the sequenceID is one more than what we last saw on
>>> + * this slot, accounting for wraparound.  Increments the slot's sequence.
>>> + *
>>> + * We don't yet implement a duplicate request cache, so at this time
>>> + * we will log replays, and process them as if we had not seen them before,
>>> + * but we don't bump the sequence in the slot.  Not too worried about it,
>>> + * since we only currently implement idempotent callbacks anyway.
>>> + *
>>> + * We have a single slot backchannel at this time, so we don't bother
>>> + * checking the used_slots bit array on the table.  The lower layer
>>> guarantees
>>> + * a single outstanding callback request at a time.
>>> + */
>>> +static int
>>> +validate_seqid(struct nfs4_slot_table *tbl, u32 slotid, u32 seqid)
>>> +{
>>> + struct nfs4_slot *slot;
>>> +
>>> + dprintk("%s enter. slotid %d seqid %d\n",
>>> +  __func__, slotid, seqid);
>>> +
>>> + if (slotid > NFS41_BC_MAX_CALLBACKS)
>>> +  return NFS4ERR_BADSLOT;
>> 
>> I thought we agreed to always return error values as _negative_? Why the
>> change here?
>> 
> 
> Will change to negative.
> 

Looking at this more closely, this is the error that we encode into the XDR
result, so it's okay for it to be positive.  I'll change it to use htonl()
in this function to avoid confusion, instead of setting htonl() in its
caller.

- ricardo

> - ricardo
> 
>>> +
>>> + slot = tbl->slots + slotid;
>>> + dprintk("%s slot table seqid: %d\n", __func__, slot->seq_nr);
>>> +
>>> + /* Normal */
>>> + if (likely(seqid == slot->seq_nr + 1)) {
>>> +  slot->seq_nr++;
>>> +  return NFS4_OK;
>>> + }
>>> +
>>> + /* Replay */
>>> + if (seqid == slot->seq_nr) {
>>> +  dprintk("%s seqid %d is a replay - no DRC available\n",
>>> +   __func__, seqid);
>>> +  return NFS4_OK;
>>> + }
>>> +
>>> + /* Wraparound */
>>> + if (seqid == 1 && (slot->seq_nr + 1) == 0) {
>>> +  slot->seq_nr = 1;
>>> +  return NFS4_OK;
>>> + }
>>> +
>>> + /* Misordered request */
>>> + return NFS4ERR_SEQ_MISORDERED;
>>> +}
>>> +
>>> +/*
>>>   * Returns a pointer to a held 'struct nfs_client' that matches the
>>> server's
>>>   * address, major version number, and session ID.  It is the caller's
>>>   * responsibility to release the returned reference.
>>> @@ -140,18 +191,27 @@ out:
>>> return NULL;
>>>  }
>>>  
>>> -/* FIXME: validate args->cbs_{sequence,slot}id */
>>>  /* FIXME: referring calls should be processed */
>>>  unsigned nfs4_callback_sequence(struct cb_sequenceargs *args,
>>> struct cb_sequenceres *res)
>>>  {
>>> - int i;
>>> - unsigned status = 0;
>>> + struct nfs_client *clp;
>>> + int i, status;
>>>  
>>> for (i = 0; i < args->csa_nrclists; i++)
>>> kfree(args->csa_rclists[i].rcl_refcalls);
>>> kfree(args->csa_rclists);
>>>  
>>> + status = NFS4ERR_BADSESSION;
>>> + clp = find_client_with_session(args->csa_addr, 4, &args->csa_sessionid);
>>> + if (clp == NULL)
>>> +  goto out;
>>> +
>>> + status = validate_seqid(&clp->cl_session->bc_slot_table,
>>> +    args->csa_slotid, args->csa_sequenceid);
>>> + if (status)
>>> +  goto out_putclient;
>>> +
>>> memcpy(&res->csr_sessionid, &args->csa_sessionid,
>>>       sizeof(res->csr_sessionid));
>>> res->csr_sequenceid = args->csa_sequenceid;
>>> @@ -159,9 +219,12 @@ unsigned nfs4_callback_sequence(struct cb_sequenceargs
>>> *args,
>>> res->csr_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>>> res->csr_target_highestslotid = NFS41_BC_MAX_CALLBACKS - 1;
>>>  
>>> - dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
>>> - res->csr_status = status;
>>> - return status;
>>> +out_putclient:
>>> + nfs_put_client(clp);
>>> +out:
>>> + dprintk("%s: exit with status = %d\n", __func__, status);
>>> + res->csr_status = htonl(status);
>>> + return res->csr_status;
>>>  }
>>>  
>>>  #endif /* CONFIG_NFS_V4_1 */
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [pnfs] [RFC 31/39] nfs41: cb_sequence xdr implementation

[Trond Myklebust]

On Mon, 2009-06-15 at 22:25 -0400, Halevy, Benny wrote:
> > > +#define READ64(x)         do {			\
> > > +	(x) = (u64)ntohl(*p++) << 32;		\
> > > +	(x) |= ntohl(*p++);			\
> > > +} while (0)
> > 
> > This doesn't appear to be used, and in any case, it duplicates the
> 
> Oops, Will do.
> 
> > existing function xdr_duplicate_hyper(). Please remove...
> 
> xdr_decode_hyper?

Err... Yes. In the non-jetlagged vernacular, it is indeed
xdr_decode_hyper()...

Trond

> Benny
> 
> -----Original Message-----
> From: Trond Myklebust [mailto:Trond.Myklebust@netapp.com]
> Sent: Tue 2009-06-16 03:18
> To: Halevy, Benny
> Cc: linux-nfs@vger.kernel.org; pnfs@linux-nfs.org
> Subject: Re: [pnfs] [RFC 31/39] nfs41: cb_sequence xdr implementation
>  
> On Fri, 2009-05-01 at 02:24 +0300, Benny Halevy wrote:
> > [nfs41: get rid of READMEM and COPYMEM for callback_xdr.c]
> > Signed-off-by: Benny Halevy <bhalevy@panasas.com>
> > ---
> >  fs/nfs/callback_xdr.c |  181 ++++++++++++++++++++++++++++++++++++++++++++++++-
> >  1 files changed, 179 insertions(+), 2 deletions(-)
> > 
> > diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
> > index 9db88ce..066273c 100644
> > --- a/fs/nfs/callback_xdr.c
> > +++ b/fs/nfs/callback_xdr.c
> > @@ -20,8 +20,18 @@
> >  				2 + 2 + 3 + 3)
> >  #define CB_OP_RECALL_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ)
> >  
> > +#if defined(CONFIG_NFS_V4_1)
> > +#define CB_OP_SEQUENCE_RES_MAXSZ	(CB_OP_HDR_RES_MAXSZ + \
> > +					4 + 1 + 3)
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  #define NFSDBG_FACILITY NFSDBG_CALLBACK
> >  
> > +#define READ64(x)         do {			\
> > +	(x) = (u64)ntohl(*p++) << 32;		\
> > +	(x) |= ntohl(*p++);			\
> > +} while (0)
> 
> This doesn't appear to be used, and in any case, it duplicates the
> existing function xdr_duplicate_hyper(). Please remove...
> 
> > +
> >  typedef __be32 (*callback_process_op_t)(void *, void *);
> >  typedef __be32 (*callback_decode_arg_t)(struct svc_rqst *, struct xdr_stream *, void *);
> >  typedef __be32 (*callback_encode_res_t)(struct svc_rqst *, struct xdr_stream *, void *);
> > @@ -210,6 +220,122 @@ out:
> >  	return status;
> >  }
> >  
> > +#if defined(CONFIG_NFS_V4_1)
> > +
> > +static unsigned decode_sessionid(struct xdr_stream *xdr,
> > +				 struct nfs4_sessionid *sid)
> > +{
> > +	uint32_t *p;
> > +	int len = NFS4_MAX_SESSIONID_LEN;
> > +
> > +	p = read_buf(xdr, len);
> > +	if (unlikely(p == NULL))
> > +		return htonl(NFS4ERR_RESOURCE);;
> > +
> > +	memcpy(sid->data, p, len);
> > +	return 0;
> > +}
> > +
> > +static unsigned decode_rc_list(struct xdr_stream *xdr,
> > +			       struct referring_call_list *rc_list)
> > +{
> > +	uint32_t *p;
> > +	int i;
> > +	unsigned status;
> > +
> > +	status = decode_sessionid(xdr, &rc_list->rcl_sessionid);
> > +	if (status)
> > +		goto out;
> > +
> > +	status = htonl(NFS4ERR_RESOURCE);
> > +	p = read_buf(xdr, sizeof(uint32_t));
> > +	if (unlikely(p == NULL))
> > +		goto out;
> > +
> > +	rc_list->rcl_nrefcalls = ntohl(*p++);
> > +	if (rc_list->rcl_nrefcalls) {
> > +		p = read_buf(xdr,
> > +			     rc_list->rcl_nrefcalls * 2 * sizeof(uint32_t));
> > +		if (unlikely(p == NULL))
> > +			goto out;
> > +		rc_list->rcl_refcalls = kmalloc(rc_list->rcl_nrefcalls *
> > +						sizeof(*rc_list->rcl_refcalls),
> > +						GFP_KERNEL);
> > +		if (unlikely(rc_list->rcl_refcalls == NULL))
> > +			goto out;
> > +		for (i = 0; i < rc_list->rcl_nrefcalls; i++) {
> > +			rc_list->rcl_refcalls[i].rc_sequenceid = ntohl(*p++);
> > +			rc_list->rcl_refcalls[i].rc_slotid = ntohl(*p++);
> > +		}
> > +	}
> > +	status = 0;
> > +
> > +out:
> > +	return status;
> > +}
> > +
> > +static unsigned decode_cb_sequence_args(struct svc_rqst *rqstp,
> > +					struct xdr_stream *xdr,
> > +					struct cb_sequenceargs *args)
> > +{
> > +	uint32_t *p;
> > +	int i;
> > +	unsigned status;
> > +
> > +	status = decode_sessionid(xdr, &args->csa_sessionid);
> > +	if (status)
> > +		goto out;
> > +
> > +	status = htonl(NFS4ERR_RESOURCE);
> > +	p = read_buf(xdr, 5 * sizeof(uint32_t));
> > +	if (unlikely(p == NULL))
> > +		goto out;
> > +
> > +	args->csa_addr = svc_addr_in(rqstp);
> > +	args->csa_sequenceid = ntohl(*p++);
> > +	args->csa_slotid = ntohl(*p++);
> > +	args->csa_highestslotid = ntohl(*p++);
> > +	args->csa_cachethis = ntohl(*p++);
> > +	args->csa_nrclists = ntohl(*p++);
> > +	args->csa_rclists = NULL;
> > +	if (args->csa_nrclists) {
> > +		args->csa_rclists = kmalloc(args->csa_nrclists *
> > +					    sizeof(*args->csa_rclists),
> > +					    GFP_KERNEL);
> > +		if (unlikely(args->csa_rclists == NULL))
> > +			goto out;
> > +
> > +		for (i = 0; i < args->csa_nrclists; i++) {
> > +			status = decode_rc_list(xdr, &args->csa_rclists[i]);
> > +			if (status)
> > +				goto out_free;
> > +		}
> > +	}
> > +	status = 0;
> > +
> > +	dprintk("%s: sessionid %x:%x:%x:%x sequenceid %u slotid %u "
> > +		"highestslotid %u cachethis %d nrclists %u\n",
> > +		__func__,
> > +		((u32 *)&args->csa_sessionid)[0],
> > +		((u32 *)&args->csa_sessionid)[1],
> > +		((u32 *)&args->csa_sessionid)[2],
> > +		((u32 *)&args->csa_sessionid)[3],
> > +		args->csa_sequenceid, args->csa_slotid,
> > +		args->csa_highestslotid, args->csa_cachethis,
> > +		args->csa_nrclists);
> > +out:
> > +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> > +	return status;
> > +
> > +out_free:
> > +	for (i = 0; i < args->csa_nrclists; i++)
> > +		kfree(args->csa_rclists[i].rcl_refcalls);
> > +	kfree(args->csa_rclists);
> > +	goto out;
> > +}
> > +
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char *str)
> >  {
> >  	__be32 *p;
> > @@ -359,6 +485,49 @@ out:
> >  	return status;
> >  }
> >  
> > +#if defined(CONFIG_NFS_V4_1)
> > +
> > +static unsigned encode_sessionid(struct xdr_stream *xdr,
> > +				 const struct nfs4_sessionid *sid)
> > +{
> > +	uint32_t *p;
> > +	int len = NFS4_MAX_SESSIONID_LEN;
> > +
> > +	p = xdr_reserve_space(xdr, len);
> > +	if (unlikely(p == NULL))
> > +		return htonl(NFS4ERR_RESOURCE);
> > +
> > +	memcpy(p, sid, len);
> > +	return 0;
> > +}
> > +
> > +static unsigned encode_cb_sequence_res(struct svc_rqst *rqstp,
> > +				       struct xdr_stream *xdr,
> > +				       const struct cb_sequenceres *res)
> > +{
> > +	uint32_t *p;
> > +	unsigned status = res->csr_status;
> > +
> > +	if (unlikely(status != 0))
> > +		goto out;
> > +
> > +	encode_sessionid(xdr, &res->csr_sessionid);
> > +
> > +	p = xdr_reserve_space(xdr, 4 * sizeof(uint32_t));
> > +	if (unlikely(p == NULL))
> > +		return htonl(NFS4ERR_RESOURCE);
> > +
> > +	*p++ = htonl(res->csr_sequenceid);
> > +	*p++ = htonl(res->csr_slotid);
> > +	*p++ = htonl(res->csr_highestslotid);
> > +	*p++ = htonl(res->csr_target_highestslotid);
> > +out:
> > +	dprintk("%s: exit with status = %d\n", __func__, ntohl(status));
> > +	return status;
> > +}
> > +
> > +#endif /* CONFIG_NFS_V4_1 */
> > +
> >  static __be32 process_op(uint32_t minorversion, int nop,
> >  		struct svc_rqst *rqstp,
> >  		struct xdr_stream *xdr_in, void *argp,
> > @@ -382,6 +551,7 @@ static __be32 process_op(uint32_t minorversion, int nop,
> >  		switch (op_nr) {
> >  		case OP_CB_GETATTR:
> >  		case OP_CB_RECALL:
> > +		case OP_CB_SEQUENCE:
> >  			op = &callback_ops[op_nr];
> >  			break;
> >  
> > @@ -391,7 +561,6 @@ static __be32 process_op(uint32_t minorversion, int nop,
> >  		case OP_CB_RECALL_ANY:
> >  		case OP_CB_RECALLABLE_OBJ_AVAIL:
> >  		case OP_CB_RECALL_SLOT:
> > -		case OP_CB_SEQUENCE:
> >  		case OP_CB_WANTS_CANCELLED:
> >  		case OP_CB_NOTIFY_LOCK:
> >  		case OP_CB_NOTIFY_DEVICEID:
> > @@ -496,7 +665,15 @@ static struct callback_op callback_ops[] = {
> >  		.process_op = (callback_process_op_t)nfs4_callback_recall,
> >  		.decode_args = (callback_decode_arg_t)decode_recall_args,
> >  		.res_maxsize = CB_OP_RECALL_RES_MAXSZ,
> > -	}
> > +	},
> > +#if defined(CONFIG_NFS_V4_1)
> > +	[OP_CB_SEQUENCE] = {
> > +		.process_op = (callback_process_op_t)nfs4_callback_sequence,
> > +		.decode_args = (callback_decode_arg_t)decode_cb_sequence_args,
> > +		.encode_res = (callback_encode_res_t)encode_cb_sequence_res,
> > +		.res_maxsize = CB_OP_SEQUENCE_RES_MAXSZ,
> > +	},
> > +#endif /* CONFIG_NFS_V4_1 */
> >  };
> >  
> >  /*
> 

-- 
Trond Myklebust
Linux NFS client maintainer

NetApp
Trond.Myklebust@netapp.com
www.netapp.com