[PATCH v7 00/10] NFSD support for asynchronous COPY

[PATCH v7 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

To do asynchronous copies, NFSD creates a new kthread to handle the request.
Upon receiving the COPY, it generates a unique copy stateid (stored in a
global list for keeping track of state for OFFLOAD_STATUS to be queried by),
starts the thread, and replies back to the client. nfsd4_copy arguments that
are allocated on the stack are copies for the kthread.

For the async copy handler, copy is done in the loop for the requested
number of bytes. If an error is encountered and partial copy has happened,
a successful partial copy is returned in the CB_OFFLOAD. vfs_copy_file_range
is called with 4MB chunks for both async and sync, allowing for 4MB
granularity of OFFLOAD_STATUS queiry. Once copy is done, the results are
queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running
the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
the ongoing do_splice() and once vfs returns we are choosing not to send
the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running
the copy and will locate within the copy state the current number of bytes
copied so far.

v7 addresses Bruce's comments:
1. hardcoding returning 0bytes for the error case
2. shutting down copy before shutting down callback
3. adding the copy to the list of copies after the copy_task is initialized
4. introducing the async code but not enabling it until after the following
patches add creating and returning the unique stateid for the async copy.
5. changing copy's refcount from atomic_t to refcount_t
6. making sure that list sc_cp_list is accessed under the pstid->sc_lock

Olga Kornievskaia (10):
  NFSD CB_OFFLOAD xdr
  NFSD OFFLOAD_STATUS xdr
  NFSD OFFLOAD_CANCEL xdr
  NFSD xdr callback stateid in async COPY reply
  NFSD introduce asynch copy feature
  NFSD return nfs4_stid in nfs4_preprocess_stateid_op
  NFSD create new stateid for async copy
  NFSD handle OFFLOAD_CANCEL op
  NFSD support OFFLOAD_STATUS
  NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  98 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 317 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  80 ++++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

-- 
1.8.3.1

[PATCH v7 01/10] NFSD CB_OFFLOAD xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4callback.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/state.h        |  1 +
 fs/nfsd/xdr4.h         |  6 ++++
 fs/nfsd/xdr4cb.h       | 10 ++++++
 4 files changed, 115 insertions(+)

diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 49b0a9e..063b8d3 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -39,6 +39,7 @@
 #include "state.h"
 #include "netns.h"
 #include "xdr4cb.h"
+#include "xdr4.h"
 
 #define NFSDDBG_FACILITY                NFSDDBG_PROC
 
@@ -105,6 +106,7 @@ enum nfs_cb_opnum4 {
 	OP_CB_WANTS_CANCELLED		= 12,
 	OP_CB_NOTIFY_LOCK		= 13,
 	OP_CB_NOTIFY_DEVICEID		= 14,
+	OP_CB_OFFLOAD			= 15,
 	OP_CB_ILLEGAL			= 10044
 };
 
@@ -683,6 +685,101 @@ static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp,
 }
 
 /*
+ * struct write_response4 {
+ *	stateid4	wr_callback_id<1>;
+ *	length4		wr_count;
+ *	stable_how4	wr_committed;
+ *	verifier4	wr_writeverf;
+ * };
+ * union offload_info4 switch (nfsstat4 coa_status) {
+ *	case NFS4_OK:
+ *		write_response4	coa_resok4;
+ *	default:
+ *	length4		coa_bytes_copied;
+ * };
+ * struct CB_OFFLOAD4args {
+ *	nfs_fh4		coa_fh;
+ *	stateid4	coa_stateid;
+ *	offload_info4	coa_offload_info;
+ * };
+ */
+static void encode_offload_info4(struct xdr_stream *xdr,
+				 __be32 nfserr,
+				 const struct nfsd4_copy *cp)
+{
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 4);
+	*p++ = nfserr;
+	if (!nfserr) {
+		p = xdr_reserve_space(xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE);
+		p = xdr_encode_empty_array(p);
+		p = xdr_encode_hyper(p, cp->cp_res.wr_bytes_written);
+		*p++ = cpu_to_be32(cp->cp_res.wr_stable_how);
+		p = xdr_encode_opaque_fixed(p, cp->cp_res.wr_verifier.data,
+					    NFS4_VERIFIER_SIZE);
+	} else {
+		p = xdr_reserve_space(xdr, 8);
+		/* We always return success if bytes were written */
+		p = xdr_encode_hyper(p, 0);
+	}
+}
+
+static void encode_cb_offload4args(struct xdr_stream *xdr,
+				   __be32 nfserr,
+				   const struct knfsd_fh *fh,
+				   const struct nfsd4_copy *cp,
+				   struct nfs4_cb_compound_hdr *hdr)
+{
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 4);
+	*p++ = cpu_to_be32(OP_CB_OFFLOAD);
+	encode_nfs_fh4(xdr, fh);
+	encode_stateid4(xdr, &cp->cp_res.cb_stateid);
+	encode_offload_info4(xdr, nfserr, cp);
+
+	hdr->nops++;
+}
+
+static void nfs4_xdr_enc_cb_offload(struct rpc_rqst *req,
+				    struct xdr_stream *xdr,
+				    const void *data)
+{
+	const struct nfsd4_callback *cb = data;
+	const struct nfsd4_copy *cp =
+		container_of(cb, struct nfsd4_copy, cp_cb);
+	struct nfs4_cb_compound_hdr hdr = {
+		.ident = 0,
+		.minorversion = cb->cb_clp->cl_minorversion,
+	};
+
+	encode_cb_compound4args(xdr, &hdr);
+	encode_cb_sequence4args(xdr, cb, &hdr);
+	encode_cb_offload4args(xdr, cp->nfserr, &cp->fh, cp, &hdr);
+	encode_cb_nops(&hdr);
+}
+
+static int nfs4_xdr_dec_cb_offload(struct rpc_rqst *rqstp,
+				   struct xdr_stream *xdr,
+				   void *data)
+{
+	struct nfsd4_callback *cb = data;
+	struct nfs4_cb_compound_hdr hdr;
+	int status;
+
+	status = decode_cb_compound4res(xdr, &hdr);
+	if (unlikely(status))
+		return status;
+
+	if (cb) {
+		status = decode_cb_sequence4res(xdr, cb);
+		if (unlikely(status || cb->cb_seq_status))
+			return status;
+	}
+	return decode_cb_op_status(xdr, OP_CB_OFFLOAD, &cb->cb_status);
+}
+/*
  * RPC procedure tables
  */
 #define PROC(proc, call, argtype, restype)				\
@@ -703,6 +800,7 @@ static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp,
 	PROC(CB_LAYOUT,	COMPOUND,	cb_layout,	cb_layout),
 #endif
 	PROC(CB_NOTIFY_LOCK,	COMPOUND,	cb_notify_lock,	cb_notify_lock),
+	PROC(CB_OFFLOAD,	COMPOUND,	cb_offload,	cb_offload),
 };
 
 static unsigned int nfs4_cb_counts[ARRAY_SIZE(nfs4_cb_procedures)];
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index f3772ea..5c16d35 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -573,6 +573,7 @@ enum nfsd4_cb_op {
 	NFSPROC4_CLNT_CB_NULL = 0,
 	NFSPROC4_CLNT_CB_RECALL,
 	NFSPROC4_CLNT_CB_LAYOUT,
+	NFSPROC4_CLNT_CB_OFFLOAD,
 	NFSPROC4_CLNT_CB_SEQUENCE,
 	NFSPROC4_CLNT_CB_NOTIFY_LOCK,
 };
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index bc29511..c573f3b 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -507,6 +507,7 @@ struct nfsd42_write_res {
 	u64			wr_bytes_written;
 	u32			wr_stable_how;
 	nfs4_verifier		wr_verifier;
+	stateid_t		cb_stateid;
 };
 
 struct nfsd4_copy {
@@ -523,6 +524,11 @@ struct nfsd4_copy {
 
 	/* response */
 	struct nfsd42_write_res	cp_res;
+
+	/* for cb_offload */
+	struct nfsd4_callback	cp_cb;
+	__be32			nfserr;
+	struct knfsd_fh		fh;
 };
 
 struct nfsd4_seek {
diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h
index 517239a..547cf07 100644
--- a/fs/nfsd/xdr4cb.h
+++ b/fs/nfsd/xdr4cb.h
@@ -38,3 +38,13 @@
 #define NFS4_dec_cb_notify_lock_sz	(cb_compound_dec_hdr_sz  +      \
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
+#define enc_cb_offload_info_sz		(1 + 1 + 2 + 1 +		\
+					XDR_QUADLEN(NFS4_VERIFIER_SIZE))
+#define NFS4_enc_cb_offload_sz		(cb_compound_enc_hdr_sz +       \
+					cb_sequence_enc_sz +            \
+					enc_nfs4_fh_sz +		\
+					enc_stateid_sz +		\
+					enc_cb_offload_info_sz)
+#define NFS4_dec_cb_offload_sz		(cb_compound_dec_hdr_sz  +      \
+					cb_sequence_dec_sz +            \
+					op_dec_sz)
-- 
1.8.3.1

[PATCH v7 02/10] NFSD OFFLOAD_STATUS xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 20 ++++++++++++++++++++
 fs/nfsd/nfs4xdr.c  | 27 +++++++++++++++++++++++++--
 fs/nfsd/xdr4.h     | 10 ++++++++++
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index a0bed2b..ef64873 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1140,6 +1140,13 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	fput(file);
 	return status;
 }
+static __be32
+nfsd4_offload_status(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return nfserr_notsupp;
+}
 
 static __be32
 nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
@@ -2040,6 +2047,14 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		1 /* cr_synchronous */) * sizeof(__be32);
 }
 
+static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp,
+					     struct nfsd4_op *op)
+{
+	return (op_encode_hdr_size +
+		2 /* osr_count */ +
+		1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32);
+}
+
 #ifdef CONFIG_NFSD_PNFS
 static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
@@ -2453,6 +2468,11 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		.op_name = "OP_SEEK",
 		.op_rsize_bop = nfsd4_seek_rsize,
 	},
+	[OP_OFFLOAD_STATUS] = {
+		.op_func = nfsd4_offload_status,
+		.op_name = "OP_OFFLOAD_STATUS",
+		.op_rsize_bop = nfsd4_offload_status_rsize,
+	},
 };
 
 /**
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index e502fd1..bd9b46d 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1767,6 +1767,13 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 }
 
 static __be32
+nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp,
+			    struct nfsd4_offload_status *os)
+{
+	return nfsd4_decode_stateid(argp, &os->stateid);
+}
+
+static __be32
 nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek)
 {
 	DECODE_HEAD;
@@ -1873,7 +1880,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_dec)nfsd4_decode_notsupp,
@@ -4220,6 +4227,22 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 }
 
 static __be32
+nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
+			    struct nfsd4_offload_status *os)
+{
+	struct xdr_stream *xdr = &resp->xdr;
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 8 + 4);
+	if (!p)
+		return nfserr_resource;
+	p = xdr_encode_hyper(p, os->count);
+	*p++ = cpu_to_be32(0);
+
+	return nfserr;
+}
+
+static __be32
 nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr,
 		  struct nfsd4_seek *seek)
 {
@@ -4322,7 +4345,7 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 	[OP_LAYOUTERROR]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_LAYOUTSTATS]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_enc)nfsd4_encode_noop,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_noop,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_SEEK]		= (nfsd4_enc)nfsd4_encode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_enc)nfsd4_encode_noop,
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index c573f3b..4fc0e35 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -542,6 +542,15 @@ struct nfsd4_seek {
 	loff_t		seek_pos;
 };
 
+struct nfsd4_offload_status {
+	/* request */
+	stateid_t	stateid;
+
+	/* response */
+	u64		count;
+	u32		status;
+};
+
 struct nfsd4_op {
 	int					opnum;
 	const struct nfsd4_operation *		opdesc;
@@ -600,6 +609,7 @@ struct nfsd4_op {
 		struct nfsd4_fallocate		deallocate;
 		struct nfsd4_clone		clone;
 		struct nfsd4_copy		copy;
+		struct nfsd4_offload_status	offload_status;
 		struct nfsd4_seek		seek;
 	} u;
 	struct nfs4_replay *			replay;
-- 
1.8.3.1

[PATCH v7 03/10] NFSD OFFLOAD_CANCEL xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 14 ++++++++++++++
 fs/nfsd/nfs4xdr.c  |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index ef64873..e11494f 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1119,6 +1119,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 }
 
 static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return 0;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		struct nfsd4_fallocate *fallocate, int flags)
 {
@@ -2473,6 +2481,12 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		.op_name = "OP_OFFLOAD_STATUS",
 		.op_rsize_bop = nfsd4_offload_status_rsize,
 	},
+	[OP_OFFLOAD_CANCEL] = {
+		.op_func = nfsd4_offload_cancel,
+		.op_flags = OP_MODIFIES_SOMETHING,
+		.op_name = "OP_OFFLOAD_CANCEL",
+		.op_rsize_bop = nfsd4_only_status_rsize,
+	},
 };
 
 /**
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index bd9b46d..2b7c507 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1879,7 +1879,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_IO_ADVISE]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
-- 
1.8.3.1

[PATCH v7 04/10] NFSD xdr callback stateid in async COPY reply

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4xdr.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 2b7c507..f9b74a3 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -4194,15 +4194,27 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 #endif /* CONFIG_NFSD_PNFS */
 
 static __be32
-nfsd42_encode_write_res(struct nfsd4_compoundres *resp, struct nfsd42_write_res *write)
+nfsd42_encode_write_res(struct nfsd4_compoundres *resp,
+		struct nfsd42_write_res *write, bool sync)
 {
 	__be32 *p;
+	p = xdr_reserve_space(&resp->xdr, 4);
+	if (!p)
+		return nfserr_resource;
 
-	p = xdr_reserve_space(&resp->xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE);
+	if (sync)
+		*p++ = cpu_to_be32(0);
+	else {
+		__be32 nfserr;
+		*p++ = cpu_to_be32(1);
+		nfserr = nfsd4_encode_stateid(&resp->xdr, &write->cb_stateid);
+		if (nfserr)
+			return nfserr;
+	}
+	p = xdr_reserve_space(&resp->xdr, 8 + 4 + NFS4_VERIFIER_SIZE);
 	if (!p)
 		return nfserr_resource;
 
-	*p++ = cpu_to_be32(0);
 	p = xdr_encode_hyper(p, write->wr_bytes_written);
 	*p++ = cpu_to_be32(write->wr_stable_how);
 	p = xdr_encode_opaque_fixed(p, write->wr_verifier.data,
@@ -4216,7 +4228,8 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 {
 	__be32 *p;
 
-	nfserr = nfsd42_encode_write_res(resp, &copy->cp_res);
+	nfserr = nfsd42_encode_write_res(resp, &copy->cp_res,
+			copy->cp_synchronous);
 	if (nfserr)
 		return nfserr;
 
-- 
1.8.3.1

[PATCH v7 05/10] NFSD introduce asynch copy feature

[Olga Kornievskaia]

Upon receiving a request for async copy, create a new kthread.
If we get asynchronous request, make sure to copy the needed
arguments/state from the stack before starting the copy. Then
start the thread and reply back to the client indicating copy
is asynchronous.

nfsd_copy_file_range() will copy in a loop over the total
number of bytes is needed to copy. In case a failure happens
in the middle, we ignore the error and return how much we
copied so far. Once done creating a workitem for the callback
workqueue and send CB_OFFLOAD with the results.

In the following patches the unique stateid will be generated
for the async COPY to return, at that point will enable the
async feature.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 176 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   2 +
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   9 +++
 4 files changed, 170 insertions(+), 19 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index e11494f..91d1544 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -35,6 +35,7 @@
 #include <linux/file.h>
 #include <linux/falloc.h>
 #include <linux/slab.h>
+#include <linux/kthread.h>
 
 #include "idmap.h"
 #include "cache.h"
@@ -1083,39 +1084,176 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+				copy->fh_dst, dst_pos, bytes_total);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0 && !copy->cp_res.wr_bytes_written)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->net = src->net;
+	dst->cp_clp = src->cp_clp;
+	atomic_inc(&dst->cp_clp->cl_refcount);
+	dst->fh_dst = get_file(src->fh_dst);
+	dst->fh_src = get_file(src->fh_src);
+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
+{
+	fput(copy->fh_dst);
+	fput(copy->fh_src);
+	if (remove) {
+		spin_lock(&copy->cp_clp->async_lock);
+		list_del(&copy->copies);
+		spin_unlock(&copy->cp_clp->async_lock);
+	}
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_do_async_copy(void *data)
+{
+	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	atomic_inc(&cb_copy->cp_clp->cl_refcount);
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	cleanup_async_copy(copy, true);
+	return 0;
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
 	struct nfsd4_copy *copy = &u->copy;
-	struct file *src, *dst;
 	__be32 status;
-	ssize_t bytes;
+	struct nfsd4_copy *async_copy = NULL;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);
 	if (status)
 		goto out;
 
-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	/* for now disable asynchronous copy feature */
+	copy->cp_synchronous = 1;
+	if (!copy->cp_synchronous) {
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+				async_copy, "%s", "copy thread");
+		if (IS_ERR(async_copy->copy_task)) {
+			status = PTR_ERR(async_copy->copy_task);
+			goto out_err;
+		}
+		spin_lock(&async_copy->cp_clp->async_lock);
+		list_add(&async_copy->copies,
+				&async_copy->cp_clp->async_copies);
+		spin_unlock(&async_copy->cp_clp->async_lock);
+		wake_up_process(async_copy->copy_task);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
+out_err:
+	cleanup_async_copy(async_copy, false);
+	goto out;
 }
 
 static __be32
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 150521c..fe665b0 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1790,6 +1790,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 #ifdef CONFIG_NFSD_PNFS
 	INIT_LIST_HEAD(&clp->cl_lo_states);
 #endif
+	INIT_LIST_HEAD(&clp->async_copies);
+	spin_lock_init(&clp->async_lock);
 	spin_lock_init(&clp->cl_lock);
 	rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
 	return clp;
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 5c16d35..491030e 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -355,6 +355,8 @@ struct nfs4_client {
 	struct rpc_wait_queue	cl_cb_waitq;	/* backchannel callers may */
 						/* wait here for slots */
 	struct net		*net;
+	struct list_head	async_copies;	/* list of async copies */
+	spinlock_t		async_lock;	/* lock for async copies */
 };
 
 /* struct nfs4_client_reset
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 4fc0e35..3b6b655 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -529,6 +529,15 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
+
+	struct list_head	copies;
+	struct task_struct	*copy_task;
 };
 
 struct nfsd4_seek {
-- 
1.8.3.1

[PATCH v7 06/10] NFSD return nfs4_stid in nfs4_preprocess_stateid_op

[Olga Kornievskaia]

Needed for copy to add nfs4_cp_state to the nfs4_stid.

Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 17 ++++++++++-------
 fs/nfsd/nfs4state.c |  8 ++++++--
 fs/nfsd/state.h     |  3 ++-
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 91d1544..aafd5a58 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -766,7 +766,8 @@ static __be32 nfsd4_do_lookupp(struct svc_rqst *rqstp, struct svc_fh *fh)
 	/* check stateid */
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					&read->rd_stateid, RD_STATE,
-					&read->rd_filp, &read->rd_tmp_file);
+					&read->rd_filp, &read->rd_tmp_file,
+					NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_read: couldn't process stateid!\n");
 		goto out;
@@ -937,7 +938,7 @@ static __be32 nfsd4_do_lookupp(struct svc_rqst *rqstp, struct svc_fh *fh)
 	if (setattr->sa_iattr.ia_valid & ATTR_SIZE) {
 		status = nfs4_preprocess_stateid_op(rqstp, cstate,
 				&cstate->current_fh, &setattr->sa_stateid,
-				WR_STATE, NULL, NULL);
+				WR_STATE, NULL, NULL, NULL);
 		if (status) {
 			dprintk("NFSD: nfsd4_setattr: couldn't process stateid!\n");
 			return status;
@@ -1003,7 +1004,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 		return nfserr_inval;
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
-						stateid, WR_STATE, &filp, NULL);
+					stateid, WR_STATE, &filp, NULL, NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_write: couldn't process stateid!\n");
 		return status;
@@ -1034,14 +1035,16 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh,
-					    src_stateid, RD_STATE, src, NULL);
+					    src_stateid, RD_STATE, src, NULL,
+					    NULL);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process src stateid!\n", __func__);
 		goto out;
 	}
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
-					    dst_stateid, WR_STATE, dst, NULL);
+					    dst_stateid, WR_STATE, dst, NULL,
+					    NULL);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;
@@ -1273,7 +1276,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &fallocate->falloc_stateid,
-					    WR_STATE, &file, NULL);
+					    WR_STATE, &file, NULL, NULL);
 	if (status != nfs_ok) {
 		dprintk("NFSD: nfsd4_fallocate: couldn't process stateid!\n");
 		return status;
@@ -1320,7 +1323,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &seek->seek_stateid,
-					    RD_STATE, &file, NULL);
+					    RD_STATE, &file, NULL, NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_seek: couldn't process stateid!\n");
 		return status;
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index fe665b0..e05832b 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -5092,7 +5092,8 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 __be32
 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
-		stateid_t *stateid, int flags, struct file **filpp, bool *tmp_file)
+		stateid_t *stateid, int flags, struct file **filpp,
+		bool *tmp_file, struct nfs4_stid **cstid)
 {
 	struct inode *ino = d_inode(fhp->fh_dentry);
 	struct net *net = SVC_NET(rqstp);
@@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 	if (!status && filpp)
 		status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
 out:
-	if (s)
+	if (s) {
+		if (!status && cstid)
+			*cstid = s;
 		nfs4_put_stid(s);
+	}
 	return status;
 }
 
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 491030e..9b7d7a0 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -605,7 +605,8 @@ struct nfsd4_blocked_lock {
 
 extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
-		stateid_t *stateid, int flags, struct file **filp, bool *tmp_file);
+		stateid_t *stateid, int flags, struct file **filp,
+		bool *tmp_file, struct nfs4_stid **cstid);
 __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     stateid_t *stateid, unsigned char typemask,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
-- 
1.8.3.1

[PATCH v7 07/10] NFSD create new stateid for async copy

[Olga Kornievskaia]

Generate a new stateid to be used for reply to the asynchronous
COPY (this would also be used later by COPY_NOTIFY as well).
Associate the stateid with the parent OPEN/LOCK/DELEG stateid
that can be freed during the free of the parent stateid. However,
right now deciding to bind the lifetime to when the vfs copy
is done. This way don't need to keep the nfsd_net structure for
the callback. The drawback is that time copy state information
is available for query by OFFLOAD_STATUS is slightly less.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
 fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 +++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 118 insertions(+), 10 deletions(-)

diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 36358d4..e83cf6e 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -122,6 +122,14 @@ struct nfsd_net {
 
 	wait_queue_head_t ntf_wq;
 	atomic_t ntf_refcnt;
+
+	/*
+	 * clientid and stateid data for construction of net unique COPY
+	 * stateids.
+	 */
+	u32		s2s_cp_cl_id;
+	struct idr	s2s_cp_stateids;
+	spinlock_t	s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index aafd5a58..eb9f528 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		  stateid_t *src_stateid, struct file **src,
-		  stateid_t *dst_stateid, struct file **dst)
+		  stateid_t *dst_stateid, struct file **dst,
+		  struct nfs4_stid **stid)
 {
 	__be32 status;
 
@@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    dst_stateid, WR_STATE, dst, NULL,
-					    NULL);
+					    stid);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;
@@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-				   &clone->cl_dst_stateid, &dst);
+				   &clone->cl_dst_stateid, &dst, NULL);
 	if (status)
 		goto out;
 
@@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-		sizeof(copy->cp_dst_stateid));
 	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
 	copy->cp_consecutive = 1;
 	copy->cp_synchronous = sync;
@@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
 	atomic_inc(&dst->cp_clp->cl_refcount);
 	dst->fh_dst = get_file(src->fh_dst);
 	dst->fh_src = get_file(src->fh_src);
+	dst->stid = src->stid;
+	dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 {
+	list_del(&copy->cps->cp_list);
+	nfs4_free_cp_state(copy->cps);
+	nfs4_put_stid(copy->stid);
 	fput(copy->fh_dst);
 	fput(copy->fh_src);
 	if (remove) {
@@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
 				   &copy->fh_src, &copy->cp_dst_stateid,
-				   &copy->fh_dst);
+				   &copy->fh_dst, &copy->stid);
 	if (status)
 		goto out;
 
@@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
 		sizeof(struct knfsd_fh));
 	copy->net = SVC_NET(rqstp);
-	/* for now disable asynchronous copy feature */
-	copy->cp_synchronous = 1;
 	if (!copy->cp_synchronous) {
+		struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
 		status = nfsd4_init_copy_res(copy, 0);
 		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 		if (!async_copy) {
 			status = nfserrno(-ENOMEM);
 			goto out;
 		}
+		copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+		if (!copy->cps) {
+			status = nfserrno(-ENOMEM);
+			kfree(async_copy);
+			goto out;
+		}
+		/* take a reference on the parent stateid so it's not
+		 * not freed by the copy compound
+		 */
+		refcount_inc(&copy->stid->sc_count);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+			sizeof(copy->cps->cp_stateid));
 		dup_copy_fields(copy, async_copy);
-		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-			sizeof(copy->cp_dst_stateid));
 		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
 				async_copy, "%s", "copy thread");
 		if (IS_ERR(async_copy->copy_task)) {
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index e05832b..bd76bd1 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	/* Will be incremented before return to client: */
 	refcount_set(&stid->sc_count, 1);
 	spin_lock_init(&stid->sc_lock);
+	INIT_LIST_HEAD(&stid->sc_cp_list);
 
 	/*
 	 * It shouldn't be a problem to reuse an opaque stateid value.
@@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+					       struct nfs4_stid *p_stid)
+{
+	struct nfs4_cp_state *cps;
+	int new_id;
+
+	cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+	if (!cps)
+		return NULL;
+	idr_preload(GFP_KERNEL);
+	spin_lock(&nn->s2s_cp_lock);
+	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+	spin_unlock(&nn->s2s_cp_lock);
+	idr_preload_end();
+	if (new_id < 0)
+		goto out_free;
+	cps->cp_stateid.si_opaque.so_id = new_id;
+	cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+	cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	cps->cp_p_stid = p_stid;
+	spin_lock(&p_stid->sc_lock);
+	list_add(&cps->cp_list, &p_stid->sc_cp_list);
+	spin_unlock(&p_stid->sc_lock);
+
+	return cps;
+out_free:
+	kfree(cps);
+	return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+	struct nfsd_net *nn;
+
+	nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+	spin_lock(&nn->s2s_cp_lock);
+	idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+	spin_unlock(&nn->s2s_cp_lock);
+
+	kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+	struct nfs4_cp_state *cps;
+
+	might_sleep();
+
+	spin_lock(&stid->sc_lock);
+	while (!list_empty(&stid->sc_cp_list)) {
+		cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+				       cp_list);
+		list_del(&cps->cp_list);
+		nfs4_free_cp_state(cps);
+	}
+	spin_unlock(&stid->sc_lock);
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
 	struct nfs4_stid *stid;
@@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
 	}
 	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
 	spin_unlock(&clp->cl_lock);
+
+	nfs4_free_cp_statelist(s);
+
 	s->sc_free(s);
 	if (fp)
 		put_nfs4_file(fp);
@@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_LIST_HEAD(&nn->close_lru);
 	INIT_LIST_HEAD(&nn->del_recall_lru);
 	spin_lock_init(&nn->client_lock);
+	spin_lock_init(&nn->s2s_cp_lock);
+	idr_init(&nn->s2s_cp_stateids);
 
 	spin_lock_init(&nn->blocked_locks_lock);
 	INIT_LIST_HEAD(&nn->blocked_locks_lru);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index d107b44..63edf68 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
 	nn->nfsd4_grace = 90;
 	nn->clverifier_counter = prandom_u32();
 	nn->clientid_counter = prandom_u32();
+	nn->s2s_cp_cl_id = nn->clientid_counter++;
 
 	atomic_set(&nn->ntf_refcnt, 0);
 	init_waitqueue_head(&nn->ntf_wq);
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 9b7d7a0..49709d1 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -94,6 +94,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+	struct list_head	sc_cp_list;
 	unsigned char		sc_type;
 	stateid_t		sc_stateid;
 	spinlock_t		sc_lock;
@@ -103,6 +104,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+	stateid_t		cp_stateid;
+	struct list_head	cp_list;	/* per parent nfs4_stid */
+	struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:
@@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
 				  void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 3b6b655..5d493b9 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -535,6 +535,8 @@ struct nfsd4_copy {
 	struct file             *fh_src;
 	struct file             *fh_dst;
 	struct net              *net;
+	struct nfs4_stid        *stid;
+	struct nfs4_cp_state    *cps;
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;
-- 
1.8.3.1

[PATCH v7 08/10] NFSD handle OFFLOAD_CANCEL op

[Olga Kornievskaia]

Upon receiving OFFLOAD_CANCEL search the list of copy stateids,
if found then set the SIGPENDING signal so that do_splice stops
copying and also send kthread_stop to the copy thread to stop
and wait for it. Take a reference on the copy from the
offload_cancel thread so that it won't go away while we are
trying to process it. Server won't be sending CB_OFFLOAD to the
client since it received a cancel.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 45 +++++++++++++++++++++++++++++++++++++++++++--
 fs/nfsd/xdr4.h     |  1 +
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index eb9f528..3d190b3 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1088,6 +1088,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+
+static void nfs4_put_copy(struct nfsd4_copy *copy)
+{
+	if (!refcount_dec_and_test(&copy->refcount))
+		return;
+	kfree(copy);
+}
+
 static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
 {
 	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
@@ -1125,6 +1133,8 @@ static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
 	u64 dst_pos = copy->cp_dst_pos;
 
 	do {
+		if (signalled() || kthread_should_stop())
+			return -1;
 		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
 				copy->fh_dst, dst_pos, bytes_total);
 		if (bytes_copied <= 0)
@@ -1143,11 +1153,16 @@ static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
 	ssize_t bytes;
 
 	bytes = _nfsd_copy_file_range(copy);
+	if (signalled() || kthread_should_stop()) {
+		status = -1;
+		goto cleanup;
+	}
 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)
 		status = nfserrno(bytes);
 	else
 		status = nfsd4_init_copy_res(copy, sync);
 
+cleanup:
 	fput(copy->fh_src);
 	fput(copy->fh_dst);
 	return status;
@@ -1187,7 +1202,7 @@ static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 		spin_unlock(&copy->cp_clp->async_lock);
 	}
 	atomic_dec(&copy->cp_clp->cl_refcount);
-	kfree(copy);
+	nfs4_put_copy(copy);
 }
 
 static int nfsd4_do_async_copy(void *data)
@@ -1196,6 +1211,9 @@ static int nfsd4_do_async_copy(void *data)
 	struct nfsd4_copy *cb_copy;
 
 	copy->nfserr = nfsd4_do_copy(copy, 0);
+	if (signalled() || kthread_should_stop())
+		goto out;
+
 	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 	if (!cb_copy)
 		goto out;
@@ -1278,7 +1296,30 @@ static int nfsd4_do_async_copy(void *data)
 		     struct nfsd4_compound_state *cstate,
 		     union nfsd4_op_u *u)
 {
-	return 0;
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	bool found = false;
+	struct nfs4_client *clp = cstate->clp;
+
+	spin_lock(&clp->async_lock);
+	list_for_each_entry(copy, &clp->async_copies, copies) {
+		if (memcmp(&copy->cps->cp_stateid, &os->stateid,
+				NFS4_STATEID_SIZE))
+			continue;
+		found = true;
+		refcount_inc(&copy->refcount);
+		break;
+	}
+	spin_unlock(&clp->async_lock);
+	if (found) {
+		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
+		kthread_stop(copy->copy_task);
+		nfs4_put_copy(copy);
+	} else
+		status = nfserr_bad_stateid;
+
+	return status;
 }
 
 static __be32
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 5d493b9..47bfe64 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -540,6 +540,7 @@ struct nfsd4_copy {
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;
+	refcount_t		refcount;
 };
 
 struct nfsd4_seek {
-- 
1.8.3.1

[PATCH v7 09/10] NFSD support OFFLOAD_STATUS

[Olga Kornievskaia]

Search the list for the asynchronous copy based on the stateid,
then lookup the number of bytes copied so far.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 44 ++++++++++++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 3d190b3..57d0daa 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1291,28 +1291,36 @@ static int nfsd4_do_async_copy(void *data)
 	goto out;
 }
 
-static __be32
-nfsd4_offload_cancel(struct svc_rqst *rqstp,
-		     struct nfsd4_compound_state *cstate,
-		     union nfsd4_op_u *u)
+static struct nfsd4_copy *
+find_async_copy(struct nfs4_client *clp, struct nfsd4_offload_status *os)
 {
-	struct nfsd4_offload_status *os = &u->offload_status;
-	__be32 status = 0;
 	struct nfsd4_copy *copy;
-	bool found = false;
-	struct nfs4_client *clp = cstate->clp;
 
 	spin_lock(&clp->async_lock);
 	list_for_each_entry(copy, &clp->async_copies, copies) {
 		if (memcmp(&copy->cps->cp_stateid, &os->stateid,
 				NFS4_STATEID_SIZE))
 			continue;
-		found = true;
 		refcount_inc(&copy->refcount);
-		break;
+		spin_unlock(&clp->async_lock);
+		return copy;
 	}
 	spin_unlock(&clp->async_lock);
-	if (found) {
+	return NULL;
+}
+
+static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	struct nfs4_client *clp = cstate->clp;
+
+	copy = find_async_copy(clp, os);
+	if (copy) {
 		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
 		kthread_stop(copy->copy_task);
 		nfs4_put_copy(copy);
@@ -1349,7 +1357,19 @@ static int nfsd4_do_async_copy(void *data)
 		     struct nfsd4_compound_state *cstate,
 		     union nfsd4_op_u *u)
 {
-	return nfserr_notsupp;
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	struct nfs4_client *clp = cstate->clp;
+
+	copy = find_async_copy(clp, os);
+	if (copy) {
+		os->count = copy->cp_res.wr_bytes_written;
+		nfs4_put_copy(copy);
+	} else
+		status = nfserr_bad_stateid;
+
+	return status;
 }
 
 static __be32
-- 
1.8.3.1

[PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[Olga Kornievskaia]

If client is shutting down and there are still async copies going
on, then stop queued async copies.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 13 +++++++++++++
 fs/nfsd/nfs4state.c |  1 +
 fs/nfsd/state.h     |  1 +
 3 files changed, 15 insertions(+)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 57d0daa..54039e4 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1096,6 +1096,19 @@ static void nfs4_put_copy(struct nfsd4_copy *copy)
 	kfree(copy);
 }
 
+void nfsd4_shutdown_copy(struct nfs4_client *clp)
+{
+	struct nfsd4_copy *copy;
+
+	spin_lock(&clp->async_lock);
+	list_for_each_entry(copy, &clp->async_copies, copies) {
+		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
+		kthread_stop(copy->copy_task);
+		nfs4_put_copy(copy);
+	}
+	spin_unlock(&clp->async_lock);
+}
+
 static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
 {
 	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index bd76bd1..f9f2383 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1965,6 +1965,7 @@ static __be32 mark_client_expired_locked(struct nfs4_client *clp)
 		release_openowner(oo);
 	}
 	nfsd4_return_all_client_layouts(clp);
+	nfsd4_shutdown_copy(clp);
 	nfsd4_shutdown_callback(clp);
 	if (clp->cl_cb_conn.cb_xprt)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 49709d1..131aefa 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -646,6 +646,7 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern int nfsd4_create_callback_queue(void);
 extern void nfsd4_destroy_callback_queue(void);
 extern void nfsd4_shutdown_callback(struct nfs4_client *);
+extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
 extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
 							struct nfsd_net *nn);
-- 
1.8.3.1

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[J. Bruce Fields]

On Tue, Feb 20, 2018 at 11:42:24AM -0500, Olga Kornievskaia wrote:
> Upon receiving a request for async copy, create a new kthread.
> If we get asynchronous request, make sure to copy the needed
> arguments/state from the stack before starting the copy. Then
> start the thread and reply back to the client indicating copy
> is asynchronous.
> 
> nfsd_copy_file_range() will copy in a loop over the total
> number of bytes is needed to copy. In case a failure happens
> in the middle, we ignore the error and return how much we
> copied so far.

Might be worth including a comment explaining this (that we ignore the
error, and that the client can always retry after a short read if it
wants an error), maybe as a comment to the

	(bytes < 0 && !copy->cp_res.wr_bytes_written)

condition in nfsd4_do_copy().

--b.

> Once done creating a workitem for the callback
> workqueue and send CB_OFFLOAD with the results.
> 
> In the following patches the unique stateid will be generated
> for the async COPY to return, at that point will enable the
> async feature.
> 
> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
> ---
>  fs/nfsd/nfs4proc.c  | 176 ++++++++++++++++++++++++++++++++++++++++++++++------
>  fs/nfsd/nfs4state.c |   2 +
>  fs/nfsd/state.h     |   2 +
>  fs/nfsd/xdr4.h      |   9 +++
>  4 files changed, 170 insertions(+), 19 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index e11494f..91d1544 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -35,6 +35,7 @@
>  #include <linux/file.h>
>  #include <linux/falloc.h>
>  #include <linux/slab.h>
> +#include <linux/kthread.h>
>  
>  #include "idmap.h"
>  #include "cache.h"
> @@ -1083,39 +1084,176 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>  out:
>  	return status;
>  }
> +static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
> +{
> +	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
> +
> +	atomic_dec(&copy->cp_clp->cl_refcount);
> +	kfree(copy);
> +}
> +
> +static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
> +				 struct rpc_task *task)
> +{
> +	return 1;
> +}
> +
> +static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
> +	.release = nfsd4_cb_offload_release,
> +	.done = nfsd4_cb_offload_done
> +};
> +
> +static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
> +{
> +	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +		sizeof(copy->cp_dst_stateid));
> +	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> +	copy->cp_consecutive = 1;
> +	copy->cp_synchronous = sync;
> +	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
> +
> +	return nfs_ok;
> +}
> +
> +static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
> +{
> +	ssize_t bytes_copied = 0;
> +	size_t bytes_total = copy->cp_count;
> +	u64 src_pos = copy->cp_src_pos;
> +	u64 dst_pos = copy->cp_dst_pos;
> +
> +	do {
> +		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
> +				copy->fh_dst, dst_pos, bytes_total);
> +		if (bytes_copied <= 0)
> +			break;
> +		bytes_total -= bytes_copied;
> +		copy->cp_res.wr_bytes_written += bytes_copied;
> +		src_pos += bytes_copied;
> +		dst_pos += bytes_copied;
> +	} while (bytes_total > 0 && !copy->cp_synchronous);
> +	return bytes_copied;
> +}
> +
> +static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
> +{
> +	__be32 status;
> +	ssize_t bytes;
> +
> +	bytes = _nfsd_copy_file_range(copy);
> +	if (bytes < 0 && !copy->cp_res.wr_bytes_written)
> +		status = nfserrno(bytes);
> +	else
> +		status = nfsd4_init_copy_res(copy, sync);
> +
> +	fput(copy->fh_src);
> +	fput(copy->fh_dst);
> +	return status;
> +}
> +
> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
> +{
> +	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
> +	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
> +	dst->cp_src_pos = src->cp_src_pos;
> +	dst->cp_dst_pos = src->cp_dst_pos;
> +	dst->cp_count = src->cp_count;
> +	dst->cp_consecutive = src->cp_consecutive;
> +	dst->cp_synchronous = src->cp_synchronous;
> +	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
> +	/* skipping nfsd4_callback */
> +	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
> +	dst->net = src->net;
> +	dst->cp_clp = src->cp_clp;
> +	atomic_inc(&dst->cp_clp->cl_refcount);
> +	dst->fh_dst = get_file(src->fh_dst);
> +	dst->fh_src = get_file(src->fh_src);
> +}
> +
> +static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
> +{
> +	fput(copy->fh_dst);
> +	fput(copy->fh_src);
> +	if (remove) {
> +		spin_lock(&copy->cp_clp->async_lock);
> +		list_del(&copy->copies);
> +		spin_unlock(&copy->cp_clp->async_lock);
> +	}
> +	atomic_dec(&copy->cp_clp->cl_refcount);
> +	kfree(copy);
> +}
> +
> +static int nfsd4_do_async_copy(void *data)
> +{
> +	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
> +	struct nfsd4_copy *cb_copy;
> +
> +	copy->nfserr = nfsd4_do_copy(copy, 0);
> +	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> +	if (!cb_copy)
> +		goto out;
> +	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
> +	cb_copy->cp_clp = copy->cp_clp;
> +	atomic_inc(&cb_copy->cp_clp->cl_refcount);
> +	cb_copy->nfserr = copy->nfserr;
> +	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
> +	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
> +			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
> +	nfsd4_run_cb(&cb_copy->cp_cb);
> +out:
> +	cleanup_async_copy(copy, true);
> +	return 0;
> +}
>  
>  static __be32
>  nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>  		union nfsd4_op_u *u)
>  {
>  	struct nfsd4_copy *copy = &u->copy;
> -	struct file *src, *dst;
>  	__be32 status;
> -	ssize_t bytes;
> +	struct nfsd4_copy *async_copy = NULL;
>  
> -	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
> -				   &copy->cp_dst_stateid, &dst);
> +	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
> +				   &copy->fh_src, &copy->cp_dst_stateid,
> +				   &copy->fh_dst);
>  	if (status)
>  		goto out;
>  
> -	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
> -			dst, copy->cp_dst_pos, copy->cp_count);
> -
> -	if (bytes < 0)
> -		status = nfserrno(bytes);
> -	else {
> -		copy->cp_res.wr_bytes_written = bytes;
> -		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> -		copy->cp_consecutive = 1;
> -		copy->cp_synchronous = 1;
> -		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
> -		status = nfs_ok;
> +	copy->cp_clp = cstate->clp;
> +	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
> +		sizeof(struct knfsd_fh));
> +	copy->net = SVC_NET(rqstp);
> +	/* for now disable asynchronous copy feature */
> +	copy->cp_synchronous = 1;
> +	if (!copy->cp_synchronous) {
> +		status = nfsd4_init_copy_res(copy, 0);
> +		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> +		if (!async_copy) {
> +			status = nfserrno(-ENOMEM);
> +			goto out;
> +		}
> +		dup_copy_fields(copy, async_copy);
> +		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +			sizeof(copy->cp_dst_stateid));
> +		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
> +				async_copy, "%s", "copy thread");
> +		if (IS_ERR(async_copy->copy_task)) {
> +			status = PTR_ERR(async_copy->copy_task);
> +			goto out_err;
> +		}
> +		spin_lock(&async_copy->cp_clp->async_lock);
> +		list_add(&async_copy->copies,
> +				&async_copy->cp_clp->async_copies);
> +		spin_unlock(&async_copy->cp_clp->async_lock);
> +		wake_up_process(async_copy->copy_task);
> +	} else {
> +		status = nfsd4_do_copy(copy, 1);
>  	}
> -
> -	fput(src);
> -	fput(dst);
>  out:
>  	return status;
> +out_err:
> +	cleanup_async_copy(async_copy, false);
> +	goto out;
>  }
>  
>  static __be32
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 150521c..fe665b0 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1790,6 +1790,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
>  #ifdef CONFIG_NFSD_PNFS
>  	INIT_LIST_HEAD(&clp->cl_lo_states);
>  #endif
> +	INIT_LIST_HEAD(&clp->async_copies);
> +	spin_lock_init(&clp->async_lock);
>  	spin_lock_init(&clp->cl_lock);
>  	rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
>  	return clp;
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index 5c16d35..491030e 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -355,6 +355,8 @@ struct nfs4_client {
>  	struct rpc_wait_queue	cl_cb_waitq;	/* backchannel callers may */
>  						/* wait here for slots */
>  	struct net		*net;
> +	struct list_head	async_copies;	/* list of async copies */
> +	spinlock_t		async_lock;	/* lock for async copies */
>  };
>  
>  /* struct nfs4_client_reset
> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> index 4fc0e35..3b6b655 100644
> --- a/fs/nfsd/xdr4.h
> +++ b/fs/nfsd/xdr4.h
> @@ -529,6 +529,15 @@ struct nfsd4_copy {
>  	struct nfsd4_callback	cp_cb;
>  	__be32			nfserr;
>  	struct knfsd_fh		fh;
> +
> +	struct nfs4_client      *cp_clp;
> +
> +	struct file             *fh_src;
> +	struct file             *fh_dst;
> +	struct net              *net;
> +
> +	struct list_head	copies;
> +	struct task_struct	*copy_task;
>  };
>  
>  struct nfsd4_seek {
> -- 
> 1.8.3.1

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[J. Bruce Fields]

On Tue, Feb 20, 2018 at 11:42:24AM -0500, Olga Kornievskaia wrote:
> +static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
> +{
> +	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +		sizeof(copy->cp_dst_stateid));
> +	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> +	copy->cp_consecutive = 1;

The cp_consecutive field was there before you came along, but I don't
really see the point of it; all we do is ignore it and always set it to
one.  I'd just hard-code it into the xdr code:

commit f0a45f80333e
Author: J. Bruce Fields <bfields@redhat.com>
Date:   Wed Mar 7 15:37:35 2018 -0500

    nfsd: remove unsused "cp_consecutive" field
    
    Signed-off-by: J. Bruce Fields <bfields@redhat.com>

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index a0bed2b2004d..40dfb759d38d 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1106,7 +1106,6 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	else {
 		copy->cp_res.wr_bytes_written = bytes;
 		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
 		copy->cp_synchronous = 1;
 		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
 		status = nfs_ok;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index e502fd16246b..9f8ef2c6e508 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1759,7 +1759,7 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy)
 	p = xdr_decode_hyper(p, &copy->cp_src_pos);
 	p = xdr_decode_hyper(p, &copy->cp_dst_pos);
 	p = xdr_decode_hyper(p, &copy->cp_count);
-	copy->cp_consecutive = be32_to_cpup(p++);
+	p++; /* ca_consecutive: we always do consecutive copies */
 	copy->cp_synchronous = be32_to_cpup(p++);
 	tmp = be32_to_cpup(p); /* Source server list not supported */
 
@@ -4214,7 +4214,7 @@ nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr,
 		return nfserr;
 
 	p = xdr_reserve_space(&resp->xdr, 4 + 4);
-	*p++ = cpu_to_be32(copy->cp_consecutive);
+	*p++ = xdr_one; /* cr_consecutive */
 	*p++ = cpu_to_be32(copy->cp_synchronous);
 	return 0;
 }
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index bc29511b6405..7cbc129092fe 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -518,7 +518,6 @@ struct nfsd4_copy {
 	u64		cp_count;
 
 	/* both */
-	bool		cp_consecutive;
 	bool		cp_synchronous;
 
 	/* response */

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[Olga Kornievskaia]

On Tue, Mar 6, 2018 at 4:27 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Feb 20, 2018 at 11:42:24AM -0500, Olga Kornievskaia wrote:
>> Upon receiving a request for async copy, create a new kthread.
>> If we get asynchronous request, make sure to copy the needed
>> arguments/state from the stack before starting the copy. Then
>> start the thread and reply back to the client indicating copy
>> is asynchronous.
>>
>> nfsd_copy_file_range() will copy in a loop over the total
>> number of bytes is needed to copy. In case a failure happens
>> in the middle, we ignore the error and return how much we
>> copied so far.
>
> Might be worth including a comment explaining this (that we ignore the
> error, and that the client can always retry after a short read if it
> wants an error), maybe as a comment to the
>
>         (bytes < 0 && !copy->cp_res.wr_bytes_written)
>
> condition in nfsd4_do_copy().

Ok will do.

>
> --b.
>
>> Once done creating a workitem for the callback
>> workqueue and send CB_OFFLOAD with the results.
>>
>> In the following patches the unique stateid will be generated
>> for the async COPY to return, at that point will enable the
>> async feature.
>>
>> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
>> ---
>>  fs/nfsd/nfs4proc.c  | 176 ++++++++++++++++++++++++++++++++++++++++++++++------
>>  fs/nfsd/nfs4state.c |   2 +
>>  fs/nfsd/state.h     |   2 +
>>  fs/nfsd/xdr4.h      |   9 +++
>>  4 files changed, 170 insertions(+), 19 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> index e11494f..91d1544 100644
>> --- a/fs/nfsd/nfs4proc.c
>> +++ b/fs/nfsd/nfs4proc.c
>> @@ -35,6 +35,7 @@
>>  #include <linux/file.h>
>>  #include <linux/falloc.h>
>>  #include <linux/slab.h>
>> +#include <linux/kthread.h>
>>
>>  #include "idmap.h"
>>  #include "cache.h"
>> @@ -1083,39 +1084,176 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>>  out:
>>       return status;
>>  }
>> +static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
>> +{
>> +     struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
>> +
>> +     atomic_dec(&copy->cp_clp->cl_refcount);
>> +     kfree(copy);
>> +}
>> +
>> +static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
>> +                              struct rpc_task *task)
>> +{
>> +     return 1;
>> +}
>> +
>> +static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
>> +     .release = nfsd4_cb_offload_release,
>> +     .done = nfsd4_cb_offload_done
>> +};
>> +
>> +static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
>> +{
>> +     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> +             sizeof(copy->cp_dst_stateid));
>> +     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>> +     copy->cp_consecutive = 1;
>> +     copy->cp_synchronous = sync;
>> +     gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
>> +
>> +     return nfs_ok;
>> +}
>> +
>> +static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
>> +{
>> +     ssize_t bytes_copied = 0;
>> +     size_t bytes_total = copy->cp_count;
>> +     u64 src_pos = copy->cp_src_pos;
>> +     u64 dst_pos = copy->cp_dst_pos;
>> +
>> +     do {
>> +             bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
>> +                             copy->fh_dst, dst_pos, bytes_total);
>> +             if (bytes_copied <= 0)
>> +                     break;
>> +             bytes_total -= bytes_copied;
>> +             copy->cp_res.wr_bytes_written += bytes_copied;
>> +             src_pos += bytes_copied;
>> +             dst_pos += bytes_copied;
>> +     } while (bytes_total > 0 && !copy->cp_synchronous);
>> +     return bytes_copied;
>> +}
>> +
>> +static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
>> +{
>> +     __be32 status;
>> +     ssize_t bytes;
>> +
>> +     bytes = _nfsd_copy_file_range(copy);
>> +     if (bytes < 0 && !copy->cp_res.wr_bytes_written)
>> +             status = nfserrno(bytes);
>> +     else
>> +             status = nfsd4_init_copy_res(copy, sync);
>> +
>> +     fput(copy->fh_src);
>> +     fput(copy->fh_dst);
>> +     return status;
>> +}
>> +
>> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>> +{
>> +     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
>> +     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
>> +     dst->cp_src_pos = src->cp_src_pos;
>> +     dst->cp_dst_pos = src->cp_dst_pos;
>> +     dst->cp_count = src->cp_count;
>> +     dst->cp_consecutive = src->cp_consecutive;
>> +     dst->cp_synchronous = src->cp_synchronous;
>> +     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
>> +     /* skipping nfsd4_callback */
>> +     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
>> +     dst->net = src->net;
>> +     dst->cp_clp = src->cp_clp;
>> +     atomic_inc(&dst->cp_clp->cl_refcount);
>> +     dst->fh_dst = get_file(src->fh_dst);
>> +     dst->fh_src = get_file(src->fh_src);
>> +}
>> +
>> +static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>> +{
>> +     fput(copy->fh_dst);
>> +     fput(copy->fh_src);
>> +     if (remove) {
>> +             spin_lock(&copy->cp_clp->async_lock);
>> +             list_del(&copy->copies);
>> +             spin_unlock(&copy->cp_clp->async_lock);
>> +     }
>> +     atomic_dec(&copy->cp_clp->cl_refcount);
>> +     kfree(copy);
>> +}
>> +
>> +static int nfsd4_do_async_copy(void *data)
>> +{
>> +     struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
>> +     struct nfsd4_copy *cb_copy;
>> +
>> +     copy->nfserr = nfsd4_do_copy(copy, 0);
>> +     cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>> +     if (!cb_copy)
>> +             goto out;
>> +     memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
>> +     cb_copy->cp_clp = copy->cp_clp;
>> +     atomic_inc(&cb_copy->cp_clp->cl_refcount);
>> +     cb_copy->nfserr = copy->nfserr;
>> +     memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
>> +     nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
>> +                     &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
>> +     nfsd4_run_cb(&cb_copy->cp_cb);
>> +out:
>> +     cleanup_async_copy(copy, true);
>> +     return 0;
>> +}
>>
>>  static __be32
>>  nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>>               union nfsd4_op_u *u)
>>  {
>>       struct nfsd4_copy *copy = &u->copy;
>> -     struct file *src, *dst;
>>       __be32 status;
>> -     ssize_t bytes;
>> +     struct nfsd4_copy *async_copy = NULL;
>>
>> -     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
>> -                                &copy->cp_dst_stateid, &dst);
>> +     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
>> +                                &copy->fh_src, &copy->cp_dst_stateid,
>> +                                &copy->fh_dst);
>>       if (status)
>>               goto out;
>>
>> -     bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
>> -                     dst, copy->cp_dst_pos, copy->cp_count);
>> -
>> -     if (bytes < 0)
>> -             status = nfserrno(bytes);
>> -     else {
>> -             copy->cp_res.wr_bytes_written = bytes;
>> -             copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>> -             copy->cp_consecutive = 1;
>> -             copy->cp_synchronous = 1;
>> -             gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
>> -             status = nfs_ok;
>> +     copy->cp_clp = cstate->clp;
>> +     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>> +             sizeof(struct knfsd_fh));
>> +     copy->net = SVC_NET(rqstp);
>> +     /* for now disable asynchronous copy feature */
>> +     copy->cp_synchronous = 1;
>> +     if (!copy->cp_synchronous) {
>> +             status = nfsd4_init_copy_res(copy, 0);
>> +             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>> +             if (!async_copy) {
>> +                     status = nfserrno(-ENOMEM);
>> +                     goto out;
>> +             }
>> +             dup_copy_fields(copy, async_copy);
>> +             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> +                     sizeof(copy->cp_dst_stateid));
>> +             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>> +                             async_copy, "%s", "copy thread");
>> +             if (IS_ERR(async_copy->copy_task)) {
>> +                     status = PTR_ERR(async_copy->copy_task);
>> +                     goto out_err;
>> +             }
>> +             spin_lock(&async_copy->cp_clp->async_lock);
>> +             list_add(&async_copy->copies,
>> +                             &async_copy->cp_clp->async_copies);
>> +             spin_unlock(&async_copy->cp_clp->async_lock);
>> +             wake_up_process(async_copy->copy_task);
>> +     } else {
>> +             status = nfsd4_do_copy(copy, 1);
>>       }
>> -
>> -     fput(src);
>> -     fput(dst);
>>  out:
>>       return status;
>> +out_err:
>> +     cleanup_async_copy(async_copy, false);
>> +     goto out;
>>  }
>>
>>  static __be32
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index 150521c..fe665b0 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -1790,6 +1790,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
>>  #ifdef CONFIG_NFSD_PNFS
>>       INIT_LIST_HEAD(&clp->cl_lo_states);
>>  #endif
>> +     INIT_LIST_HEAD(&clp->async_copies);
>> +     spin_lock_init(&clp->async_lock);
>>       spin_lock_init(&clp->cl_lock);
>>       rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
>>       return clp;
>> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
>> index 5c16d35..491030e 100644
>> --- a/fs/nfsd/state.h
>> +++ b/fs/nfsd/state.h
>> @@ -355,6 +355,8 @@ struct nfs4_client {
>>       struct rpc_wait_queue   cl_cb_waitq;    /* backchannel callers may */
>>                                               /* wait here for slots */
>>       struct net              *net;
>> +     struct list_head        async_copies;   /* list of async copies */
>> +     spinlock_t              async_lock;     /* lock for async copies */
>>  };
>>
>>  /* struct nfs4_client_reset
>> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
>> index 4fc0e35..3b6b655 100644
>> --- a/fs/nfsd/xdr4.h
>> +++ b/fs/nfsd/xdr4.h
>> @@ -529,6 +529,15 @@ struct nfsd4_copy {
>>       struct nfsd4_callback   cp_cb;
>>       __be32                  nfserr;
>>       struct knfsd_fh         fh;
>> +
>> +     struct nfs4_client      *cp_clp;
>> +
>> +     struct file             *fh_src;
>> +     struct file             *fh_dst;
>> +     struct net              *net;
>> +
>> +     struct list_head        copies;
>> +     struct task_struct      *copy_task;
>>  };
>>
>>  struct nfsd4_seek {
>> --
>> 1.8.3.1
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[Olga Kornievskaia]

On Wed, Mar 7, 2018 at 3:38 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Feb 20, 2018 at 11:42:24AM -0500, Olga Kornievskaia wrote:
>> +static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
>> +{
>> +     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> +             sizeof(copy->cp_dst_stateid));
>> +     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>> +     copy->cp_consecutive = 1;
>
> The cp_consecutive field was there before you came along, but I don't
> really see the point of it; all we do is ignore it and always set it to
> one.  I'd just hard-code it into the xdr code:
>
> commit f0a45f80333e
> Author: J. Bruce Fields <bfields@redhat.com>
> Date:   Wed Mar 7 15:37:35 2018 -0500
>
>     nfsd: remove unsused "cp_consecutive" field
>
>     Signed-off-by: J. Bruce Fields <bfields@redhat.com>
>
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index a0bed2b2004d..40dfb759d38d 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -1106,7 +1106,6 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>         else {
>                 copy->cp_res.wr_bytes_written = bytes;
>                 copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> -               copy->cp_consecutive = 1;
>                 copy->cp_synchronous = 1;
>                 gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
>                 status = nfs_ok;
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index e502fd16246b..9f8ef2c6e508 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -1759,7 +1759,7 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy)
>         p = xdr_decode_hyper(p, &copy->cp_src_pos);
>         p = xdr_decode_hyper(p, &copy->cp_dst_pos);
>         p = xdr_decode_hyper(p, &copy->cp_count);
> -       copy->cp_consecutive = be32_to_cpup(p++);
> +       p++; /* ca_consecutive: we always do consecutive copies */
>         copy->cp_synchronous = be32_to_cpup(p++);
>         tmp = be32_to_cpup(p); /* Source server list not supported */
>
> @@ -4214,7 +4214,7 @@ nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr,
>                 return nfserr;
>
>         p = xdr_reserve_space(&resp->xdr, 4 + 4);
> -       *p++ = cpu_to_be32(copy->cp_consecutive);
> +       *p++ = xdr_one; /* cr_consecutive */
>         *p++ = cpu_to_be32(copy->cp_synchronous);
>         return 0;
>  }
> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> index bc29511b6405..7cbc129092fe 100644
> --- a/fs/nfsd/xdr4.h
> +++ b/fs/nfsd/xdr4.h
> @@ -518,7 +518,6 @@ struct nfsd4_copy {
>         u64             cp_count;
>
>         /* both */
> -       bool            cp_consecutive;
>         bool            cp_synchronous;
>
>         /* response */

Ok will add this.

> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[J. Bruce Fields]

It looks like "struct nfsd_copy" is being used in at least three
different ways: to hold the xdr arguments and reply, to hold the state
needed while doing a copy, and to hold the callback arguments.  I wonder
if the code would be clearer if those were three different data
structures.

> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
> +{
> +	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
> +	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
> +	dst->cp_src_pos = src->cp_src_pos;
> +	dst->cp_dst_pos = src->cp_dst_pos;
> +	dst->cp_count = src->cp_count;
> +	dst->cp_consecutive = src->cp_consecutive;
> +	dst->cp_synchronous = src->cp_synchronous;
> +	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
> +	/* skipping nfsd4_callback */
> +	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
> +	dst->net = src->net;
> +	dst->cp_clp = src->cp_clp;
> +	atomic_inc(&dst->cp_clp->cl_refcount);
> +	dst->fh_dst = get_file(src->fh_dst);
> +	dst->fh_src = get_file(src->fh_src);

Just another minor gripe, but: could we name those fd_* or file_* or
something instead of fh?  I tend to assume "fh" means "nfs filehandle".

> +}
> +
> +static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
> +{
> +	fput(copy->fh_dst);
> +	fput(copy->fh_src);
> +	if (remove) {
> +		spin_lock(&copy->cp_clp->async_lock);
> +		list_del(&copy->copies);
> +		spin_unlock(&copy->cp_clp->async_lock);
> +	}

I don't understand yet why you need these two cases.  Anyway, I'd rather
you did this in the caller.  So the caller can do either:

	spin_lock()
	list_del()
	spin_unlock()
	cleanup_async_copy()

or just

	cleanup_async_copy()

Or define another helper for the first case if you're really doing it a
lot.  Just don't make me have to remember what that second argument
means each time I see it used.

> +	atomic_dec(&copy->cp_clp->cl_refcount);
> +	kfree(copy);
> +}
...
> +	copy->cp_clp = cstate->clp;
> +	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
> +		sizeof(struct knfsd_fh));
> +	copy->net = SVC_NET(rqstp);
> +	/* for now disable asynchronous copy feature */
> +	copy->cp_synchronous = 1;
> +	if (!copy->cp_synchronous) {
> +		status = nfsd4_init_copy_res(copy, 0);
> +		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> +		if (!async_copy) {
> +			status = nfserrno(-ENOMEM);
> +			goto out;
> +		}
> +		dup_copy_fields(copy, async_copy);
> +		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +			sizeof(copy->cp_dst_stateid));
> +		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
> +				async_copy, "%s", "copy thread");
> +		if (IS_ERR(async_copy->copy_task)) {
> +			status = PTR_ERR(async_copy->copy_task);

status should be an nfs error.

--b.

> +			goto out_err;
> +		}
> +		spin_lock(&async_copy->cp_clp->async_lock);
> +		list_add(&async_copy->copies,
> +				&async_copy->cp_clp->async_copies);
> +		spin_unlock(&async_copy->cp_clp->async_lock);
> +		wake_up_process(async_copy->copy_task);
> +	} else {
> +		status = nfsd4_do_copy(copy, 1);
>  	}
> -
> -	fput(src);
> -	fput(dst);
>  out:
>  	return status;
> +out_err:
> +	cleanup_async_copy(async_copy, false);
> +	goto out;
>  }
>  
>  static __be32

Re: [PATCH v7 06/10] NFSD return nfs4_stid in nfs4_preprocess_stateid_op

[J. Bruce Fields]

On Tue, Feb 20, 2018 at 11:42:25AM -0500, Olga Kornievskaia wrote:
> @@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
>  	if (!status && filpp)
>  		status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
>  out:
> -	if (s)
> +	if (s) {
> +		if (!status && cstid)
> +			*cstid = s;
>  		nfs4_put_stid(s);

Are you sure that's right?  Normally you wouldn't drop a reference on
something you're returning.

--b.

Re: [PATCH v7 06/10] NFSD return nfs4_stid in nfs4_preprocess_stateid_op

[Olga Kornievskaia]

On Wed, Mar 7, 2018 at 4:43 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Feb 20, 2018 at 11:42:25AM -0500, Olga Kornievskaia wrote:
>> @@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
>>       if (!status && filpp)
>>               status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
>>  out:
>> -     if (s)
>> +     if (s) {
>> +             if (!status && cstid)
>> +                     *cstid = s;
>>               nfs4_put_stid(s);
>
> Are you sure that's right?  Normally you wouldn't drop a reference on
> something you're returning.

Hm. Should it be taken here then? I do up the reference on the stateid
later in the code in the nfds4_copy().

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[Olga Kornievskaia]

On Wed, Mar 7, 2018 at 4:05 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> It looks like "struct nfsd_copy" is being used in at least three
> different ways: to hold the xdr arguments and reply, to hold the state
> needed while doing a copy, and to hold the callback arguments.  I wonder
> if the code would be clearer if those were three different data
> structures.

I'll take a look and see what I could do.

>> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>> +{
>> +     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
>> +     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
>> +     dst->cp_src_pos = src->cp_src_pos;
>> +     dst->cp_dst_pos = src->cp_dst_pos;
>> +     dst->cp_count = src->cp_count;
>> +     dst->cp_consecutive = src->cp_consecutive;
>> +     dst->cp_synchronous = src->cp_synchronous;
>> +     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
>> +     /* skipping nfsd4_callback */
>> +     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
>> +     dst->net = src->net;
>> +     dst->cp_clp = src->cp_clp;
>> +     atomic_inc(&dst->cp_clp->cl_refcount);
>> +     dst->fh_dst = get_file(src->fh_dst);
>> +     dst->fh_src = get_file(src->fh_src);
>
> Just another minor gripe, but: could we name those fd_* or file_* or
> something instead of fh?  I tend to assume "fh" means "nfs filehandle".

Ok. "file_dst" and "file_src" it will be.


>
>> +}
>> +
>> +static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>> +{
>> +     fput(copy->fh_dst);
>> +     fput(copy->fh_src);
>> +     if (remove) {
>> +             spin_lock(&copy->cp_clp->async_lock);
>> +             list_del(&copy->copies);
>> +             spin_unlock(&copy->cp_clp->async_lock);
>> +     }
>
> I don't understand yet why you need these two cases.  Anyway, I'd rather
> you did this in the caller.  So the caller can do either:
>
>         spin_lock()
>         list_del()
>         spin_unlock()
>         cleanup_async_copy()
>
> or just
>
>         cleanup_async_copy()
>
> Or define another helper for the first case if you're really doing it a
> lot.  Just don't make me have to remember what that second argument
> means each time I see it used.

Ok I'll fix it. It was added after I changed where copy was added to
the list. In one case, now I needed to do all of the cleanup but the
copy wasn't added to the list yet.

>> +     atomic_dec(&copy->cp_clp->cl_refcount);
>> +     kfree(copy);
>> +}
> ...
>> +     copy->cp_clp = cstate->clp;
>> +     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>> +             sizeof(struct knfsd_fh));
>> +     copy->net = SVC_NET(rqstp);
>> +     /* for now disable asynchronous copy feature */
>> +     copy->cp_synchronous = 1;
>> +     if (!copy->cp_synchronous) {
>> +             status = nfsd4_init_copy_res(copy, 0);
>> +             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>> +             if (!async_copy) {
>> +                     status = nfserrno(-ENOMEM);
>> +                     goto out;
>> +             }
>> +             dup_copy_fields(copy, async_copy);
>> +             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> +                     sizeof(copy->cp_dst_stateid));
>> +             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>> +                             async_copy, "%s", "copy thread");
>> +             if (IS_ERR(async_copy->copy_task)) {
>> +                     status = PTR_ERR(async_copy->copy_task);
>
> status should be an nfs error.

Will fix that.

>
> --b.
>
>> +                     goto out_err;
>> +             }
>> +             spin_lock(&async_copy->cp_clp->async_lock);
>> +             list_add(&async_copy->copies,
>> +                             &async_copy->cp_clp->async_copies);
>> +             spin_unlock(&async_copy->cp_clp->async_lock);
>> +             wake_up_process(async_copy->copy_task);
>> +     } else {
>> +             status = nfsd4_do_copy(copy, 1);
>>       }
>> -
>> -     fput(src);
>> -     fput(dst);
>>  out:
>>       return status;
>> +out_err:
>> +     cleanup_async_copy(async_copy, false);
>> +     goto out;
>>  }
>>
>>  static __be32
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[J. Bruce Fields]

On Wed, Mar 07, 2018 at 05:06:21PM -0500, Olga Kornievskaia wrote:
> On Wed, Mar 7, 2018 at 4:05 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> > I don't understand yet why you need these two cases.  Anyway, I'd rather
> > you did this in the caller.  So the caller can do either:
> >
> >         spin_lock()
> >         list_del()
> >         spin_unlock()
> >         cleanup_async_copy()
> >
> > or just
> >
> >         cleanup_async_copy()
> >
> > Or define another helper for the first case if you're really doing it a
> > lot.  Just don't make me have to remember what that second argument
> > means each time I see it used.
> 
> Ok I'll fix it. It was added after I changed where copy was added to
> the list. In one case, now I needed to do all of the cleanup but the
> copy wasn't added to the list yet.

As long as you've at least initialized the list, it's OK to call
list_del() on it.  So another alternative might just be to call
list_del() in both cases.

--b.

Re: [PATCH v7 06/10] NFSD return nfs4_stid in nfs4_preprocess_stateid_op

[J. Bruce Fields]

On Wed, Mar 07, 2018 at 04:54:50PM -0500, Olga Kornievskaia wrote:
> On Wed, Mar 7, 2018 at 4:43 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> > On Tue, Feb 20, 2018 at 11:42:25AM -0500, Olga Kornievskaia wrote:
> >> @@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
> >>       if (!status && filpp)
> >>               status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
> >>  out:
> >> -     if (s)
> >> +     if (s) {
> >> +             if (!status && cstid)
> >> +                     *cstid = s;
> >>               nfs4_put_stid(s);
> >
> > Are you sure that's right?  Normally you wouldn't drop a reference on
> > something you're returning.
> 
> Hm. Should it be taken here then? I do up the reference on the stateid
> later in the code in the nfds4_copy().

Yeah, you probably want

		if (!status && cstid)
			*cstid = s;
		else
			nfs4_put_stid(s);

instead of doing the increment later in nfsd4_copy() code.

Otherwise, in theory a close or free_stateid or something could
race in and destroy the stateid before you get the chance to increment
the reference again.

--b.

Re: [PATCH v7 07/10] NFSD create new stateid for async copy

[J. Bruce Fields]

On Tue, Feb 20, 2018 at 11:42:26AM -0500, Olga Kornievskaia wrote:
> Generate a new stateid to be used for reply to the asynchronous
> COPY (this would also be used later by COPY_NOTIFY as well).
> Associate the stateid with the parent OPEN/LOCK/DELEG stateid
> that can be freed during the free of the parent stateid. However,
> right now deciding to bind the lifetime to when the vfs copy
> is done. This way don't need to keep the nfsd_net structure for
> the callback. The drawback is that time copy state information
> is available for query by OFFLOAD_STATUS is slightly less.

I don't understand yet the purpose of the parent stateid.

I understand that it's part of the COPY arguments and that we need to
check it when we first process the COPY.  But what's it used for after
that?  Why do we need to keep it around?

--b.

> 
> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
> ---
>  fs/nfsd/netns.h     |  8 +++++++
>  fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
>  fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  fs/nfsd/nfsctl.c    |  1 +
>  fs/nfsd/state.h     | 14 +++++++++++
>  fs/nfsd/xdr4.h      |  2 ++
>  6 files changed, 118 insertions(+), 10 deletions(-)
> 
> diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
> index 36358d4..e83cf6e 100644
> --- a/fs/nfsd/netns.h
> +++ b/fs/nfsd/netns.h
> @@ -122,6 +122,14 @@ struct nfsd_net {
>  
>  	wait_queue_head_t ntf_wq;
>  	atomic_t ntf_refcnt;
> +
> +	/*
> +	 * clientid and stateid data for construction of net unique COPY
> +	 * stateids.
> +	 */
> +	u32		s2s_cp_cl_id;
> +	struct idr	s2s_cp_stateids;
> +	spinlock_t	s2s_cp_lock;
>  };
>  
>  /* Simple check to find out if a given net was properly initialized */
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index aafd5a58..eb9f528 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>  static __be32
>  nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>  		  stateid_t *src_stateid, struct file **src,
> -		  stateid_t *dst_stateid, struct file **dst)
> +		  stateid_t *dst_stateid, struct file **dst,
> +		  struct nfs4_stid **stid)
>  {
>  	__be32 status;
>  
> @@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>  
>  	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
>  					    dst_stateid, WR_STATE, dst, NULL,
> -					    NULL);
> +					    stid);
>  	if (status) {
>  		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
>  		goto out_put_src;
> @@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>  	__be32 status;
>  
>  	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
> -				   &clone->cl_dst_stateid, &dst);
> +				   &clone->cl_dst_stateid, &dst, NULL);
>  	if (status)
>  		goto out;
>  
> @@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
>  
>  static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
>  {
> -	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> -		sizeof(copy->cp_dst_stateid));
>  	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>  	copy->cp_consecutive = 1;
>  	copy->cp_synchronous = sync;
> @@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>  	atomic_inc(&dst->cp_clp->cl_refcount);
>  	dst->fh_dst = get_file(src->fh_dst);
>  	dst->fh_src = get_file(src->fh_src);
> +	dst->stid = src->stid;
> +	dst->cps = src->cps;
>  }
>  
>  static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>  {
> +	list_del(&copy->cps->cp_list);
> +	nfs4_free_cp_state(copy->cps);
> +	nfs4_put_stid(copy->stid);
>  	fput(copy->fh_dst);
>  	fput(copy->fh_src);
>  	if (remove) {
> @@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
>  
>  	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
>  				   &copy->fh_src, &copy->cp_dst_stateid,
> -				   &copy->fh_dst);
> +				   &copy->fh_dst, &copy->stid);
>  	if (status)
>  		goto out;
>  
> @@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
>  	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>  		sizeof(struct knfsd_fh));
>  	copy->net = SVC_NET(rqstp);
> -	/* for now disable asynchronous copy feature */
> -	copy->cp_synchronous = 1;
>  	if (!copy->cp_synchronous) {
> +		struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
> +
>  		status = nfsd4_init_copy_res(copy, 0);
>  		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>  		if (!async_copy) {
>  			status = nfserrno(-ENOMEM);
>  			goto out;
>  		}
> +		copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
> +		if (!copy->cps) {
> +			status = nfserrno(-ENOMEM);
> +			kfree(async_copy);
> +			goto out;
> +		}
> +		/* take a reference on the parent stateid so it's not
> +		 * not freed by the copy compound
> +		 */
> +		refcount_inc(&copy->stid->sc_count);
> +		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
> +			sizeof(copy->cps->cp_stateid));
>  		dup_copy_fields(copy, async_copy);
> -		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> -			sizeof(copy->cp_dst_stateid));
>  		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>  				async_copy, "%s", "copy thread");
>  		if (IS_ERR(async_copy->copy_task)) {
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index e05832b..bd76bd1 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>  	/* Will be incremented before return to client: */
>  	refcount_set(&stid->sc_count, 1);
>  	spin_lock_init(&stid->sc_lock);
> +	INIT_LIST_HEAD(&stid->sc_cp_list);
>  
>  	/*
>  	 * It shouldn't be a problem to reuse an opaque stateid value.
> @@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>  	return NULL;
>  }
>  
> +/*
> + * Create a unique stateid_t to represent each COPY. Hang the copy
> + * stateids off the OPEN/LOCK/DELEG stateid from the client open
> + * the source file.
> + */
> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
> +					       struct nfs4_stid *p_stid)
> +{
> +	struct nfs4_cp_state *cps;
> +	int new_id;
> +
> +	cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
> +	if (!cps)
> +		return NULL;
> +	idr_preload(GFP_KERNEL);
> +	spin_lock(&nn->s2s_cp_lock);
> +	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
> +	spin_unlock(&nn->s2s_cp_lock);
> +	idr_preload_end();
> +	if (new_id < 0)
> +		goto out_free;
> +	cps->cp_stateid.si_opaque.so_id = new_id;
> +	cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
> +	cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
> +	cps->cp_p_stid = p_stid;
> +	spin_lock(&p_stid->sc_lock);
> +	list_add(&cps->cp_list, &p_stid->sc_cp_list);
> +	spin_unlock(&p_stid->sc_lock);
> +
> +	return cps;
> +out_free:
> +	kfree(cps);
> +	return NULL;
> +}
> +
> +void nfs4_free_cp_state(struct nfs4_cp_state *cps)
> +{
> +	struct nfsd_net *nn;
> +
> +	nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
> +	spin_lock(&nn->s2s_cp_lock);
> +	idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
> +	spin_unlock(&nn->s2s_cp_lock);
> +
> +	kfree(cps);
> +}
> +
> +static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
> +{
> +	struct nfs4_cp_state *cps;
> +
> +	might_sleep();
> +
> +	spin_lock(&stid->sc_lock);
> +	while (!list_empty(&stid->sc_cp_list)) {
> +		cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
> +				       cp_list);
> +		list_del(&cps->cp_list);
> +		nfs4_free_cp_state(cps);
> +	}
> +	spin_unlock(&stid->sc_lock);
> +}
> +
>  static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
>  {
>  	struct nfs4_stid *stid;
> @@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
>  	}
>  	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
>  	spin_unlock(&clp->cl_lock);
> +
> +	nfs4_free_cp_statelist(s);
> +
>  	s->sc_free(s);
>  	if (fp)
>  		put_nfs4_file(fp);
> @@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
>  	INIT_LIST_HEAD(&nn->close_lru);
>  	INIT_LIST_HEAD(&nn->del_recall_lru);
>  	spin_lock_init(&nn->client_lock);
> +	spin_lock_init(&nn->s2s_cp_lock);
> +	idr_init(&nn->s2s_cp_stateids);
>  
>  	spin_lock_init(&nn->blocked_locks_lock);
>  	INIT_LIST_HEAD(&nn->blocked_locks_lru);
> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
> index d107b44..63edf68 100644
> --- a/fs/nfsd/nfsctl.c
> +++ b/fs/nfsd/nfsctl.c
> @@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
>  	nn->nfsd4_grace = 90;
>  	nn->clverifier_counter = prandom_u32();
>  	nn->clientid_counter = prandom_u32();
> +	nn->s2s_cp_cl_id = nn->clientid_counter++;
>  
>  	atomic_set(&nn->ntf_refcnt, 0);
>  	init_waitqueue_head(&nn->ntf_wq);
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index 9b7d7a0..49709d1 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -94,6 +94,7 @@ struct nfs4_stid {
>  #define NFS4_REVOKED_DELEG_STID 16
>  #define NFS4_CLOSED_DELEG_STID 32
>  #define NFS4_LAYOUT_STID 64
> +	struct list_head	sc_cp_list;
>  	unsigned char		sc_type;
>  	stateid_t		sc_stateid;
>  	spinlock_t		sc_lock;
> @@ -103,6 +104,17 @@ struct nfs4_stid {
>  };
>  
>  /*
> + * Keep a list of stateids issued by the COPY, associate it with the
> + * parent OPEN/LOCK/DELEG stateid. Used for lookup by
> + * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
> + */
> +struct nfs4_cp_state {
> +	stateid_t		cp_stateid;
> +	struct list_head	cp_list;	/* per parent nfs4_stid */
> +	struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
> +};
> +
> +/*
>   * Represents a delegation stateid. The nfs4_client holds references to these
>   * and they are put when it is being destroyed or when the delegation is
>   * returned by the client:
> @@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
>  		     struct nfs4_stid **s, struct nfsd_net *nn);
>  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
>  				  void (*sc_free)(struct nfs4_stid *));
> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
> +void nfs4_free_cp_state(struct nfs4_cp_state *cps);
>  void nfs4_unhash_stid(struct nfs4_stid *s);
>  void nfs4_put_stid(struct nfs4_stid *s);
>  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> index 3b6b655..5d493b9 100644
> --- a/fs/nfsd/xdr4.h
> +++ b/fs/nfsd/xdr4.h
> @@ -535,6 +535,8 @@ struct nfsd4_copy {
>  	struct file             *fh_src;
>  	struct file             *fh_dst;
>  	struct net              *net;
> +	struct nfs4_stid        *stid;
> +	struct nfs4_cp_state    *cps;
>  
>  	struct list_head	copies;
>  	struct task_struct	*copy_task;
> -- 
> 1.8.3.1

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[J. Bruce Fields]

On Tue, Feb 20, 2018 at 11:42:29AM -0500, Olga Kornievskaia wrote:
> If client is shutting down and there are still async copies going
> on, then stop queued async copies.
> 
> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
> ---
>  fs/nfsd/nfs4proc.c  | 13 +++++++++++++
>  fs/nfsd/nfs4state.c |  1 +
>  fs/nfsd/state.h     |  1 +
>  3 files changed, 15 insertions(+)
> 
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index 57d0daa..54039e4 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -1096,6 +1096,19 @@ static void nfs4_put_copy(struct nfsd4_copy *copy)
>  	kfree(copy);
>  }
>  
> +void nfsd4_shutdown_copy(struct nfs4_client *clp)
> +{
> +	struct nfsd4_copy *copy;
> +
> +	spin_lock(&clp->async_lock);
> +	list_for_each_entry(copy, &clp->async_copies, copies) {
> +		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
> +		kthread_stop(copy->copy_task);
> +		nfs4_put_copy(copy);

Those three lines could go in a common helper to be called here and from
nfsd4_offload_cancel().

--b.

> +	}
> +	spin_unlock(&clp->async_lock);
> +}
> +
>  static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
>  {
>  	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index bd76bd1..f9f2383 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1965,6 +1965,7 @@ static __be32 mark_client_expired_locked(struct nfs4_client *clp)
>  		release_openowner(oo);
>  	}
>  	nfsd4_return_all_client_layouts(clp);
> +	nfsd4_shutdown_copy(clp);
>  	nfsd4_shutdown_callback(clp);
>  	if (clp->cl_cb_conn.cb_xprt)
>  		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index 49709d1..131aefa 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -646,6 +646,7 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
>  extern int nfsd4_create_callback_queue(void);
>  extern void nfsd4_destroy_callback_queue(void);
>  extern void nfsd4_shutdown_callback(struct nfs4_client *);
> +extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
>  extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
>  extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
>  							struct nfsd_net *nn);
> -- 
> 1.8.3.1

Re: [PATCH v7 05/10] NFSD introduce asynch copy feature

[Olga Kornievskaia]

On Wed, Mar 7, 2018 at 4:05 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> It looks like "struct nfsd_copy" is being used in at least three
> different ways: to hold the xdr arguments and reply, to hold the state
> needed while doing a copy, and to hold the callback arguments.  I wonder
> if the code would be clearer if those were three different data
> structures.

I'm having trouble with this one. Because things are done in 3
different running contexts and we need to pass information between
them then having a single data structure make more sense to me.
Furthermore, because there is also synchronous and asynchronous copies
then things like state and xdr needs to be accessed by the original
thread for the synchronous case.

nfsd4_copy structure needs to the passed into xdr decode/encode. But
for instance, offsets, count, synchronous need to be also a part of
the "copy_state" structure. xdr encode can be done from either the
main thread or the callback thread. to pass into the callback thread,
it first needs to be copied to the copy thread. only a single data
structure can be passed as arguments into the thread start function.

I have detailed each of the fields of where they are used (the intent
is to specify the need to access from multiple places). I have located
3 fields that I think don't need to be dup_copy_fields().  But I
haven't tested it yet.

struct nfsd4_copy {
       /* request */
        stateid_t       cp_src_stateid;  xdr args field needed by the
main copy (i should skip copying it in dup_copy_fields)
        stateid_t       cp_dst_stateid; xdr args field needed by the
main copy (i should skip copying it in the dup_copy_fileds)
        u64             cp_src_pos; xdr args field needed by the main
copy and then copy thread
        u64             cp_dst_pos; xdr args field needed by the main
copy and then copy thread
        u64             cp_count; xdr field needed by the main copy
and then copy thread

        /* both */
        bool            cp_synchronous; xdr args/res field need by the
main copy and then copy thread

        /* response */
        struct nfsd42_write_res cp_res; xdr res field needed by the
main copy and copy thread and callback thread

        /* for cb_offload */
        struct nfsd4_callback cp_cb; needed by the callback but has to
be setup by the structure not on the stack
        __be32                        nfserr; copy state info needed
by the main thread, copy thread, and callback thread
        struct knfsd_fh               fh; copy state available in the
main thread then copied to the copy thread and needed by the callback
thread

        struct nfs4_client      *cp_clp; copy state info needed by the
main thread, copy thread, and callback thread

        struct file             *file_src; copy state info set by the
main thread, needed by the copy thread
        struct file             *file_dst; copy state info set by the
main thread, needed by the copy thread
        struct net              *net; copy state info available/set in
the main thread, needed by the main thread and the copy thread
        struct nfs4_stid        *stid; copy state info available/set
in the main thread, needed by the main thread. (i should skip copying
it in the dup_copy_fields. Now I don't recall how "inter" copy uses
this.)
        struct nfs4_cp_state    *cps; copy state info set in the main
thread, needed by the callback so copied thru the copy thread

        struct list_head        copies; copy state information, needed
by the main thread and other parallel operations
        struct task_struct      *copy_task; copy state information/
        refcount_t              refcount; copy state information
needed by the main thread, needed by the callback so copied thru the
copy thread
        bool                    stopped; copy state information needed
by the copy thread, used by parallel threads (offload_cancel, close).

I also want to say that the data structure (will be) is used by the
"inter/intra" copy so some fields are there before they make more
sense in the "inter" case. Like the nfs4_cp_state is used for the
inter but the generated stateid is saved in that data structure
because it's share by what's COPY_NOTIFY will use to store it's copy
stateids it gives out and what COPY uses.

>
>> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>> +{
>> +     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
>> +     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
>> +     dst->cp_src_pos = src->cp_src_pos;
>> +     dst->cp_dst_pos = src->cp_dst_pos;
>> +     dst->cp_count = src->cp_count;
>> +     dst->cp_consecutive = src->cp_consecutive;
>> +     dst->cp_synchronous = src->cp_synchronous;
>> +     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
>> +     /* skipping nfsd4_callback */
>> +     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
>> +     dst->net = src->net;
>> +     dst->cp_clp = src->cp_clp;
>> +     atomic_inc(&dst->cp_clp->cl_refcount);
>> +     dst->fh_dst = get_file(src->fh_dst);
>> +     dst->fh_src = get_file(src->fh_src);
>
> Just another minor gripe, but: could we name those fd_* or file_* or
> something instead of fh?  I tend to assume "fh" means "nfs filehandle".
>
>> +}
>> +
>> +static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>> +{
>> +     fput(copy->fh_dst);
>> +     fput(copy->fh_src);
>> +     if (remove) {
>> +             spin_lock(&copy->cp_clp->async_lock);
>> +             list_del(&copy->copies);
>> +             spin_unlock(&copy->cp_clp->async_lock);
>> +     }
>
> I don't understand yet why you need these two cases.  Anyway, I'd rather
> you did this in the caller.  So the caller can do either:
>
>         spin_lock()
>         list_del()
>         spin_unlock()
>         cleanup_async_copy()
>
> or just
>
>         cleanup_async_copy()
>
> Or define another helper for the first case if you're really doing it a
> lot.  Just don't make me have to remember what that second argument
> means each time I see it used.
>
>> +     atomic_dec(&copy->cp_clp->cl_refcount);
>> +     kfree(copy);
>> +}
> ...
>> +     copy->cp_clp = cstate->clp;
>> +     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>> +             sizeof(struct knfsd_fh));
>> +     copy->net = SVC_NET(rqstp);
>> +     /* for now disable asynchronous copy feature */
>> +     copy->cp_synchronous = 1;
>> +     if (!copy->cp_synchronous) {
>> +             status = nfsd4_init_copy_res(copy, 0);
>> +             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>> +             if (!async_copy) {
>> +                     status = nfserrno(-ENOMEM);
>> +                     goto out;
>> +             }
>> +             dup_copy_fields(copy, async_copy);
>> +             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> +                     sizeof(copy->cp_dst_stateid));
>> +             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>> +                             async_copy, "%s", "copy thread");
>> +             if (IS_ERR(async_copy->copy_task)) {
>> +                     status = PTR_ERR(async_copy->copy_task);
>
> status should be an nfs error.
>
> --b.
>
>> +                     goto out_err;
>> +             }
>> +             spin_lock(&async_copy->cp_clp->async_lock);
>> +             list_add(&async_copy->copies,
>> +                             &async_copy->cp_clp->async_copies);
>> +             spin_unlock(&async_copy->cp_clp->async_lock);
>> +             wake_up_process(async_copy->copy_task);
>> +     } else {
>> +             status = nfsd4_do_copy(copy, 1);
>>       }
>> -
>> -     fput(src);
>> -     fput(dst);
>>  out:
>>       return status;
>> +out_err:
>> +     cleanup_async_copy(async_copy, false);
>> +     goto out;
>>  }
>>
>>  static __be32
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 07/10] NFSD create new stateid for async copy

[Olga Kornievskaia]

On Thu, Mar 8, 2018 at 11:31 AM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Feb 20, 2018 at 11:42:26AM -0500, Olga Kornievskaia wrote:
>> Generate a new stateid to be used for reply to the asynchronous
>> COPY (this would also be used later by COPY_NOTIFY as well).
>> Associate the stateid with the parent OPEN/LOCK/DELEG stateid
>> that can be freed during the free of the parent stateid. However,
>> right now deciding to bind the lifetime to when the vfs copy
>> is done. This way don't need to keep the nfsd_net structure for
>> the callback. The drawback is that time copy state information
>> is available for query by OFFLOAD_STATUS is slightly less.
>
> I don't understand yet the purpose of the parent stateid.
>
> I understand that it's part of the COPY arguments and that we need to
> check it when we first process the COPY.  But what's it used for after
> that?  Why do we need to keep it around?

Parent stateid tie is needed for when the "CLOSE" is received. CLOSE
only has the parent state. We have a global list of async copies and
then each copy has a pointer to parent to provide the match.

>
> --b.
>
>>
>> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
>> ---
>>  fs/nfsd/netns.h     |  8 +++++++
>>  fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
>>  fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>  fs/nfsd/nfsctl.c    |  1 +
>>  fs/nfsd/state.h     | 14 +++++++++++
>>  fs/nfsd/xdr4.h      |  2 ++
>>  6 files changed, 118 insertions(+), 10 deletions(-)
>>
>> diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
>> index 36358d4..e83cf6e 100644
>> --- a/fs/nfsd/netns.h
>> +++ b/fs/nfsd/netns.h
>> @@ -122,6 +122,14 @@ struct nfsd_net {
>>
>>       wait_queue_head_t ntf_wq;
>>       atomic_t ntf_refcnt;
>> +
>> +     /*
>> +      * clientid and stateid data for construction of net unique COPY
>> +      * stateids.
>> +      */
>> +     u32             s2s_cp_cl_id;
>> +     struct idr      s2s_cp_stateids;
>> +     spinlock_t      s2s_cp_lock;
>>  };
>>
>>  /* Simple check to find out if a given net was properly initialized */
>> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> index aafd5a58..eb9f528 100644
>> --- a/fs/nfsd/nfs4proc.c
>> +++ b/fs/nfsd/nfs4proc.c
>> @@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>>  static __be32
>>  nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>>                 stateid_t *src_stateid, struct file **src,
>> -               stateid_t *dst_stateid, struct file **dst)
>> +               stateid_t *dst_stateid, struct file **dst,
>> +               struct nfs4_stid **stid)
>>  {
>>       __be32 status;
>>
>> @@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>>
>>       status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
>>                                           dst_stateid, WR_STATE, dst, NULL,
>> -                                         NULL);
>> +                                         stid);
>>       if (status) {
>>               dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
>>               goto out_put_src;
>> @@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>>       __be32 status;
>>
>>       status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
>> -                                &clone->cl_dst_stateid, &dst);
>> +                                &clone->cl_dst_stateid, &dst, NULL);
>>       if (status)
>>               goto out;
>>
>> @@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
>>
>>  static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
>>  {
>> -     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> -             sizeof(copy->cp_dst_stateid));
>>       copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>>       copy->cp_consecutive = 1;
>>       copy->cp_synchronous = sync;
>> @@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>>       atomic_inc(&dst->cp_clp->cl_refcount);
>>       dst->fh_dst = get_file(src->fh_dst);
>>       dst->fh_src = get_file(src->fh_src);
>> +     dst->stid = src->stid;
>> +     dst->cps = src->cps;
>>  }
>>
>>  static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>>  {
>> +     list_del(&copy->cps->cp_list);
>> +     nfs4_free_cp_state(copy->cps);
>> +     nfs4_put_stid(copy->stid);
>>       fput(copy->fh_dst);
>>       fput(copy->fh_src);
>>       if (remove) {
>> @@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
>>
>>       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
>>                                  &copy->fh_src, &copy->cp_dst_stateid,
>> -                                &copy->fh_dst);
>> +                                &copy->fh_dst, &copy->stid);
>>       if (status)
>>               goto out;
>>
>> @@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
>>       memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>>               sizeof(struct knfsd_fh));
>>       copy->net = SVC_NET(rqstp);
>> -     /* for now disable asynchronous copy feature */
>> -     copy->cp_synchronous = 1;
>>       if (!copy->cp_synchronous) {
>> +             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
>> +
>>               status = nfsd4_init_copy_res(copy, 0);
>>               async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>>               if (!async_copy) {
>>                       status = nfserrno(-ENOMEM);
>>                       goto out;
>>               }
>> +             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
>> +             if (!copy->cps) {
>> +                     status = nfserrno(-ENOMEM);
>> +                     kfree(async_copy);
>> +                     goto out;
>> +             }
>> +             /* take a reference on the parent stateid so it's not
>> +              * not freed by the copy compound
>> +              */
>> +             refcount_inc(&copy->stid->sc_count);
>> +             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
>> +                     sizeof(copy->cps->cp_stateid));
>>               dup_copy_fields(copy, async_copy);
>> -             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> -                     sizeof(copy->cp_dst_stateid));
>>               async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>>                               async_copy, "%s", "copy thread");
>>               if (IS_ERR(async_copy->copy_task)) {
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index e05832b..bd76bd1 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>>       /* Will be incremented before return to client: */
>>       refcount_set(&stid->sc_count, 1);
>>       spin_lock_init(&stid->sc_lock);
>> +     INIT_LIST_HEAD(&stid->sc_cp_list);
>>
>>       /*
>>        * It shouldn't be a problem to reuse an opaque stateid value.
>> @@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>>       return NULL;
>>  }
>>
>> +/*
>> + * Create a unique stateid_t to represent each COPY. Hang the copy
>> + * stateids off the OPEN/LOCK/DELEG stateid from the client open
>> + * the source file.
>> + */
>> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
>> +                                            struct nfs4_stid *p_stid)
>> +{
>> +     struct nfs4_cp_state *cps;
>> +     int new_id;
>> +
>> +     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
>> +     if (!cps)
>> +             return NULL;
>> +     idr_preload(GFP_KERNEL);
>> +     spin_lock(&nn->s2s_cp_lock);
>> +     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
>> +     spin_unlock(&nn->s2s_cp_lock);
>> +     idr_preload_end();
>> +     if (new_id < 0)
>> +             goto out_free;
>> +     cps->cp_stateid.si_opaque.so_id = new_id;
>> +     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
>> +     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
>> +     cps->cp_p_stid = p_stid;
>> +     spin_lock(&p_stid->sc_lock);
>> +     list_add(&cps->cp_list, &p_stid->sc_cp_list);
>> +     spin_unlock(&p_stid->sc_lock);
>> +
>> +     return cps;
>> +out_free:
>> +     kfree(cps);
>> +     return NULL;
>> +}
>> +
>> +void nfs4_free_cp_state(struct nfs4_cp_state *cps)
>> +{
>> +     struct nfsd_net *nn;
>> +
>> +     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
>> +     spin_lock(&nn->s2s_cp_lock);
>> +     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
>> +     spin_unlock(&nn->s2s_cp_lock);
>> +
>> +     kfree(cps);
>> +}
>> +
>> +static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
>> +{
>> +     struct nfs4_cp_state *cps;
>> +
>> +     might_sleep();
>> +
>> +     spin_lock(&stid->sc_lock);
>> +     while (!list_empty(&stid->sc_cp_list)) {
>> +             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
>> +                                    cp_list);
>> +             list_del(&cps->cp_list);
>> +             nfs4_free_cp_state(cps);
>> +     }
>> +     spin_unlock(&stid->sc_lock);
>> +}
>> +
>>  static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
>>  {
>>       struct nfs4_stid *stid;
>> @@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
>>       }
>>       idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
>>       spin_unlock(&clp->cl_lock);
>> +
>> +     nfs4_free_cp_statelist(s);
>> +
>>       s->sc_free(s);
>>       if (fp)
>>               put_nfs4_file(fp);
>> @@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
>>       INIT_LIST_HEAD(&nn->close_lru);
>>       INIT_LIST_HEAD(&nn->del_recall_lru);
>>       spin_lock_init(&nn->client_lock);
>> +     spin_lock_init(&nn->s2s_cp_lock);
>> +     idr_init(&nn->s2s_cp_stateids);
>>
>>       spin_lock_init(&nn->blocked_locks_lock);
>>       INIT_LIST_HEAD(&nn->blocked_locks_lru);
>> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
>> index d107b44..63edf68 100644
>> --- a/fs/nfsd/nfsctl.c
>> +++ b/fs/nfsd/nfsctl.c
>> @@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
>>       nn->nfsd4_grace = 90;
>>       nn->clverifier_counter = prandom_u32();
>>       nn->clientid_counter = prandom_u32();
>> +     nn->s2s_cp_cl_id = nn->clientid_counter++;
>>
>>       atomic_set(&nn->ntf_refcnt, 0);
>>       init_waitqueue_head(&nn->ntf_wq);
>> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
>> index 9b7d7a0..49709d1 100644
>> --- a/fs/nfsd/state.h
>> +++ b/fs/nfsd/state.h
>> @@ -94,6 +94,7 @@ struct nfs4_stid {
>>  #define NFS4_REVOKED_DELEG_STID 16
>>  #define NFS4_CLOSED_DELEG_STID 32
>>  #define NFS4_LAYOUT_STID 64
>> +     struct list_head        sc_cp_list;
>>       unsigned char           sc_type;
>>       stateid_t               sc_stateid;
>>       spinlock_t              sc_lock;
>> @@ -103,6 +104,17 @@ struct nfs4_stid {
>>  };
>>
>>  /*
>> + * Keep a list of stateids issued by the COPY, associate it with the
>> + * parent OPEN/LOCK/DELEG stateid. Used for lookup by
>> + * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
>> + */
>> +struct nfs4_cp_state {
>> +     stateid_t               cp_stateid;
>> +     struct list_head        cp_list;        /* per parent nfs4_stid */
>> +     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
>> +};
>> +
>> +/*
>>   * Represents a delegation stateid. The nfs4_client holds references to these
>>   * and they are put when it is being destroyed or when the delegation is
>>   * returned by the client:
>> @@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
>>                    struct nfs4_stid **s, struct nfsd_net *nn);
>>  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
>>                                 void (*sc_free)(struct nfs4_stid *));
>> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
>> +void nfs4_free_cp_state(struct nfs4_cp_state *cps);
>>  void nfs4_unhash_stid(struct nfs4_stid *s);
>>  void nfs4_put_stid(struct nfs4_stid *s);
>>  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
>> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
>> index 3b6b655..5d493b9 100644
>> --- a/fs/nfsd/xdr4.h
>> +++ b/fs/nfsd/xdr4.h
>> @@ -535,6 +535,8 @@ struct nfsd4_copy {
>>       struct file             *fh_src;
>>       struct file             *fh_dst;
>>       struct net              *net;
>> +     struct nfs4_stid        *stid;
>> +     struct nfs4_cp_state    *cps;
>>
>>       struct list_head        copies;
>>       struct task_struct      *copy_task;
>> --
>> 1.8.3.1
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 07/10] NFSD create new stateid for async copy

[J. Bruce Fields]

On Thu, Mar 22, 2018 at 11:12:23AM -0400, Olga Kornievskaia wrote:
> On Thu, Mar 8, 2018 at 11:31 AM, J. Bruce Fields <bfields@fieldses.org> wrote:
> > On Tue, Feb 20, 2018 at 11:42:26AM -0500, Olga Kornievskaia wrote:
> >> Generate a new stateid to be used for reply to the asynchronous
> >> COPY (this would also be used later by COPY_NOTIFY as well).
> >> Associate the stateid with the parent OPEN/LOCK/DELEG stateid
> >> that can be freed during the free of the parent stateid. However,
> >> right now deciding to bind the lifetime to when the vfs copy
> >> is done. This way don't need to keep the nfsd_net structure for
> >> the callback. The drawback is that time copy state information
> >> is available for query by OFFLOAD_STATUS is slightly less.
> >
> > I don't understand yet the purpose of the parent stateid.
> >
> > I understand that it's part of the COPY arguments and that we need to
> > check it when we first process the COPY.  But what's it used for after
> > that?  Why do we need to keep it around?
> 
> Parent stateid tie is needed for when the "CLOSE" is received. CLOSE
> only has the parent state. We have a global list of async copies and
> then each copy has a pointer to parent to provide the match.

Why do you need it on CLOSE?  Is it that CLOSE is supposed to find any
related copies and shut them down or return an error?  (I thought we
decided that we could leave that to the client--but maybe I've lost
track of that discussion.)

--b.

> 
> >
> > --b.
> >
> >>
> >> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
> >> ---
> >>  fs/nfsd/netns.h     |  8 +++++++
> >>  fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
> >>  fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> >>  fs/nfsd/nfsctl.c    |  1 +
> >>  fs/nfsd/state.h     | 14 +++++++++++
> >>  fs/nfsd/xdr4.h      |  2 ++
> >>  6 files changed, 118 insertions(+), 10 deletions(-)
> >>
> >> diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
> >> index 36358d4..e83cf6e 100644
> >> --- a/fs/nfsd/netns.h
> >> +++ b/fs/nfsd/netns.h
> >> @@ -122,6 +122,14 @@ struct nfsd_net {
> >>
> >>       wait_queue_head_t ntf_wq;
> >>       atomic_t ntf_refcnt;
> >> +
> >> +     /*
> >> +      * clientid and stateid data for construction of net unique COPY
> >> +      * stateids.
> >> +      */
> >> +     u32             s2s_cp_cl_id;
> >> +     struct idr      s2s_cp_stateids;
> >> +     spinlock_t      s2s_cp_lock;
> >>  };
> >>
> >>  /* Simple check to find out if a given net was properly initialized */
> >> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> >> index aafd5a58..eb9f528 100644
> >> --- a/fs/nfsd/nfs4proc.c
> >> +++ b/fs/nfsd/nfs4proc.c
> >> @@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
> >>  static __be32
> >>  nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
> >>                 stateid_t *src_stateid, struct file **src,
> >> -               stateid_t *dst_stateid, struct file **dst)
> >> +               stateid_t *dst_stateid, struct file **dst,
> >> +               struct nfs4_stid **stid)
> >>  {
> >>       __be32 status;
> >>
> >> @@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
> >>
> >>       status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
> >>                                           dst_stateid, WR_STATE, dst, NULL,
> >> -                                         NULL);
> >> +                                         stid);
> >>       if (status) {
> >>               dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
> >>               goto out_put_src;
> >> @@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
> >>       __be32 status;
> >>
> >>       status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
> >> -                                &clone->cl_dst_stateid, &dst);
> >> +                                &clone->cl_dst_stateid, &dst, NULL);
> >>       if (status)
> >>               goto out;
> >>
> >> @@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
> >>
> >>  static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
> >>  {
> >> -     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> >> -             sizeof(copy->cp_dst_stateid));
> >>       copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> >>       copy->cp_consecutive = 1;
> >>       copy->cp_synchronous = sync;
> >> @@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
> >>       atomic_inc(&dst->cp_clp->cl_refcount);
> >>       dst->fh_dst = get_file(src->fh_dst);
> >>       dst->fh_src = get_file(src->fh_src);
> >> +     dst->stid = src->stid;
> >> +     dst->cps = src->cps;
> >>  }
> >>
> >>  static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
> >>  {
> >> +     list_del(&copy->cps->cp_list);
> >> +     nfs4_free_cp_state(copy->cps);
> >> +     nfs4_put_stid(copy->stid);
> >>       fput(copy->fh_dst);
> >>       fput(copy->fh_src);
> >>       if (remove) {
> >> @@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
> >>
> >>       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
> >>                                  &copy->fh_src, &copy->cp_dst_stateid,
> >> -                                &copy->fh_dst);
> >> +                                &copy->fh_dst, &copy->stid);
> >>       if (status)
> >>               goto out;
> >>
> >> @@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
> >>       memcpy(&copy->fh, &cstate->current_fh.fh_handle,
> >>               sizeof(struct knfsd_fh));
> >>       copy->net = SVC_NET(rqstp);
> >> -     /* for now disable asynchronous copy feature */
> >> -     copy->cp_synchronous = 1;
> >>       if (!copy->cp_synchronous) {
> >> +             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
> >> +
> >>               status = nfsd4_init_copy_res(copy, 0);
> >>               async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> >>               if (!async_copy) {
> >>                       status = nfserrno(-ENOMEM);
> >>                       goto out;
> >>               }
> >> +             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
> >> +             if (!copy->cps) {
> >> +                     status = nfserrno(-ENOMEM);
> >> +                     kfree(async_copy);
> >> +                     goto out;
> >> +             }
> >> +             /* take a reference on the parent stateid so it's not
> >> +              * not freed by the copy compound
> >> +              */
> >> +             refcount_inc(&copy->stid->sc_count);
> >> +             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
> >> +                     sizeof(copy->cps->cp_stateid));
> >>               dup_copy_fields(copy, async_copy);
> >> -             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> >> -                     sizeof(copy->cp_dst_stateid));
> >>               async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
> >>                               async_copy, "%s", "copy thread");
> >>               if (IS_ERR(async_copy->copy_task)) {
> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> >> index e05832b..bd76bd1 100644
> >> --- a/fs/nfsd/nfs4state.c
> >> +++ b/fs/nfsd/nfs4state.c
> >> @@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
> >>       /* Will be incremented before return to client: */
> >>       refcount_set(&stid->sc_count, 1);
> >>       spin_lock_init(&stid->sc_lock);
> >> +     INIT_LIST_HEAD(&stid->sc_cp_list);
> >>
> >>       /*
> >>        * It shouldn't be a problem to reuse an opaque stateid value.
> >> @@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
> >>       return NULL;
> >>  }
> >>
> >> +/*
> >> + * Create a unique stateid_t to represent each COPY. Hang the copy
> >> + * stateids off the OPEN/LOCK/DELEG stateid from the client open
> >> + * the source file.
> >> + */
> >> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
> >> +                                            struct nfs4_stid *p_stid)
> >> +{
> >> +     struct nfs4_cp_state *cps;
> >> +     int new_id;
> >> +
> >> +     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
> >> +     if (!cps)
> >> +             return NULL;
> >> +     idr_preload(GFP_KERNEL);
> >> +     spin_lock(&nn->s2s_cp_lock);
> >> +     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
> >> +     spin_unlock(&nn->s2s_cp_lock);
> >> +     idr_preload_end();
> >> +     if (new_id < 0)
> >> +             goto out_free;
> >> +     cps->cp_stateid.si_opaque.so_id = new_id;
> >> +     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
> >> +     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
> >> +     cps->cp_p_stid = p_stid;
> >> +     spin_lock(&p_stid->sc_lock);
> >> +     list_add(&cps->cp_list, &p_stid->sc_cp_list);
> >> +     spin_unlock(&p_stid->sc_lock);
> >> +
> >> +     return cps;
> >> +out_free:
> >> +     kfree(cps);
> >> +     return NULL;
> >> +}
> >> +
> >> +void nfs4_free_cp_state(struct nfs4_cp_state *cps)
> >> +{
> >> +     struct nfsd_net *nn;
> >> +
> >> +     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
> >> +     spin_lock(&nn->s2s_cp_lock);
> >> +     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
> >> +     spin_unlock(&nn->s2s_cp_lock);
> >> +
> >> +     kfree(cps);
> >> +}
> >> +
> >> +static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
> >> +{
> >> +     struct nfs4_cp_state *cps;
> >> +
> >> +     might_sleep();
> >> +
> >> +     spin_lock(&stid->sc_lock);
> >> +     while (!list_empty(&stid->sc_cp_list)) {
> >> +             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
> >> +                                    cp_list);
> >> +             list_del(&cps->cp_list);
> >> +             nfs4_free_cp_state(cps);
> >> +     }
> >> +     spin_unlock(&stid->sc_lock);
> >> +}
> >> +
> >>  static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
> >>  {
> >>       struct nfs4_stid *stid;
> >> @@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
> >>       }
> >>       idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
> >>       spin_unlock(&clp->cl_lock);
> >> +
> >> +     nfs4_free_cp_statelist(s);
> >> +
> >>       s->sc_free(s);
> >>       if (fp)
> >>               put_nfs4_file(fp);
> >> @@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
> >>       INIT_LIST_HEAD(&nn->close_lru);
> >>       INIT_LIST_HEAD(&nn->del_recall_lru);
> >>       spin_lock_init(&nn->client_lock);
> >> +     spin_lock_init(&nn->s2s_cp_lock);
> >> +     idr_init(&nn->s2s_cp_stateids);
> >>
> >>       spin_lock_init(&nn->blocked_locks_lock);
> >>       INIT_LIST_HEAD(&nn->blocked_locks_lru);
> >> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
> >> index d107b44..63edf68 100644
> >> --- a/fs/nfsd/nfsctl.c
> >> +++ b/fs/nfsd/nfsctl.c
> >> @@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
> >>       nn->nfsd4_grace = 90;
> >>       nn->clverifier_counter = prandom_u32();
> >>       nn->clientid_counter = prandom_u32();
> >> +     nn->s2s_cp_cl_id = nn->clientid_counter++;
> >>
> >>       atomic_set(&nn->ntf_refcnt, 0);
> >>       init_waitqueue_head(&nn->ntf_wq);
> >> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> >> index 9b7d7a0..49709d1 100644
> >> --- a/fs/nfsd/state.h
> >> +++ b/fs/nfsd/state.h
> >> @@ -94,6 +94,7 @@ struct nfs4_stid {
> >>  #define NFS4_REVOKED_DELEG_STID 16
> >>  #define NFS4_CLOSED_DELEG_STID 32
> >>  #define NFS4_LAYOUT_STID 64
> >> +     struct list_head        sc_cp_list;
> >>       unsigned char           sc_type;
> >>       stateid_t               sc_stateid;
> >>       spinlock_t              sc_lock;
> >> @@ -103,6 +104,17 @@ struct nfs4_stid {
> >>  };
> >>
> >>  /*
> >> + * Keep a list of stateids issued by the COPY, associate it with the
> >> + * parent OPEN/LOCK/DELEG stateid. Used for lookup by
> >> + * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
> >> + */
> >> +struct nfs4_cp_state {
> >> +     stateid_t               cp_stateid;
> >> +     struct list_head        cp_list;        /* per parent nfs4_stid */
> >> +     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
> >> +};
> >> +
> >> +/*
> >>   * Represents a delegation stateid. The nfs4_client holds references to these
> >>   * and they are put when it is being destroyed or when the delegation is
> >>   * returned by the client:
> >> @@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
> >>                    struct nfs4_stid **s, struct nfsd_net *nn);
> >>  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
> >>                                 void (*sc_free)(struct nfs4_stid *));
> >> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
> >> +void nfs4_free_cp_state(struct nfs4_cp_state *cps);
> >>  void nfs4_unhash_stid(struct nfs4_stid *s);
> >>  void nfs4_put_stid(struct nfs4_stid *s);
> >>  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> >> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> >> index 3b6b655..5d493b9 100644
> >> --- a/fs/nfsd/xdr4.h
> >> +++ b/fs/nfsd/xdr4.h
> >> @@ -535,6 +535,8 @@ struct nfsd4_copy {
> >>       struct file             *fh_src;
> >>       struct file             *fh_dst;
> >>       struct net              *net;
> >> +     struct nfs4_stid        *stid;
> >> +     struct nfs4_cp_state    *cps;
> >>
> >>       struct list_head        copies;
> >>       struct task_struct      *copy_task;
> >> --
> >> 1.8.3.1
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 07/10] NFSD create new stateid for async copy

[Olga Kornievskaia]

On Thu, Mar 22, 2018 at 11:17 AM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Thu, Mar 22, 2018 at 11:12:23AM -0400, Olga Kornievskaia wrote:
>> On Thu, Mar 8, 2018 at 11:31 AM, J. Bruce Fields <bfields@fieldses.org> wrote:
>> > On Tue, Feb 20, 2018 at 11:42:26AM -0500, Olga Kornievskaia wrote:
>> >> Generate a new stateid to be used for reply to the asynchronous
>> >> COPY (this would also be used later by COPY_NOTIFY as well).
>> >> Associate the stateid with the parent OPEN/LOCK/DELEG stateid
>> >> that can be freed during the free of the parent stateid. However,
>> >> right now deciding to bind the lifetime to when the vfs copy
>> >> is done. This way don't need to keep the nfsd_net structure for
>> >> the callback. The drawback is that time copy state information
>> >> is available for query by OFFLOAD_STATUS is slightly less.
>> >
>> > I don't understand yet the purpose of the parent stateid.
>> >
>> > I understand that it's part of the COPY arguments and that we need to
>> > check it when we first process the COPY.  But what's it used for after
>> > that?  Why do we need to keep it around?
>>
>> Parent stateid tie is needed for when the "CLOSE" is received. CLOSE
>> only has the parent state. We have a global list of async copies and
>> then each copy has a pointer to parent to provide the match.
>
> Why do you need it on CLOSE?  Is it that CLOSE is supposed to find any
> related copies and shut them down or return an error?  (I thought we
> decided that we could leave that to the client--but maybe I've lost
> track of that discussion.)

Yes I believe we decided that CLOSE/UNLOCK is suppose to shutdown the copy.

>
> --b.
>
>>
>> >
>> > --b.
>> >
>> >>
>> >> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
>> >> ---
>> >>  fs/nfsd/netns.h     |  8 +++++++
>> >>  fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
>> >>  fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>> >>  fs/nfsd/nfsctl.c    |  1 +
>> >>  fs/nfsd/state.h     | 14 +++++++++++
>> >>  fs/nfsd/xdr4.h      |  2 ++
>> >>  6 files changed, 118 insertions(+), 10 deletions(-)
>> >>
>> >> diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
>> >> index 36358d4..e83cf6e 100644
>> >> --- a/fs/nfsd/netns.h
>> >> +++ b/fs/nfsd/netns.h
>> >> @@ -122,6 +122,14 @@ struct nfsd_net {
>> >>
>> >>       wait_queue_head_t ntf_wq;
>> >>       atomic_t ntf_refcnt;
>> >> +
>> >> +     /*
>> >> +      * clientid and stateid data for construction of net unique COPY
>> >> +      * stateids.
>> >> +      */
>> >> +     u32             s2s_cp_cl_id;
>> >> +     struct idr      s2s_cp_stateids;
>> >> +     spinlock_t      s2s_cp_lock;
>> >>  };
>> >>
>> >>  /* Simple check to find out if a given net was properly initialized */
>> >> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> >> index aafd5a58..eb9f528 100644
>> >> --- a/fs/nfsd/nfs4proc.c
>> >> +++ b/fs/nfsd/nfs4proc.c
>> >> @@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>> >>  static __be32
>> >>  nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>> >>                 stateid_t *src_stateid, struct file **src,
>> >> -               stateid_t *dst_stateid, struct file **dst)
>> >> +               stateid_t *dst_stateid, struct file **dst,
>> >> +               struct nfs4_stid **stid)
>> >>  {
>> >>       __be32 status;
>> >>
>> >> @@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>> >>
>> >>       status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
>> >>                                           dst_stateid, WR_STATE, dst, NULL,
>> >> -                                         NULL);
>> >> +                                         stid);
>> >>       if (status) {
>> >>               dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
>> >>               goto out_put_src;
>> >> @@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>> >>       __be32 status;
>> >>
>> >>       status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
>> >> -                                &clone->cl_dst_stateid, &dst);
>> >> +                                &clone->cl_dst_stateid, &dst, NULL);
>> >>       if (status)
>> >>               goto out;
>> >>
>> >> @@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
>> >>
>> >>  static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
>> >>  {
>> >> -     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> >> -             sizeof(copy->cp_dst_stateid));
>> >>       copy->cp_res.wr_stable_how = NFS_UNSTABLE;
>> >>       copy->cp_consecutive = 1;
>> >>       copy->cp_synchronous = sync;
>> >> @@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
>> >>       atomic_inc(&dst->cp_clp->cl_refcount);
>> >>       dst->fh_dst = get_file(src->fh_dst);
>> >>       dst->fh_src = get_file(src->fh_src);
>> >> +     dst->stid = src->stid;
>> >> +     dst->cps = src->cps;
>> >>  }
>> >>
>> >>  static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
>> >>  {
>> >> +     list_del(&copy->cps->cp_list);
>> >> +     nfs4_free_cp_state(copy->cps);
>> >> +     nfs4_put_stid(copy->stid);
>> >>       fput(copy->fh_dst);
>> >>       fput(copy->fh_src);
>> >>       if (remove) {
>> >> @@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
>> >>
>> >>       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
>> >>                                  &copy->fh_src, &copy->cp_dst_stateid,
>> >> -                                &copy->fh_dst);
>> >> +                                &copy->fh_dst, &copy->stid);
>> >>       if (status)
>> >>               goto out;
>> >>
>> >> @@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
>> >>       memcpy(&copy->fh, &cstate->current_fh.fh_handle,
>> >>               sizeof(struct knfsd_fh));
>> >>       copy->net = SVC_NET(rqstp);
>> >> -     /* for now disable asynchronous copy feature */
>> >> -     copy->cp_synchronous = 1;
>> >>       if (!copy->cp_synchronous) {
>> >> +             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
>> >> +
>> >>               status = nfsd4_init_copy_res(copy, 0);
>> >>               async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
>> >>               if (!async_copy) {
>> >>                       status = nfserrno(-ENOMEM);
>> >>                       goto out;
>> >>               }
>> >> +             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
>> >> +             if (!copy->cps) {
>> >> +                     status = nfserrno(-ENOMEM);
>> >> +                     kfree(async_copy);
>> >> +                     goto out;
>> >> +             }
>> >> +             /* take a reference on the parent stateid so it's not
>> >> +              * not freed by the copy compound
>> >> +              */
>> >> +             refcount_inc(&copy->stid->sc_count);
>> >> +             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
>> >> +                     sizeof(copy->cps->cp_stateid));
>> >>               dup_copy_fields(copy, async_copy);
>> >> -             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
>> >> -                     sizeof(copy->cp_dst_stateid));
>> >>               async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
>> >>                               async_copy, "%s", "copy thread");
>> >>               if (IS_ERR(async_copy->copy_task)) {
>> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> >> index e05832b..bd76bd1 100644
>> >> --- a/fs/nfsd/nfs4state.c
>> >> +++ b/fs/nfsd/nfs4state.c
>> >> @@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>> >>       /* Will be incremented before return to client: */
>> >>       refcount_set(&stid->sc_count, 1);
>> >>       spin_lock_init(&stid->sc_lock);
>> >> +     INIT_LIST_HEAD(&stid->sc_cp_list);
>> >>
>> >>       /*
>> >>        * It shouldn't be a problem to reuse an opaque stateid value.
>> >> @@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
>> >>       return NULL;
>> >>  }
>> >>
>> >> +/*
>> >> + * Create a unique stateid_t to represent each COPY. Hang the copy
>> >> + * stateids off the OPEN/LOCK/DELEG stateid from the client open
>> >> + * the source file.
>> >> + */
>> >> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
>> >> +                                            struct nfs4_stid *p_stid)
>> >> +{
>> >> +     struct nfs4_cp_state *cps;
>> >> +     int new_id;
>> >> +
>> >> +     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
>> >> +     if (!cps)
>> >> +             return NULL;
>> >> +     idr_preload(GFP_KERNEL);
>> >> +     spin_lock(&nn->s2s_cp_lock);
>> >> +     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
>> >> +     spin_unlock(&nn->s2s_cp_lock);
>> >> +     idr_preload_end();
>> >> +     if (new_id < 0)
>> >> +             goto out_free;
>> >> +     cps->cp_stateid.si_opaque.so_id = new_id;
>> >> +     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
>> >> +     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
>> >> +     cps->cp_p_stid = p_stid;
>> >> +     spin_lock(&p_stid->sc_lock);
>> >> +     list_add(&cps->cp_list, &p_stid->sc_cp_list);
>> >> +     spin_unlock(&p_stid->sc_lock);
>> >> +
>> >> +     return cps;
>> >> +out_free:
>> >> +     kfree(cps);
>> >> +     return NULL;
>> >> +}
>> >> +
>> >> +void nfs4_free_cp_state(struct nfs4_cp_state *cps)
>> >> +{
>> >> +     struct nfsd_net *nn;
>> >> +
>> >> +     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
>> >> +     spin_lock(&nn->s2s_cp_lock);
>> >> +     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
>> >> +     spin_unlock(&nn->s2s_cp_lock);
>> >> +
>> >> +     kfree(cps);
>> >> +}
>> >> +
>> >> +static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
>> >> +{
>> >> +     struct nfs4_cp_state *cps;
>> >> +
>> >> +     might_sleep();
>> >> +
>> >> +     spin_lock(&stid->sc_lock);
>> >> +     while (!list_empty(&stid->sc_cp_list)) {
>> >> +             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
>> >> +                                    cp_list);
>> >> +             list_del(&cps->cp_list);
>> >> +             nfs4_free_cp_state(cps);
>> >> +     }
>> >> +     spin_unlock(&stid->sc_lock);
>> >> +}
>> >> +
>> >>  static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
>> >>  {
>> >>       struct nfs4_stid *stid;
>> >> @@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
>> >>       }
>> >>       idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
>> >>       spin_unlock(&clp->cl_lock);
>> >> +
>> >> +     nfs4_free_cp_statelist(s);
>> >> +
>> >>       s->sc_free(s);
>> >>       if (fp)
>> >>               put_nfs4_file(fp);
>> >> @@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
>> >>       INIT_LIST_HEAD(&nn->close_lru);
>> >>       INIT_LIST_HEAD(&nn->del_recall_lru);
>> >>       spin_lock_init(&nn->client_lock);
>> >> +     spin_lock_init(&nn->s2s_cp_lock);
>> >> +     idr_init(&nn->s2s_cp_stateids);
>> >>
>> >>       spin_lock_init(&nn->blocked_locks_lock);
>> >>       INIT_LIST_HEAD(&nn->blocked_locks_lru);
>> >> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
>> >> index d107b44..63edf68 100644
>> >> --- a/fs/nfsd/nfsctl.c
>> >> +++ b/fs/nfsd/nfsctl.c
>> >> @@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
>> >>       nn->nfsd4_grace = 90;
>> >>       nn->clverifier_counter = prandom_u32();
>> >>       nn->clientid_counter = prandom_u32();
>> >> +     nn->s2s_cp_cl_id = nn->clientid_counter++;
>> >>
>> >>       atomic_set(&nn->ntf_refcnt, 0);
>> >>       init_waitqueue_head(&nn->ntf_wq);
>> >> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
>> >> index 9b7d7a0..49709d1 100644
>> >> --- a/fs/nfsd/state.h
>> >> +++ b/fs/nfsd/state.h
>> >> @@ -94,6 +94,7 @@ struct nfs4_stid {
>> >>  #define NFS4_REVOKED_DELEG_STID 16
>> >>  #define NFS4_CLOSED_DELEG_STID 32
>> >>  #define NFS4_LAYOUT_STID 64
>> >> +     struct list_head        sc_cp_list;
>> >>       unsigned char           sc_type;
>> >>       stateid_t               sc_stateid;
>> >>       spinlock_t              sc_lock;
>> >> @@ -103,6 +104,17 @@ struct nfs4_stid {
>> >>  };
>> >>
>> >>  /*
>> >> + * Keep a list of stateids issued by the COPY, associate it with the
>> >> + * parent OPEN/LOCK/DELEG stateid. Used for lookup by
>> >> + * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
>> >> + */
>> >> +struct nfs4_cp_state {
>> >> +     stateid_t               cp_stateid;
>> >> +     struct list_head        cp_list;        /* per parent nfs4_stid */
>> >> +     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
>> >> +};
>> >> +
>> >> +/*
>> >>   * Represents a delegation stateid. The nfs4_client holds references to these
>> >>   * and they are put when it is being destroyed or when the delegation is
>> >>   * returned by the client:
>> >> @@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
>> >>                    struct nfs4_stid **s, struct nfsd_net *nn);
>> >>  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
>> >>                                 void (*sc_free)(struct nfs4_stid *));
>> >> +struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
>> >> +void nfs4_free_cp_state(struct nfs4_cp_state *cps);
>> >>  void nfs4_unhash_stid(struct nfs4_stid *s);
>> >>  void nfs4_put_stid(struct nfs4_stid *s);
>> >>  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
>> >> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
>> >> index 3b6b655..5d493b9 100644
>> >> --- a/fs/nfsd/xdr4.h
>> >> +++ b/fs/nfsd/xdr4.h
>> >> @@ -535,6 +535,8 @@ struct nfsd4_copy {
>> >>       struct file             *fh_src;
>> >>       struct file             *fh_dst;
>> >>       struct net              *net;
>> >> +     struct nfs4_stid        *stid;
>> >> +     struct nfs4_cp_state    *cps;
>> >>
>> >>       struct list_head        copies;
>> >>       struct task_struct      *copy_task;
>> >> --
>> >> 1.8.3.1
>> > --
>> > To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>> > the body of a message to majordomo@vger.kernel.org
>> > More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[Olga Kornievskaia]

On Thu, Mar 8, 2018 at 12:05 PM, J. Bruce Fields <bfields@fieldses.org> wro=
te:
> On Tue, Feb 20, 2018 at 11:42:29AM -0500, Olga Kornievskaia wrote:
>> If client is shutting down and there are still async copies going
>> on, then stop queued async copies.
>>
>> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
>> ---
>>  fs/nfsd/nfs4proc.c  | 13 +++++++++++++
>>  fs/nfsd/nfs4state.c |  1 +
>>  fs/nfsd/state.h     |  1 +
>>  3 files changed, 15 insertions(+)
>>
>> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> index 57d0daa..54039e4 100644
>> --- a/fs/nfsd/nfs4proc.c
>> +++ b/fs/nfsd/nfs4proc.c
>> @@ -1096,6 +1096,19 @@ static void nfs4_put_copy(struct nfsd4_copy *copy=
)
>>       kfree(copy);
>>  }
>>
>> +void nfsd4_shutdown_copy(struct nfs4_client *clp)
>> +{
>> +     struct nfsd4_copy *copy;
>> +
>> +     spin_lock(&clp->async_lock);
>> +     list_for_each_entry(copy, &clp->async_copies, copies) {
>> +             set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
>> +             kthread_stop(copy->copy_task);
>> +             nfs4_put_copy(copy);
>
> Those three lines could go in a common helper to be called here and from
> nfsd4_offload_cancel().

Hi Bruce,

I have a question. I=E2=80=99m testing how the code works in
nfsd4_shutdown_copy. I have disabled canceling the copy (just
returning ok) and so then the client can stop its copy and do the
close and then I can issue an unmount. What I see is that the server
returns =E2=80=9Cerr_delay=E2=80=9D until the copy finishes. That=E2=80=99s=
 because in
nfsd4_destroy_clientid() it calls mark_client_expired_locked() which
checks the refcount on the client structure and since copy holds a
reference server returns err_delay. Once the copy finished and
decrements the reference, and nfsd4_destroy_clientid() gets past that
then calling nfsd4_shutdown_copy() doesn=E2=80=99t find any copies.

Should the logic of nfsd4_destroy_clientid() be changed to stop copies
then instead of during destruction of the client structure?

>
> --b.
>
>> +     }
>> +     spin_unlock(&clp->async_lock);
>> +}
>> +
>>  static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
>>  {
>>       struct nfsd4_copy *copy =3D container_of(cb, struct nfsd4_copy, cp=
_cb);
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index bd76bd1..f9f2383 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -1965,6 +1965,7 @@ static __be32 mark_client_expired_locked(struct nf=
s4_client *clp)
>>               release_openowner(oo);
>>       }
>>       nfsd4_return_all_client_layouts(clp);
>> +     nfsd4_shutdown_copy(clp);
>>       nfsd4_shutdown_callback(clp);
>>       if (clp->cl_cb_conn.cb_xprt)
>>               svc_xprt_put(clp->cl_cb_conn.cb_xprt);
>> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
>> index 49709d1..131aefa 100644
>> --- a/fs/nfsd/state.h
>> +++ b/fs/nfsd/state.h
>> @@ -646,6 +646,7 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb,=
 struct nfs4_client *clp,
>>  extern int nfsd4_create_callback_queue(void);
>>  extern void nfsd4_destroy_callback_queue(void);
>>  extern void nfsd4_shutdown_callback(struct nfs4_client *);
>> +extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
>>  extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
>>  extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *n=
ame,
>>                                                       struct nfsd_net *n=
n);
>> --
>> 1.8.3.1
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[J. Bruce Fields]

On Tue, Apr 10, 2018 at 04:09:09PM -0400, Olga Kornievskaia wrote:
> I have a question. I’m testing how the code works in
> nfsd4_shutdown_copy. I have disabled canceling the copy (just
> returning ok) and so then the client can stop its copy and do the
> close and then I can issue an unmount. What I see is that the server
> returns “err_delay” until the copy finishes. That’s because in
> nfsd4_destroy_clientid() it calls mark_client_expired_locked() which
> checks the refcount on the client structure and since copy holds a
> reference server returns err_delay. Once the copy finished and
> decrements the reference, and nfsd4_destroy_clientid() gets past that
> then calling nfsd4_shutdown_copy() doesn’t find any copies.
> 
> Should the logic of nfsd4_destroy_clientid() be changed to stop copies
> then instead of during destruction of the client structure?

Oh, good question, I don't know.  Thinking about it....

DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
only meant to be called after the client has already cleaned up
everything else.  So:

	https://tools.ietf.org/html/rfc5661#section-18.50.3

	If there are sessions (both idle and non-idle), opens, locks,
	delegations, layouts, and/or wants (Section 18.49) associated
	with the unexpired lease of the client ID, the server MUST
	return NFS4ERR_CLIENTID_BUSY.

My feeling is that "ongoing copies" also belongs on that list.

So the server behavior you're seeing sounds correct to me--the client
should cancel any ongoing copies before calling DESTROY_CLIENTID.

--b.

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[Olga Kornievskaia]

On Tue, Apr 10, 2018 at 4:21 PM, J. Bruce Fields <bfields@redhat.com> wrote=
:
> On Tue, Apr 10, 2018 at 04:09:09PM -0400, Olga Kornievskaia wrote:
>> I have a question. I=E2=80=99m testing how the code works in
>> nfsd4_shutdown_copy. I have disabled canceling the copy (just
>> returning ok) and so then the client can stop its copy and do the
>> close and then I can issue an unmount. What I see is that the server
>> returns =E2=80=9Cerr_delay=E2=80=9D until the copy finishes. That=E2=80=
=99s because in
>> nfsd4_destroy_clientid() it calls mark_client_expired_locked() which
>> checks the refcount on the client structure and since copy holds a
>> reference server returns err_delay. Once the copy finished and
>> decrements the reference, and nfsd4_destroy_clientid() gets past that
>> then calling nfsd4_shutdown_copy() doesn=E2=80=99t find any copies.
>>
>> Should the logic of nfsd4_destroy_clientid() be changed to stop copies
>> then instead of during destruction of the client structure?
>
> Oh, good question, I don't know.  Thinking about it....
>
> DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
> only meant to be called after the client has already cleaned up
> everything else.  So:
>
>         https://tools.ietf.org/html/rfc5661#section-18.50.3
>
>         If there are sessions (both idle and non-idle), opens, locks,
>         delegations, layouts, and/or wants (Section 18.49) associated
>         with the unexpired lease of the client ID, the server MUST
>         return NFS4ERR_CLIENTID_BUSY.
>
> My feeling is that "ongoing copies" also belongs on that list.
>
> So the server behavior you're seeing sounds correct to me--the client
> should cancel any ongoing copies before calling DESTROY_CLIENTID.

If the behavior of returning ERR_DELAY until the copy is done is
correct one, then I don't think I need this patch at all. Since copy
takes a reference on the nfs4_client structure, then in
__destroy_client() where nfsd4_shutdown_copy() is called the list will
always be empty.

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[J. Bruce Fields]

On Tue, Apr 10, 2018 at 05:07:02PM -0400, Olga Kornievskaia wrote:
> On Tue, Apr 10, 2018 at 4:21 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> > On Tue, Apr 10, 2018 at 04:09:09PM -0400, Olga Kornievskaia wrote:
> >> I have a question. I’m testing how the code works in
> >> nfsd4_shutdown_copy. I have disabled canceling the copy (just
> >> returning ok) and so then the client can stop its copy and do the
> >> close and then I can issue an unmount. What I see is that the server
> >> returns “err_delay” until the copy finishes. That’s because in
> >> nfsd4_destroy_clientid() it calls mark_client_expired_locked() which
> >> checks the refcount on the client structure and since copy holds a
> >> reference server returns err_delay. Once the copy finished and
> >> decrements the reference, and nfsd4_destroy_clientid() gets past that
> >> then calling nfsd4_shutdown_copy() doesn’t find any copies.
> >>
> >> Should the logic of nfsd4_destroy_clientid() be changed to stop copies
> >> then instead of during destruction of the client structure?
> >
> > Oh, good question, I don't know.  Thinking about it....
> >
> > DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
> > only meant to be called after the client has already cleaned up
> > everything else.  So:
> >
> >         https://tools.ietf.org/html/rfc5661#section-18.50.3
> >
> >         If there are sessions (both idle and non-idle), opens, locks,
> >         delegations, layouts, and/or wants (Section 18.49) associated
> >         with the unexpired lease of the client ID, the server MUST
> >         return NFS4ERR_CLIENTID_BUSY.
> >
> > My feeling is that "ongoing copies" also belongs on that list.
> >
> > So the server behavior you're seeing sounds correct to me--the client
> > should cancel any ongoing copies before calling DESTROY_CLIENTID.
> 
> If the behavior of returning ERR_DELAY until the copy is done is
> correct one, then I don't think I need this patch at all. Since copy
> takes a reference on the nfs4_client structure, then in
> __destroy_client() where nfsd4_shutdown_copy() is called the list will
> always be empty.

Actually I guess it should be returning CLIENTID_BUSY.  Maybe that's a
preexisting bug.

We do still need to be able to handle the case where the lease expires.
I guess the copy needs to be forcibly canceled in that case.

--b.

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[J. Bruce Fields]

On Tue, Apr 10, 2018 at 05:13:07PM -0400, J. Bruce Fields wrote:
> On Tue, Apr 10, 2018 at 05:07:02PM -0400, Olga Kornievskaia wrote:
> > On Tue, Apr 10, 2018 at 4:21 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> > > DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
> > > only meant to be called after the client has already cleaned up
> > > everything else.  So:
> > >
> > >         https://tools.ietf.org/html/rfc5661#section-18.50.3
> > >
> > >         If there are sessions (both idle and non-idle), opens, locks,
> > >         delegations, layouts, and/or wants (Section 18.49) associated
> > >         with the unexpired lease of the client ID, the server MUST
> > >         return NFS4ERR_CLIENTID_BUSY.
> > >
> > > My feeling is that "ongoing copies" also belongs on that list.

And come to think of it we should actually be adding that check to
client_has_state()--it should return clientid_busy if there are any
copies in progress.

> > > So the server behavior you're seeing sounds correct to me--the client
> > > should cancel any ongoing copies before calling DESTROY_CLIENTID.
> > 
> > If the behavior of returning ERR_DELAY until the copy is done is
> > correct one, then I don't think I need this patch at all. Since copy
> > takes a reference on the nfs4_client structure, then in
> > __destroy_client() where nfsd4_shutdown_copy() is called the list will
> > always be empty.
> 
> Actually I guess it should be returning CLIENTID_BUSY.  Maybe that's a
> preexisting bug.

So the copy should be caught by the earlier client_has_state() check
before it gets to the later mark_client_expired_locked().

And after reminding myself how this works....  We only hold references
on clients temporarily such as while we're actually processing an RPC
from a client.

An elevated cl_refcount prevents the server from removing the client
even after the lease expires, or after the client reboots and attempts
to clear its old state with a new EXCHANGE_ID/CREATE_SESSION.  I don't
think that's what we want.  Clients still need to renew their lease in
the usual way, a long-running async copy doesn't keep the lease renewed
automatically.

So, the asynchronous copy shouldn't hold a reference on the client.

The copy thread can still safely use the client while it's running,
because it knows that anyone destroying the client will first cancel the
copy and wait for the thread to die.

--b.

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[Olga Kornievskaia]

On Wed, Apr 11, 2018 at 1:33 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> On Tue, Apr 10, 2018 at 05:13:07PM -0400, J. Bruce Fields wrote:
>> On Tue, Apr 10, 2018 at 05:07:02PM -0400, Olga Kornievskaia wrote:
>> > On Tue, Apr 10, 2018 at 4:21 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>> > > DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
>> > > only meant to be called after the client has already cleaned up
>> > > everything else.  So:
>> > >
>> > >         https://tools.ietf.org/html/rfc5661#section-18.50.3
>> > >
>> > >         If there are sessions (both idle and non-idle), opens, locks,
>> > >         delegations, layouts, and/or wants (Section 18.49) associated
>> > >         with the unexpired lease of the client ID, the server MUST
>> > >         return NFS4ERR_CLIENTID_BUSY.
>> > >
>> > > My feeling is that "ongoing copies" also belongs on that list.
>
> And come to think of it we should actually be adding that check to
> client_has_state()--it should return clientid_busy if there are any
> copies in progress.
>
>> > > So the server behavior you're seeing sounds correct to me--the client
>> > > should cancel any ongoing copies before calling DESTROY_CLIENTID.
>> >
>> > If the behavior of returning ERR_DELAY until the copy is done is
>> > correct one, then I don't think I need this patch at all. Since copy
>> > takes a reference on the nfs4_client structure, then in
>> > __destroy_client() where nfsd4_shutdown_copy() is called the list will
>> > always be empty.
>>
>> Actually I guess it should be returning CLIENTID_BUSY.  Maybe that's a
>> preexisting bug.
>
> So the copy should be caught by the earlier client_has_state() check
> before it gets to the later mark_client_expired_locked().
>
> And after reminding myself how this works....  We only hold references
> on clients temporarily such as while we're actually processing an RPC
> from a client.
>
> An elevated cl_refcount prevents the server from removing the client
> even after the lease expires, or after the client reboots and attempts
> to clear its old state with a new EXCHANGE_ID/CREATE_SESSION.  I don't
> think that's what we want.  Clients still need to renew their lease in
> the usual way, a long-running async copy doesn't keep the lease renewed
> automatically.
>
> So, the asynchronous copy shouldn't hold a reference on the client.
>
> The copy thread can still safely use the client while it's running,
> because it knows that anyone destroying the client will first cancel the
> copy and wait for the thread to die.

Ok no reference on the client. I will add a check to
client_has_state() to check for on-going copies and then server would
return CLIENTID_BUSY. What was already in the patch was that during
client shutdown it would stop copies. I have tested for when the
server is expiring client's lease, it also shuts down any on-going
copies. I think I'm ready for the next version submission...

Re: [PATCH v7 10/10] NFSD stop queued async copies on client shutdown

[J. Bruce Fields]

On Thu, Apr 12, 2018 at 04:05:39PM -0400, Olga Kornievskaia wrote:
> On Wed, Apr 11, 2018 at 1:33 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> > On Tue, Apr 10, 2018 at 05:13:07PM -0400, J. Bruce Fields wrote:
> >> On Tue, Apr 10, 2018 at 05:07:02PM -0400, Olga Kornievskaia wrote:
> >> > On Tue, Apr 10, 2018 at 4:21 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> >> > > DESTROY_CLIENTID doesn't throw away all the client's state for it, it's
> >> > > only meant to be called after the client has already cleaned up
> >> > > everything else.  So:
> >> > >
> >> > >         https://tools.ietf.org/html/rfc5661#section-18.50.3
> >> > >
> >> > >         If there are sessions (both idle and non-idle), opens, locks,
> >> > >         delegations, layouts, and/or wants (Section 18.49) associated
> >> > >         with the unexpired lease of the client ID, the server MUST
> >> > >         return NFS4ERR_CLIENTID_BUSY.
> >> > >
> >> > > My feeling is that "ongoing copies" also belongs on that list.
> >
> > And come to think of it we should actually be adding that check to
> > client_has_state()--it should return clientid_busy if there are any
> > copies in progress.
> >
> >> > > So the server behavior you're seeing sounds correct to me--the client
> >> > > should cancel any ongoing copies before calling DESTROY_CLIENTID.
> >> >
> >> > If the behavior of returning ERR_DELAY until the copy is done is
> >> > correct one, then I don't think I need this patch at all. Since copy
> >> > takes a reference on the nfs4_client structure, then in
> >> > __destroy_client() where nfsd4_shutdown_copy() is called the list will
> >> > always be empty.
> >>
> >> Actually I guess it should be returning CLIENTID_BUSY.  Maybe that's a
> >> preexisting bug.
> >
> > So the copy should be caught by the earlier client_has_state() check
> > before it gets to the later mark_client_expired_locked().
> >
> > And after reminding myself how this works....  We only hold references
> > on clients temporarily such as while we're actually processing an RPC
> > from a client.
> >
> > An elevated cl_refcount prevents the server from removing the client
> > even after the lease expires, or after the client reboots and attempts
> > to clear its old state with a new EXCHANGE_ID/CREATE_SESSION.  I don't
> > think that's what we want.  Clients still need to renew their lease in
> > the usual way, a long-running async copy doesn't keep the lease renewed
> > automatically.
> >
> > So, the asynchronous copy shouldn't hold a reference on the client.
> >
> > The copy thread can still safely use the client while it's running,
> > because it knows that anyone destroying the client will first cancel the
> > copy and wait for the thread to die.
> 
> Ok no reference on the client. I will add a check to
> client_has_state() to check for on-going copies and then server would
> return CLIENTID_BUSY. What was already in the patch was that during
> client shutdown it would stop copies. I have tested for when the
> server is expiring client's lease, it also shuts down any on-going
> copies. I think I'm ready for the next version submission...

OK, great.--b.