[PATCH v5 00/10] NFSD support for asynchronous COPY

[PATCH v5 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

To do asynchronous copies, NFSD creates a new kthread to handle the request.
Upon receiving the COPY, it generates a unique copy stateid (stored in a
global list for keeping track of state for OFFLOAD_STATUS to be queried by),
starts the thread, and replies back to the client. nfsd4_copy arguments that
are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch
we keep the 4MB chunk and requested size for the async copy). If error is
encountered it's saved but also we save the amount of data copied so far.
Once done, the results are queued for the callback workqueue and sent via
CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running
the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
the ongoing do_splice() and once vfs returns we are choosing not to send
the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running
the copy and will query the i_size_read() of the associated filehandle of
the destination file and return the result.

v5:
1. reimplementing asynchronous copy to use kthreads instead of the workqueue
2. store asynchronous copies in the list of the nfs4_client structure
3. when copying nfsd4_copy datastructure for the async copy refcount the
nfs4_client structure as well as struct file src/dst structure
4. add refcount to the nfsd4_copy structure to coordinate with offload_status
and offload_cancel
5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to
interrupt do_splice and calls kthread_stop to wait for the copy to stop
6. offload_cancel adds MODIFIES_SOMETHING to flags
7. offload_status reports the size of the destination file form i_size_read()
instead of before 4MB loop updated chunk size.
8. for async copy call vfs_copy_file_range() with the whole copy size. still
keep the loop to do more the MAX_RW_COUNT that do_splice can do.

Olga Kornievskaia (10):
  NFSD CB_OFFLOAD xdr
  NFSD OFFLOAD_STATUS xdr
  NFSD OFFLOAD_CANCEL xdr
  NFSD xdr callback stateid in async COPY reply
  NFSD first draft of async copy
  NFSD return nfs4_stid in nfs4_preprocess_stateid_op
  NFSD create new stateid for async copy
  NFSD handle OFFLOAD_CANCEL op
  NFSD support OFFLOAD_STATUS
  NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  97 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  77 +++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

-- 
1.8.3.1

[PATCH v5 01/10] NFSD CB_OFFLOAD xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4callback.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/state.h        |  1 +
 fs/nfsd/xdr4.h         |  6 ++++
 fs/nfsd/xdr4cb.h       | 10 ++++++
 4 files changed, 114 insertions(+)

diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 49b0a9e..d12d914 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -39,6 +39,7 @@
 #include "state.h"
 #include "netns.h"
 #include "xdr4cb.h"
+#include "xdr4.h"
 
 #define NFSDDBG_FACILITY                NFSDDBG_PROC
 
@@ -105,6 +106,7 @@ enum nfs_cb_opnum4 {
 	OP_CB_WANTS_CANCELLED		= 12,
 	OP_CB_NOTIFY_LOCK		= 13,
 	OP_CB_NOTIFY_DEVICEID		= 14,
+	OP_CB_OFFLOAD			= 15,
 	OP_CB_ILLEGAL			= 10044
 };
 
@@ -683,6 +685,100 @@ static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp,
 }
 
 /*
+ * struct write_response4 {
+ *	stateid4	wr_callback_id<1>;
+ *	length4		wr_count;
+ *	stable_how4	wr_committed;
+ *	verifier4	wr_writeverf;
+ * };
+ * union offload_info4 switch (nfsstat4 coa_status) {
+ *	case NFS4_OK:
+ *		write_response4	coa_resok4;
+ *	default:
+ *	length4		coa_bytes_copied;
+ * };
+ * struct CB_OFFLOAD4args {
+ *	nfs_fh4		coa_fh;
+ *	stateid4	coa_stateid;
+ *	offload_info4	coa_offload_info;
+ * };
+ */
+static void encode_offload_info4(struct xdr_stream *xdr,
+				 __be32 nfserr,
+				 const struct nfsd4_copy *cp)
+{
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 4);
+	*p++ = nfserr;
+	if (!nfserr) {
+		p = xdr_reserve_space(xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE);
+		p = xdr_encode_empty_array(p);
+		p = xdr_encode_hyper(p, cp->cp_res.wr_bytes_written);
+		*p++ = cpu_to_be32(cp->cp_res.wr_stable_how);
+		p = xdr_encode_opaque_fixed(p, cp->cp_res.wr_verifier.data,
+					    NFS4_VERIFIER_SIZE);
+	} else {
+		p = xdr_reserve_space(xdr, 8);
+		p = xdr_encode_hyper(p, cp->cp_res.wr_bytes_written);
+	}
+}
+
+static void encode_cb_offload4args(struct xdr_stream *xdr,
+				   __be32 nfserr,
+				   const struct knfsd_fh *fh,
+				   const struct nfsd4_copy *cp,
+				   struct nfs4_cb_compound_hdr *hdr)
+{
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 4);
+	*p++ = cpu_to_be32(OP_CB_OFFLOAD);
+	encode_nfs_fh4(xdr, fh);
+	encode_stateid4(xdr, &cp->cp_res.cb_stateid);
+	encode_offload_info4(xdr, nfserr, cp);
+
+	hdr->nops++;
+}
+
+static void nfs4_xdr_enc_cb_offload(struct rpc_rqst *req,
+				    struct xdr_stream *xdr,
+				    const void *data)
+{
+	const struct nfsd4_callback *cb = data;
+	const struct nfsd4_copy *cp =
+		container_of(cb, struct nfsd4_copy, cp_cb);
+	struct nfs4_cb_compound_hdr hdr = {
+		.ident = 0,
+		.minorversion = cb->cb_clp->cl_minorversion,
+	};
+
+	encode_cb_compound4args(xdr, &hdr);
+	encode_cb_sequence4args(xdr, cb, &hdr);
+	encode_cb_offload4args(xdr, cp->nfserr, &cp->fh, cp, &hdr);
+	encode_cb_nops(&hdr);
+}
+
+static int nfs4_xdr_dec_cb_offload(struct rpc_rqst *rqstp,
+				   struct xdr_stream *xdr,
+				   void *data)
+{
+	struct nfsd4_callback *cb = data;
+	struct nfs4_cb_compound_hdr hdr;
+	int status;
+
+	status = decode_cb_compound4res(xdr, &hdr);
+	if (unlikely(status))
+		return status;
+
+	if (cb) {
+		status = decode_cb_sequence4res(xdr, cb);
+		if (unlikely(status || cb->cb_seq_status))
+			return status;
+	}
+	return decode_cb_op_status(xdr, OP_CB_OFFLOAD, &cb->cb_status);
+}
+/*
  * RPC procedure tables
  */
 #define PROC(proc, call, argtype, restype)				\
@@ -703,6 +799,7 @@ static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp,
 	PROC(CB_LAYOUT,	COMPOUND,	cb_layout,	cb_layout),
 #endif
 	PROC(CB_NOTIFY_LOCK,	COMPOUND,	cb_notify_lock,	cb_notify_lock),
+	PROC(CB_OFFLOAD,	COMPOUND,	cb_offload,	cb_offload),
 };
 
 static unsigned int nfs4_cb_counts[ARRAY_SIZE(nfs4_cb_procedures)];
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 005c911..f8b0210 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -570,6 +570,7 @@ enum nfsd4_cb_op {
 	NFSPROC4_CLNT_CB_NULL = 0,
 	NFSPROC4_CLNT_CB_RECALL,
 	NFSPROC4_CLNT_CB_LAYOUT,
+	NFSPROC4_CLNT_CB_OFFLOAD,
 	NFSPROC4_CLNT_CB_SEQUENCE,
 	NFSPROC4_CLNT_CB_NOTIFY_LOCK,
 };
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 1e4edbf..4ac2676 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -507,6 +507,7 @@ struct nfsd42_write_res {
 	u64			wr_bytes_written;
 	u32			wr_stable_how;
 	nfs4_verifier		wr_verifier;
+	stateid_t		cb_stateid;
 };
 
 struct nfsd4_copy {
@@ -523,6 +524,11 @@ struct nfsd4_copy {
 
 	/* response */
 	struct nfsd42_write_res	cp_res;
+
+	/* for cb_offload */
+	struct nfsd4_callback	cp_cb;
+	__be32			nfserr;
+	struct knfsd_fh		fh;
 };
 
 struct nfsd4_seek {
diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h
index 49b719d..7e39913 100644
--- a/fs/nfsd/xdr4cb.h
+++ b/fs/nfsd/xdr4cb.h
@@ -37,3 +37,13 @@
 #define NFS4_dec_cb_notify_lock_sz	(cb_compound_dec_hdr_sz  +      \
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
+#define enc_cb_offload_info_sz		(1 + 1 + 2 + 1 +		\
+					XDR_QUADLEN(NFS4_VERIFIER_SIZE))
+#define NFS4_enc_cb_offload_sz		(cb_compound_enc_hdr_sz +       \
+					cb_sequence_enc_sz +            \
+					enc_nfs4_fh_sz +		\
+					enc_stateid_sz +		\
+					enc_cb_offload_info_sz)
+#define NFS4_dec_cb_offload_sz		(cb_compound_dec_hdr_sz  +      \
+					cb_sequence_dec_sz +            \
+					op_dec_sz)
-- 
1.8.3.1

[PATCH v5 02/10] NFSD OFFLOAD_STATUS xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 20 ++++++++++++++++++++
 fs/nfsd/nfs4xdr.c  | 27 +++++++++++++++++++++++++--
 fs/nfsd/xdr4.h     | 10 ++++++++++
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 3c69db7..8601fc4 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1142,6 +1142,13 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	fput(file);
 	return status;
 }
+static __be32
+nfsd4_offload_status(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return nfserr_notsupp;
+}
 
 static __be32
 nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
@@ -2039,6 +2046,14 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		1 /* cr_synchronous */) * sizeof(__be32);
 }
 
+static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp,
+					     struct nfsd4_op *op)
+{
+	return (op_encode_hdr_size +
+		2 /* osr_count */ +
+		1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32);
+}
+
 #ifdef CONFIG_NFSD_PNFS
 static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {
@@ -2452,6 +2467,11 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		.op_name = "OP_SEEK",
 		.op_rsize_bop = nfsd4_seek_rsize,
 	},
+	[OP_OFFLOAD_STATUS] = {
+		.op_func = nfsd4_offload_status,
+		.op_name = "OP_OFFLOAD_STATUS",
+		.op_rsize_bop = nfsd4_offload_status_rsize,
+	},
 };
 
 /**
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 2c61c6b..f8111bb 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1768,6 +1768,13 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 }
 
 static __be32
+nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp,
+			    struct nfsd4_offload_status *os)
+{
+	return nfsd4_decode_stateid(argp, &os->stateid);
+}
+
+static __be32
 nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek)
 {
 	DECODE_HEAD;
@@ -1874,7 +1881,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_dec)nfsd4_decode_notsupp,
@@ -4216,6 +4223,22 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 }
 
 static __be32
+nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr,
+			    struct nfsd4_offload_status *os)
+{
+	struct xdr_stream *xdr = &resp->xdr;
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 8 + 4);
+	if (!p)
+		return nfserr_resource;
+	p = xdr_encode_hyper(p, os->count);
+	*p++ = cpu_to_be32(0);
+
+	return nfserr;
+}
+
+static __be32
 nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr,
 		  struct nfsd4_seek *seek)
 {
@@ -4318,7 +4341,7 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 	[OP_LAYOUTERROR]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_LAYOUTSTATS]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_enc)nfsd4_encode_noop,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_noop,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_SEEK]		= (nfsd4_enc)nfsd4_encode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_enc)nfsd4_encode_noop,
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 4ac2676..9b0c099 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -542,6 +542,15 @@ struct nfsd4_seek {
 	loff_t		seek_pos;
 };
 
+struct nfsd4_offload_status {
+	/* request */
+	stateid_t	stateid;
+
+	/* response */
+	u64		count;
+	u32		status;
+};
+
 struct nfsd4_op {
 	int					opnum;
 	const struct nfsd4_operation *		opdesc;
@@ -600,6 +609,7 @@ struct nfsd4_op {
 		struct nfsd4_fallocate		deallocate;
 		struct nfsd4_clone		clone;
 		struct nfsd4_copy		copy;
+		struct nfsd4_offload_status	offload_status;
 		struct nfsd4_seek		seek;
 	} u;
 	struct nfs4_replay *			replay;
-- 
1.8.3.1

[PATCH v5 03/10] NFSD OFFLOAD_CANCEL xdr

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 14 ++++++++++++++
 fs/nfsd/nfs4xdr.c  |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 8601fc4..ea698a4 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1121,6 +1121,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 }
 
 static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return 0;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		struct nfsd4_fallocate *fallocate, int flags)
 {
@@ -2472,6 +2480,12 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		.op_name = "OP_OFFLOAD_STATUS",
 		.op_rsize_bop = nfsd4_offload_status_rsize,
 	},
+	[OP_OFFLOAD_CANCEL] = {
+		.op_func = nfsd4_offload_cancel,
+		.op_flags = OP_MODIFIES_SOMETHING,
+		.op_name = "OP_OFFLOAD_CANCEL",
+		.op_rsize_bop = nfsd4_only_status_rsize,
+	},
 };
 
 /**
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index f8111bb..55db3c4 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -1880,7 +1880,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_IO_ADVISE]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
-- 
1.8.3.1

[PATCH v5 04/10] NFSD xdr callback stateid in async COPY reply

[Olga Kornievskaia]

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4xdr.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 55db3c4..27c4509 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -4190,15 +4190,27 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 #endif /* CONFIG_NFSD_PNFS */
 
 static __be32
-nfsd42_encode_write_res(struct nfsd4_compoundres *resp, struct nfsd42_write_res *write)
+nfsd42_encode_write_res(struct nfsd4_compoundres *resp,
+		struct nfsd42_write_res *write, bool sync)
 {
 	__be32 *p;
+	p = xdr_reserve_space(&resp->xdr, 4);
+	if (!p)
+		return nfserr_resource;
 
-	p = xdr_reserve_space(&resp->xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE);
+	if (sync)
+		*p++ = cpu_to_be32(0);
+	else {
+		__be32 nfserr;
+		*p++ = cpu_to_be32(1);
+		nfserr = nfsd4_encode_stateid(&resp->xdr, &write->cb_stateid);
+		if (nfserr)
+			return nfserr;
+	}
+	p = xdr_reserve_space(&resp->xdr, 8 + 4 + NFS4_VERIFIER_SIZE);
 	if (!p)
 		return nfserr_resource;
 
-	*p++ = cpu_to_be32(0);
 	p = xdr_encode_hyper(p, write->wr_bytes_written);
 	*p++ = cpu_to_be32(write->wr_stable_how);
 	p = xdr_encode_opaque_fixed(p, write->wr_verifier.data,
@@ -4212,7 +4224,8 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 {
 	__be32 *p;
 
-	nfserr = nfsd42_encode_write_res(resp, &copy->cp_res);
+	nfserr = nfsd42_encode_write_res(resp, &copy->cp_res,
+			copy->cp_synchronous);
 	if (nfserr)
 		return nfserr;
 
-- 
1.8.3.1

[PATCH v5 05/10] NFSD first draft of async copy

[Olga Kornievskaia]

Upon receiving a request for async copy, create a new kthread.
If we get asynchronous request, make sure to copy the needed
arguments/state from the stack before starting the copy. Then
start the thread and reply back to the client indicating copy
is asynchronous.

nfsd_copy_file_range() will copy in a loop over the total
number of bytes is needed to copy. In case a failure happens
in the middle, we can return an error as well as how much we
copied so far. Once done creating a workitem for the callback
workqueue and send CB_OFFLOAD with the results.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 177 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   2 +
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   9 +++
 4 files changed, 171 insertions(+), 19 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index ea698a4..2787c95 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -35,6 +35,7 @@
 #include <linux/file.h>
 #include <linux/falloc.h>
 #include <linux/slab.h>
+#include <linux/kthread.h>
 
 #include "idmap.h"
 #include "cache.h"
@@ -1085,39 +1086,177 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	size_t bytes_to_copy;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		if (copy->cp_synchronous)
+			bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+		else
+			bytes_to_copy = bytes_total;
+		bytes_copied = vfs_copy_file_range(copy->fh_src, src_pos,
+				copy->fh_dst, dst_pos, bytes_to_copy, 0);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->net = src->net;
+	dst->cp_clp = src->cp_clp;
+	atomic_inc(&dst->cp_clp->cl_refcount);
+	dst->fh_dst = get_file(src->fh_dst);
+	dst->fh_src = get_file(src->fh_src);
+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy)
+{
+	fput(copy->fh_dst);
+	fput(copy->fh_src);
+	spin_lock(&copy->cp_clp->async_lock);
+	list_del(&copy->copies);
+	spin_unlock(&copy->cp_clp->async_lock);
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_do_async_copy(void *data)
+{
+	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	atomic_inc(&cb_copy->cp_clp->cl_refcount);
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	cleanup_async_copy(copy);
+	return 0;
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
 	struct nfsd4_copy *copy = &u->copy;
-	struct file *src, *dst;
 	__be32 status;
-	ssize_t bytes;
+	struct nfsd4_copy *async_copy = NULL;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);
 	if (status)
 		goto out;
 
-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	if (!copy->cp_synchronous) {
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		spin_lock(&async_copy->cp_clp->async_lock);
+		list_add(&async_copy->copies,
+				&async_copy->cp_clp->async_copies);
+		spin_unlock(&async_copy->cp_clp->async_lock);
+		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+				async_copy, "%s", "copy thread");
+		if (IS_ERR(async_copy->copy_task)) {
+			status = PTR_ERR(async_copy->copy_task);
+			goto out_err_dec;
+		}
+		wake_up_process(async_copy->copy_task);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
+out_err_dec:
+	cleanup_async_copy(async_copy);
+	goto out;
 }
 
 static __be32
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 0c04f81..d7767a1 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1774,6 +1774,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 #ifdef CONFIG_NFSD_PNFS
 	INIT_LIST_HEAD(&clp->cl_lo_states);
 #endif
+	INIT_LIST_HEAD(&clp->async_copies);
+	spin_lock_init(&clp->async_lock);
 	spin_lock_init(&clp->cl_lock);
 	rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
 	return clp;
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index f8b0210..9189062 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -352,6 +352,8 @@ struct nfs4_client {
 	struct rpc_wait_queue	cl_cb_waitq;	/* backchannel callers may */
 						/* wait here for slots */
 	struct net		*net;
+	struct list_head	async_copies;	/* list of async copies */
+	spinlock_t		async_lock;	/* lock for async copies */
 };
 
 /* struct nfs4_client_reset
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 9b0c099..0a19954 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -529,6 +529,15 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
+
+	struct list_head	copies;
+	struct task_struct	*copy_task;
 };
 
 struct nfsd4_seek {
-- 
1.8.3.1

[PATCH v5 06/10] NFSD return nfs4_stid in nfs4_preprocess_stateid_op

[Olga Kornievskaia]

Needed for copy to add nfs4_cp_state to the nfs4_stid.

Signed-off-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 17 ++++++++++-------
 fs/nfsd/nfs4state.c |  8 ++++++--
 fs/nfsd/state.h     |  3 ++-
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 2787c95..6b6e47f 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -773,7 +773,8 @@ static __be32 nfsd4_do_lookupp(struct svc_rqst *rqstp, struct svc_fh *fh)
 	/* check stateid */
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					&read->rd_stateid, RD_STATE,
-					&read->rd_filp, &read->rd_tmp_file);
+					&read->rd_filp, &read->rd_tmp_file,
+					NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_read: couldn't process stateid!\n");
 		goto out;
@@ -939,7 +940,7 @@ static __be32 nfsd4_do_lookupp(struct svc_rqst *rqstp, struct svc_fh *fh)
 	if (setattr->sa_iattr.ia_valid & ATTR_SIZE) {
 		status = nfs4_preprocess_stateid_op(rqstp, cstate,
 				&cstate->current_fh, &setattr->sa_stateid,
-				WR_STATE, NULL, NULL);
+				WR_STATE, NULL, NULL, NULL);
 		if (status) {
 			dprintk("NFSD: nfsd4_setattr: couldn't process stateid!\n");
 			return status;
@@ -1005,7 +1006,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 		return nfserr_inval;
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
-						stateid, WR_STATE, &filp, NULL);
+					stateid, WR_STATE, &filp, NULL, NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_write: couldn't process stateid!\n");
 		return status;
@@ -1036,14 +1037,16 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh,
-					    src_stateid, RD_STATE, src, NULL);
+					    src_stateid, RD_STATE, src, NULL,
+					    NULL);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process src stateid!\n", __func__);
 		goto out;
 	}
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
-					    dst_stateid, WR_STATE, dst, NULL);
+					    dst_stateid, WR_STATE, dst, NULL,
+					    NULL);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;
@@ -1276,7 +1279,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &fallocate->falloc_stateid,
-					    WR_STATE, &file, NULL);
+					    WR_STATE, &file, NULL, NULL);
 	if (status != nfs_ok) {
 		dprintk("NFSD: nfsd4_fallocate: couldn't process stateid!\n");
 		return status;
@@ -1323,7 +1326,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &seek->seek_stateid,
-					    RD_STATE, &file, NULL);
+					    RD_STATE, &file, NULL, NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_seek: couldn't process stateid!\n");
 		return status;
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index d7767a1..af40762 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -4947,7 +4947,8 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 __be32
 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
-		stateid_t *stateid, int flags, struct file **filpp, bool *tmp_file)
+		stateid_t *stateid, int flags, struct file **filpp,
+		bool *tmp_file, struct nfs4_stid **cstid)
 {
 	struct inode *ino = d_inode(fhp->fh_dentry);
 	struct net *net = SVC_NET(rqstp);
@@ -4998,8 +4999,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 	if (!status && filpp)
 		status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
 out:
-	if (s)
+	if (s) {
+		if (!status && cstid)
+			*cstid = s;
 		nfs4_put_stid(s);
+	}
 	return status;
 }
 
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 9189062..d58507b 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -602,7 +602,8 @@ struct nfsd4_blocked_lock {
 
 extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
-		stateid_t *stateid, int flags, struct file **filp, bool *tmp_file);
+		stateid_t *stateid, int flags, struct file **filp,
+		bool *tmp_file, struct nfs4_stid **cstid);
 __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     stateid_t *stateid, unsigned char typemask,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
-- 
1.8.3.1

[PATCH v5 07/10] NFSD create new stateid for async copy

[Olga Kornievskaia]

Generate a new stateid to be used for reply to the asynchronous
COPY (this would also be used later by COPY_NOTIFY as well).
Associate the stateid with the parent OPEN/LOCK/DELEG stateid
that can be freed during the free of the parent stateid. However,
right now deciding to bind the lifetime to when the vfs copy
is done. This way don't need to keep the nfsd_net structure for
the callback. The drawback is that time copy state information
is available for query by OFFLOAD_STATUS is slightly less.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 32 +++++++++++++++++++-------
 fs/nfsd/nfs4state.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 ++++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 115 insertions(+), 8 deletions(-)

diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 3714231..2c88a95 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -119,6 +119,14 @@ struct nfsd_net {
 	u32 clverifier_counter;
 
 	struct svc_serv *nfsd_serv;
+
+	/*
+	 * clientid and stateid data for construction of net unique COPY
+	 * stateids.
+	 */
+	u32		s2s_cp_cl_id;
+	struct idr	s2s_cp_stateids;
+	spinlock_t	s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 6b6e47f..5786ed9 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1032,7 +1032,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		  stateid_t *src_stateid, struct file **src,
-		  stateid_t *dst_stateid, struct file **dst)
+		  stateid_t *dst_stateid, struct file **dst,
+		  struct nfs4_stid **stid)
 {
 	__be32 status;
 
@@ -1046,7 +1047,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    dst_stateid, WR_STATE, dst, NULL,
-					    NULL);
+					    stid);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;
@@ -1077,7 +1078,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-				   &clone->cl_dst_stateid, &dst);
+				   &clone->cl_dst_stateid, &dst, NULL);
 	if (status)
 		goto out;
 
@@ -1110,8 +1111,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-		sizeof(copy->cp_dst_stateid));
 	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
 	copy->cp_consecutive = 1;
 	copy->cp_synchronous = sync;
@@ -1178,10 +1177,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
 	atomic_inc(&dst->cp_clp->cl_refcount);
 	dst->fh_dst = get_file(src->fh_dst);
 	dst->fh_src = get_file(src->fh_src);
+	dst->stid = src->stid;
+	dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy)
 {
+	list_del(&copy->cps->cp_list);
+	nfs4_free_cp_state(copy->cps);
+	nfs4_put_stid(copy->stid);
 	fput(copy->fh_dst);
 	fput(copy->fh_src);
 	spin_lock(&copy->cp_clp->async_lock);
@@ -1223,7 +1227,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
 				   &copy->fh_src, &copy->cp_dst_stateid,
-				   &copy->fh_dst);
+				   &copy->fh_dst, &copy->stid);
 	if (status)
 		goto out;
 
@@ -1232,15 +1236,27 @@ static int nfsd4_do_async_copy(void *data)
 		sizeof(struct knfsd_fh));
 	copy->net = SVC_NET(rqstp);
 	if (!copy->cp_synchronous) {
+		struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
 		status = nfsd4_init_copy_res(copy, 0);
 		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 		if (!async_copy) {
 			status = nfserrno(-ENOMEM);
 			goto out;
 		}
+		copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+		if (!copy->cps) {
+			status = nfserrno(-ENOMEM);
+			kfree(async_copy);
+			goto out;
+		}
+		/* take a reference on the parent stateid so it's not
+		 * not freed by the copy compound
+		 */
+		atomic_inc(&copy->stid->sc_count);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+			sizeof(copy->cps->cp_stateid));
 		dup_copy_fields(copy, async_copy);
-		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-			sizeof(copy->cp_dst_stateid));
 		spin_lock(&async_copy->cp_clp->async_lock);
 		list_add(&async_copy->copies,
 				&async_copy->cp_clp->async_copies);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index af40762..f9151f2 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -658,6 +658,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	/* Will be incremented before return to client: */
 	atomic_set(&stid->sc_count, 1);
 	spin_lock_init(&stid->sc_lock);
+	INIT_LIST_HEAD(&stid->sc_cp_list);
 
 	/*
 	 * It shouldn't be a problem to reuse an opaque stateid value.
@@ -674,6 +675,66 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+					       struct nfs4_stid *p_stid)
+{
+	struct nfs4_cp_state *cps;
+	int new_id;
+
+	cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+	if (!cps)
+		return NULL;
+	idr_preload(GFP_KERNEL);
+	spin_lock(&nn->s2s_cp_lock);
+	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+	spin_unlock(&nn->s2s_cp_lock);
+	idr_preload_end();
+	if (new_id < 0)
+		goto out_free;
+	cps->cp_stateid.si_opaque.so_id = new_id;
+	cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+	cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	cps->cp_p_stid = p_stid;
+	INIT_LIST_HEAD(&cps->cp_list);
+	list_add(&cps->cp_list, &p_stid->sc_cp_list);
+
+	return cps;
+out_free:
+	kfree(cps);
+	return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+	struct nfsd_net *nn;
+
+	nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+	spin_lock(&nn->s2s_cp_lock);
+	idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+	spin_unlock(&nn->s2s_cp_lock);
+
+	kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+	struct nfs4_cp_state *cps;
+
+	might_sleep();
+
+	while (!list_empty(&stid->sc_cp_list)) {
+		cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+				       cp_list);
+		list_del(&cps->cp_list);
+		nfs4_free_cp_state(cps);
+	}
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
 	struct nfs4_stid *stid;
@@ -819,6 +880,9 @@ static void block_delegations(struct knfsd_fh *fh)
 	}
 	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
 	spin_unlock(&clp->cl_lock);
+
+	nfs4_free_cp_statelist(s);
+
 	s->sc_free(s);
 	if (fp)
 		put_nfs4_file(fp);
@@ -6954,6 +7018,8 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_LIST_HEAD(&nn->close_lru);
 	INIT_LIST_HEAD(&nn->del_recall_lru);
 	spin_lock_init(&nn->client_lock);
+	spin_lock_init(&nn->s2s_cp_lock);
+	idr_init(&nn->s2s_cp_stateids);
 
 	spin_lock_init(&nn->blocked_locks_lock);
 	INIT_LIST_HEAD(&nn->blocked_locks_lru);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 6493df6..232270d 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
 	nn->nfsd4_grace = 90;
 	nn->clverifier_counter = prandom_u32();
 	nn->clientid_counter = prandom_u32();
+	nn->s2s_cp_cl_id = nn->clientid_counter++;
 	return 0;
 
 out_idmap_error:
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index d58507b..b4de8ae 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -93,6 +93,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+	struct list_head	sc_cp_list;
 	unsigned char		sc_type;
 	stateid_t		sc_stateid;
 	spinlock_t		sc_lock;
@@ -102,6 +103,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+	stateid_t		cp_stateid;
+	struct list_head	cp_list;	/* per parent nfs4_stid */
+	struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:
@@ -609,6 +621,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
 				  void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 0a19954..27bac6a 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -535,6 +535,8 @@ struct nfsd4_copy {
 	struct file             *fh_src;
 	struct file             *fh_dst;
 	struct net              *net;
+	struct nfs4_stid        *stid;
+	struct nfs4_cp_state    *cps;
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;
-- 
1.8.3.1

[PATCH v5 08/10] NFSD handle OFFLOAD_CANCEL op

[Olga Kornievskaia]

Upon receiving OFFLOAD_CANCEL search the list of copy stateids,
if found then set the SIGPENDING signal so that do_splice stops
copying and also send kthread_stop to the copy thread to stop
and wait for it. Take a reference on the copy from the
offload_cancel thread so that it won't go away while we are
trying to process it. Server won't be sending CB_OFFLOAD to the
client since it received a cancel.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++--
 fs/nfsd/xdr4.h     |  1 +
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 5786ed9..6757387 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1090,6 +1090,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+
+static void nfs4_put_copy(struct nfsd4_copy *copy)
+{
+	if (!atomic_dec_and_test(&copy->refcount))
+		return;
+	kfree(copy);
+}
+
 static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
 {
 	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
@@ -1128,6 +1136,8 @@ static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
 	u64 dst_pos = copy->cp_dst_pos;
 
 	do {
+		if (signalled() || kthread_should_stop())
+			return -1;
 		if (copy->cp_synchronous)
 			bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
 		else
@@ -1150,11 +1160,16 @@ static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
 	ssize_t bytes;
 
 	bytes = _nfsd_copy_file_range(copy);
+	if (signalled() || kthread_should_stop()) {
+		status = -1;
+		goto cleanup;
+	}
 	if (bytes < 0)
 		status = nfserrno(bytes);
 	else
 		status = nfsd4_init_copy_res(copy, sync);
 
+cleanup:
 	fput(copy->fh_src);
 	fput(copy->fh_dst);
 	return status;
@@ -1192,7 +1207,7 @@ static void cleanup_async_copy(struct nfsd4_copy *copy)
 	list_del(&copy->copies);
 	spin_unlock(&copy->cp_clp->async_lock);
 	atomic_dec(&copy->cp_clp->cl_refcount);
-	kfree(copy);
+	nfs4_put_copy(copy);
 }
 
 static int nfsd4_do_async_copy(void *data)
@@ -1201,6 +1216,9 @@ static int nfsd4_do_async_copy(void *data)
 	struct nfsd4_copy *cb_copy;
 
 	copy->nfserr = nfsd4_do_copy(copy, 0);
+	if (signalled() || kthread_should_stop())
+		goto out;
+
 	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 	if (!cb_copy)
 		goto out;
@@ -1257,6 +1275,7 @@ static int nfsd4_do_async_copy(void *data)
 		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
 			sizeof(copy->cps->cp_stateid));
 		dup_copy_fields(copy, async_copy);
+		atomic_set(&async_copy->refcount, 1);
 		spin_lock(&async_copy->cp_clp->async_lock);
 		list_add(&async_copy->copies,
 				&async_copy->cp_clp->async_copies);
@@ -1283,7 +1302,30 @@ static int nfsd4_do_async_copy(void *data)
 		     struct nfsd4_compound_state *cstate,
 		     union nfsd4_op_u *u)
 {
-	return 0;
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	bool found = false;
+	struct nfs4_client *clp = cstate->clp;
+
+	spin_lock(&clp->async_lock);
+	list_for_each_entry(copy, &clp->async_copies, copies) {
+		if (memcmp(&copy->cps->cp_stateid, &os->stateid,
+				NFS4_STATEID_SIZE))
+			continue;
+		found = true;
+		atomic_inc(&copy->refcount);
+		break;
+	}
+	spin_unlock(&clp->async_lock);
+	if (found) {
+		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
+		kthread_stop(copy->copy_task);
+		nfs4_put_copy(copy);
+	} else
+		status = nfserr_bad_stateid;
+
+	return status;
 }
 
 static __be32
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 27bac6a..3127b58 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -540,6 +540,7 @@ struct nfsd4_copy {
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;
+	atomic_t		refcount;
 };
 
 struct nfsd4_seek {
-- 
1.8.3.1

[PATCH v5 09/10] NFSD support OFFLOAD_STATUS

[Olga Kornievskaia]

Search the list for the asynchronous copy based on the stateid,
then query the inode for the file size.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c | 44 ++++++++++++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 6757387..78e25db 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1297,28 +1297,36 @@ static int nfsd4_do_async_copy(void *data)
 	goto out;
 }
 
-static __be32
-nfsd4_offload_cancel(struct svc_rqst *rqstp,
-		     struct nfsd4_compound_state *cstate,
-		     union nfsd4_op_u *u)
+static struct nfsd4_copy *
+find_async_copy(struct nfs4_client *clp, struct nfsd4_offload_status *os)
 {
-	struct nfsd4_offload_status *os = &u->offload_status;
-	__be32 status = 0;
 	struct nfsd4_copy *copy;
-	bool found = false;
-	struct nfs4_client *clp = cstate->clp;
 
 	spin_lock(&clp->async_lock);
 	list_for_each_entry(copy, &clp->async_copies, copies) {
 		if (memcmp(&copy->cps->cp_stateid, &os->stateid,
 				NFS4_STATEID_SIZE))
 			continue;
-		found = true;
 		atomic_inc(&copy->refcount);
-		break;
+		spin_unlock(&clp->async_lock);
+		return copy;
 	}
 	spin_unlock(&clp->async_lock);
-	if (found) {
+	return NULL;
+}
+
+static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	struct nfs4_client *clp = cstate->clp;
+
+	copy = find_async_copy(clp, os);
+	if (copy) {
 		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
 		kthread_stop(copy->copy_task);
 		nfs4_put_copy(copy);
@@ -1355,7 +1363,19 @@ static int nfsd4_do_async_copy(void *data)
 		     struct nfsd4_compound_state *cstate,
 		     union nfsd4_op_u *u)
 {
-	return nfserr_notsupp;
+	struct nfsd4_offload_status *os = &u->offload_status;
+	__be32 status = 0;
+	struct nfsd4_copy *copy;
+	struct nfs4_client *clp = cstate->clp;
+
+	copy = find_async_copy(clp, os);
+	if (copy) {
+		os->count = i_size_read(file_inode(copy->fh_dst));
+		nfs4_put_copy(copy);
+	} else
+		status = nfserr_bad_stateid;
+
+	return status;
 }
 
 static __be32
-- 
1.8.3.1

[PATCH v5 10/10] NFSD stop queued async copies on client shutdown

[Olga Kornievskaia]

If client is shutting down and there are still async copies going
on, then stop queued async copies.

Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
 fs/nfsd/nfs4proc.c  | 13 +++++++++++++
 fs/nfsd/nfs4state.c |  1 +
 fs/nfsd/state.h     |  1 +
 3 files changed, 15 insertions(+)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 78e25db..b00701d 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -1098,6 +1098,19 @@ static void nfs4_put_copy(struct nfsd4_copy *copy)
 	kfree(copy);
 }
 
+void nfsd4_shutdown_copy(struct nfs4_client *clp)
+{
+	struct nfsd4_copy *copy;
+
+	spin_lock(&clp->async_lock);
+	list_for_each_entry(copy, &clp->async_copies, copies) {
+		set_tsk_thread_flag(copy->copy_task, TIF_SIGPENDING);
+		kthread_stop(copy->copy_task);
+		nfs4_put_copy(copy);
+	}
+	spin_unlock(&clp->async_lock);
+}
+
 static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
 {
 	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index f9151f2..efac39d 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1947,6 +1947,7 @@ static __be32 mark_client_expired_locked(struct nfs4_client *clp)
 	}
 	nfsd4_return_all_client_layouts(clp);
 	nfsd4_shutdown_callback(clp);
+	nfsd4_shutdown_copy(clp);
 	if (clp->cl_cb_conn.cb_xprt)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
 	free_client(clp);
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index b4de8ae..88869df 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -643,6 +643,7 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern int nfsd4_create_callback_queue(void);
 extern void nfsd4_destroy_callback_queue(void);
 extern void nfsd4_shutdown_callback(struct nfs4_client *);
+extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
 extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
 							struct nfsd_net *nn);
-- 
1.8.3.1

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[J. Bruce Fields]

On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
> To do asynchronous copies, NFSD creates a new kthread to handle the request.
> Upon receiving the COPY, it generates a unique copy stateid (stored in a
> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
> starts the thread, and replies back to the client. nfsd4_copy arguments that
> are allocated on the stack are copies for the kthread.
> 
> In the async copy handler, it calls into VFS copy_file_range() (for synch
> we keep the 4MB chunk and requested size for the async copy). If error is
> encountered it's saved but also we save the amount of data copied so far.
> Once done, the results are queued for the callback workqueue and sent via
> CB_OFFLOAD.
> 
> When the server received an OFFLOAD_CANCEL, it will find the kthread running
> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
> the ongoing do_splice() and once vfs returns we are choosing not to send
> the CB_OFFLOAD back to the client.
> 
> When the server receives an OFFLOAD_STATUS, it will find the kthread running
> the copy and will query the i_size_read() of the associated filehandle of
> the destination file and return the result.

That assumes we're copying into a previously empty file?

--b.

> 
> v5:
> 1. reimplementing asynchronous copy to use kthreads instead of the workqueue
> 2. store asynchronous copies in the list of the nfs4_client structure
> 3. when copying nfsd4_copy datastructure for the async copy refcount the
> nfs4_client structure as well as struct file src/dst structure
> 4. add refcount to the nfsd4_copy structure to coordinate with offload_status
> and offload_cancel
> 5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to
> interrupt do_splice and calls kthread_stop to wait for the copy to stop
> 6. offload_cancel adds MODIFIES_SOMETHING to flags
> 7. offload_status reports the size of the destination file form i_size_read()
> instead of before 4MB loop updated chunk size.
> 8. for async copy call vfs_copy_file_range() with the whole copy size. still
> keep the loop to do more the MAX_RW_COUNT that do_splice can do.
> 
> Olga Kornievskaia (10):
>   NFSD CB_OFFLOAD xdr
>   NFSD OFFLOAD_STATUS xdr
>   NFSD OFFLOAD_CANCEL xdr
>   NFSD xdr callback stateid in async COPY reply
>   NFSD first draft of async copy
>   NFSD return nfs4_stid in nfs4_preprocess_stateid_op
>   NFSD create new stateid for async copy
>   NFSD handle OFFLOAD_CANCEL op
>   NFSD support OFFLOAD_STATUS
>   NFSD stop queued async copies on client shutdown
> 
>  fs/nfsd/netns.h        |   8 ++
>  fs/nfsd/nfs4callback.c |  97 +++++++++++++++
>  fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
>  fs/nfsd/nfs4state.c    |  77 +++++++++++-
>  fs/nfsd/nfs4xdr.c      |  50 ++++++--
>  fs/nfsd/nfsctl.c       |   1 +
>  fs/nfsd/state.h        |  21 +++-
>  fs/nfsd/xdr4.h         |  28 +++++
>  fs/nfsd/xdr4cb.h       |  10 ++
>  9 files changed, 576 insertions(+), 37 deletions(-)
> 
> -- 
> 1.8.3.1
> 

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>> are allocated on the stack are copies for the kthread.
>>
>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>> we keep the 4MB chunk and requested size for the async copy). If error is
>> encountered it's saved but also we save the amount of data copied so far.
>> Once done, the results are queued for the callback workqueue and sent via
>> CB_OFFLOAD.
>>
>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>> the ongoing do_splice() and once vfs returns we are choosing not to send
>> the CB_OFFLOAD back to the client.
>>
>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>> the copy and will query the i_size_read() of the associated filehandle of
>> the destination file and return the result.
>
> That assumes we're copying into a previously empty file?

Sigh. Alright, then it's back to my original solution where I broke
everything into 4MB calls and kept track of bytes copies so far.

>
> --b.
>
>>
>> v5:
>> 1. reimplementing asynchronous copy to use kthreads instead of the workqueue
>> 2. store asynchronous copies in the list of the nfs4_client structure
>> 3. when copying nfsd4_copy datastructure for the async copy refcount the
>> nfs4_client structure as well as struct file src/dst structure
>> 4. add refcount to the nfsd4_copy structure to coordinate with offload_status
>> and offload_cancel
>> 5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to
>> interrupt do_splice and calls kthread_stop to wait for the copy to stop
>> 6. offload_cancel adds MODIFIES_SOMETHING to flags
>> 7. offload_status reports the size of the destination file form i_size_read()
>> instead of before 4MB loop updated chunk size.
>> 8. for async copy call vfs_copy_file_range() with the whole copy size. still
>> keep the loop to do more the MAX_RW_COUNT that do_splice can do.
>>
>> Olga Kornievskaia (10):
>>   NFSD CB_OFFLOAD xdr
>>   NFSD OFFLOAD_STATUS xdr
>>   NFSD OFFLOAD_CANCEL xdr
>>   NFSD xdr callback stateid in async COPY reply
>>   NFSD first draft of async copy
>>   NFSD return nfs4_stid in nfs4_preprocess_stateid_op
>>   NFSD create new stateid for async copy
>>   NFSD handle OFFLOAD_CANCEL op
>>   NFSD support OFFLOAD_STATUS
>>   NFSD stop queued async copies on client shutdown
>>
>>  fs/nfsd/netns.h        |   8 ++
>>  fs/nfsd/nfs4callback.c |  97 +++++++++++++++
>>  fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
>>  fs/nfsd/nfs4state.c    |  77 +++++++++++-
>>  fs/nfsd/nfs4xdr.c      |  50 ++++++--
>>  fs/nfsd/nfsctl.c       |   1 +
>>  fs/nfsd/state.h        |  21 +++-
>>  fs/nfsd/xdr4.h         |  28 +++++
>>  fs/nfsd/xdr4cb.h       |  10 ++
>>  9 files changed, 576 insertions(+), 37 deletions(-)
>>
>> --
>> 1.8.3.1
>>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v5 05/10] NFSD first draft of async copy

[Olga Kornievskaia]

On Fri, Oct 13, 2017 at 4:54 PM, Olga Kornievskaia <kolga@netapp.com> wrote:
> Upon receiving a request for async copy, create a new kthread.
> If we get asynchronous request, make sure to copy the needed
> arguments/state from the stack before starting the copy. Then
> start the thread and reply back to the client indicating copy
> is asynchronous.
>
> nfsd_copy_file_range() will copy in a loop over the total
> number of bytes is needed to copy. In case a failure happens
> in the middle, we can return an error as well as how much we
> copied so far. Once done creating a workitem for the callback
> workqueue and send CB_OFFLOAD with the results.
>
> Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
> ---
>  fs/nfsd/nfs4proc.c  | 177 ++++++++++++++++++++++++++++++++++++++++++++++------
>  fs/nfsd/nfs4state.c |   2 +
>  fs/nfsd/state.h     |   2 +
>  fs/nfsd/xdr4.h      |   9 +++
>  4 files changed, 171 insertions(+), 19 deletions(-)
>
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index ea698a4..2787c95 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -35,6 +35,7 @@
>  #include <linux/file.h>
>  #include <linux/falloc.h>
>  #include <linux/slab.h>
> +#include <linux/kthread.h>
>
>  #include "idmap.h"
>  #include "cache.h"
> @@ -1085,39 +1086,177 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
>  out:
>         return status;
>  }
> +static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
> +{
> +       struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
> +
> +       atomic_dec(&copy->cp_clp->cl_refcount);
> +       kfree(copy);
> +}
> +
> +static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
> +                                struct rpc_task *task)
> +{
> +       return 1;
> +}
> +
> +static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
> +       .release = nfsd4_cb_offload_release,
> +       .done = nfsd4_cb_offload_done
> +};
> +
> +static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
> +{
> +       memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +               sizeof(copy->cp_dst_stateid));
> +       copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> +       copy->cp_consecutive = 1;
> +       copy->cp_synchronous = sync;
> +       gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
> +
> +       return nfs_ok;
> +}
> +
> +static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
> +{
> +       ssize_t bytes_copied = 0;
> +       size_t bytes_total = copy->cp_count;
> +       size_t bytes_to_copy;
> +       u64 src_pos = copy->cp_src_pos;
> +       u64 dst_pos = copy->cp_dst_pos;
> +
> +       do {
> +               if (copy->cp_synchronous)
> +                       bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);

urgh that should have been 4MB value from the vfs.c file not the
do_splice max. I should also remove the nds_copy_file_range() from
vfs.c as it's no longer called. To implement the offload_status, if
you don't object, I'd rather go back for both sync and async to do 4MB
calls into VFS to get some kind of copy progress to report.

> +               else
> +                       bytes_to_copy = bytes_total;
> +               bytes_copied = vfs_copy_file_range(copy->fh_src, src_pos,
> +                               copy->fh_dst, dst_pos, bytes_to_copy, 0);
> +               if (bytes_copied <= 0)
> +                       break;
> +               bytes_total -= bytes_copied;
> +               copy->cp_res.wr_bytes_written += bytes_copied;
> +               src_pos += bytes_copied;
> +               dst_pos += bytes_copied;
> +       } while (bytes_total > 0 && !copy->cp_synchronous);
> +       return bytes_copied;
> +}
> +
> +static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
> +{
> +       __be32 status;
> +       ssize_t bytes;
> +
> +       bytes = _nfsd_copy_file_range(copy);
> +       if (bytes < 0)
> +               status = nfserrno(bytes);
> +       else
> +               status = nfsd4_init_copy_res(copy, sync);
> +
> +       fput(copy->fh_src);
> +       fput(copy->fh_dst);
> +       return status;
> +}
> +
> +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
> +{
> +       memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
> +       memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
> +       dst->cp_src_pos = src->cp_src_pos;
> +       dst->cp_dst_pos = src->cp_dst_pos;
> +       dst->cp_count = src->cp_count;
> +       dst->cp_consecutive = src->cp_consecutive;
> +       dst->cp_synchronous = src->cp_synchronous;
> +       memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
> +       /* skipping nfsd4_callback */
> +       memcpy(&dst->fh, &src->fh, sizeof(src->fh));
> +       dst->net = src->net;
> +       dst->cp_clp = src->cp_clp;
> +       atomic_inc(&dst->cp_clp->cl_refcount);
> +       dst->fh_dst = get_file(src->fh_dst);
> +       dst->fh_src = get_file(src->fh_src);
> +}
> +
> +static void cleanup_async_copy(struct nfsd4_copy *copy)
> +{
> +       fput(copy->fh_dst);
> +       fput(copy->fh_src);
> +       spin_lock(&copy->cp_clp->async_lock);
> +       list_del(&copy->copies);
> +       spin_unlock(&copy->cp_clp->async_lock);
> +       atomic_dec(&copy->cp_clp->cl_refcount);
> +       kfree(copy);
> +}
> +
> +static int nfsd4_do_async_copy(void *data)
> +{
> +       struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
> +       struct nfsd4_copy *cb_copy;
> +
> +       copy->nfserr = nfsd4_do_copy(copy, 0);
> +       cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> +       if (!cb_copy)
> +               goto out;
> +       memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
> +       cb_copy->cp_clp = copy->cp_clp;
> +       atomic_inc(&cb_copy->cp_clp->cl_refcount);
> +       cb_copy->nfserr = copy->nfserr;
> +       memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
> +       nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
> +                       &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
> +       nfsd4_run_cb(&cb_copy->cp_cb);
> +out:
> +       cleanup_async_copy(copy);
> +       return 0;
> +}
>
>  static __be32
>  nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
>                 union nfsd4_op_u *u)
>  {
>         struct nfsd4_copy *copy = &u->copy;
> -       struct file *src, *dst;
>         __be32 status;
> -       ssize_t bytes;
> +       struct nfsd4_copy *async_copy = NULL;
>
> -       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
> -                                  &copy->cp_dst_stateid, &dst);
> +       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
> +                                  &copy->fh_src, &copy->cp_dst_stateid,
> +                                  &copy->fh_dst);
>         if (status)
>                 goto out;
>
> -       bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
> -                       dst, copy->cp_dst_pos, copy->cp_count);
> -
> -       if (bytes < 0)
> -               status = nfserrno(bytes);
> -       else {
> -               copy->cp_res.wr_bytes_written = bytes;
> -               copy->cp_res.wr_stable_how = NFS_UNSTABLE;
> -               copy->cp_consecutive = 1;
> -               copy->cp_synchronous = 1;
> -               gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
> -               status = nfs_ok;
> +       copy->cp_clp = cstate->clp;
> +       memcpy(&copy->fh, &cstate->current_fh.fh_handle,
> +               sizeof(struct knfsd_fh));
> +       copy->net = SVC_NET(rqstp);
> +       if (!copy->cp_synchronous) {
> +               status = nfsd4_init_copy_res(copy, 0);
> +               async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
> +               if (!async_copy) {
> +                       status = nfserrno(-ENOMEM);
> +                       goto out;
> +               }
> +               dup_copy_fields(copy, async_copy);
> +               memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
> +                       sizeof(copy->cp_dst_stateid));
> +               spin_lock(&async_copy->cp_clp->async_lock);
> +               list_add(&async_copy->copies,
> +                               &async_copy->cp_clp->async_copies);
> +               spin_unlock(&async_copy->cp_clp->async_lock);
> +               async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
> +                               async_copy, "%s", "copy thread");
> +               if (IS_ERR(async_copy->copy_task)) {
> +                       status = PTR_ERR(async_copy->copy_task);
> +                       goto out_err_dec;
> +               }
> +               wake_up_process(async_copy->copy_task);
> +       } else {
> +               status = nfsd4_do_copy(copy, 1);
>         }
> -
> -       fput(src);
> -       fput(dst);
>  out:
>         return status;
> +out_err_dec:
> +       cleanup_async_copy(async_copy);
> +       goto out;
>  }
>
>  static __be32
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 0c04f81..d7767a1 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1774,6 +1774,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
>  #ifdef CONFIG_NFSD_PNFS
>         INIT_LIST_HEAD(&clp->cl_lo_states);
>  #endif
> +       INIT_LIST_HEAD(&clp->async_copies);
> +       spin_lock_init(&clp->async_lock);
>         spin_lock_init(&clp->cl_lock);
>         rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
>         return clp;
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index f8b0210..9189062 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -352,6 +352,8 @@ struct nfs4_client {
>         struct rpc_wait_queue   cl_cb_waitq;    /* backchannel callers may */
>                                                 /* wait here for slots */
>         struct net              *net;
> +       struct list_head        async_copies;   /* list of async copies */
> +       spinlock_t              async_lock;     /* lock for async copies */
>  };
>
>  /* struct nfs4_client_reset
> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> index 9b0c099..0a19954 100644
> --- a/fs/nfsd/xdr4.h
> +++ b/fs/nfsd/xdr4.h
> @@ -529,6 +529,15 @@ struct nfsd4_copy {
>         struct nfsd4_callback   cp_cb;
>         __be32                  nfserr;
>         struct knfsd_fh         fh;
> +
> +       struct nfs4_client      *cp_clp;
> +
> +       struct file             *fh_src;
> +       struct file             *fh_dst;
> +       struct net              *net;
> +
> +       struct list_head        copies;
> +       struct task_struct      *copy_task;
>  };
>
>  struct nfsd4_seek {
> --
> 1.8.3.1
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Anna Schumaker]

On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
> On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>>> are allocated on the stack are copies for the kthread.
>>>
>>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>>> we keep the 4MB chunk and requested size for the async copy). If error is
>>> encountered it's saved but also we save the amount of data copied so far.
>>> Once done, the results are queued for the callback workqueue and sent via
>>> CB_OFFLOAD.
>>>
>>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>>> the ongoing do_splice() and once vfs returns we are choosing not to send
>>> the CB_OFFLOAD back to the client.
>>>
>>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>>> the copy and will query the i_size_read() of the associated filehandle of
>>> the destination file and return the result.
>>
>> That assumes we're copying into a previously empty file?
> 
> Sigh. Alright, then it's back to my original solution where I broke
> everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

> 
>>
>> --b.
>>
>>>
>>> v5:
>>> 1. reimplementing asynchronous copy to use kthreads instead of the workqueue
>>> 2. store asynchronous copies in the list of the nfs4_client structure
>>> 3. when copying nfsd4_copy datastructure for the async copy refcount the
>>> nfs4_client structure as well as struct file src/dst structure
>>> 4. add refcount to the nfsd4_copy structure to coordinate with offload_status
>>> and offload_cancel
>>> 5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to
>>> interrupt do_splice and calls kthread_stop to wait for the copy to stop
>>> 6. offload_cancel adds MODIFIES_SOMETHING to flags
>>> 7. offload_status reports the size of the destination file form i_size_read()
>>> instead of before 4MB loop updated chunk size.
>>> 8. for async copy call vfs_copy_file_range() with the whole copy size. still
>>> keep the loop to do more the MAX_RW_COUNT that do_splice can do.
>>>
>>> Olga Kornievskaia (10):
>>>   NFSD CB_OFFLOAD xdr
>>>   NFSD OFFLOAD_STATUS xdr
>>>   NFSD OFFLOAD_CANCEL xdr
>>>   NFSD xdr callback stateid in async COPY reply
>>>   NFSD first draft of async copy
>>>   NFSD return nfs4_stid in nfs4_preprocess_stateid_op
>>>   NFSD create new stateid for async copy
>>>   NFSD handle OFFLOAD_CANCEL op
>>>   NFSD support OFFLOAD_STATUS
>>>   NFSD stop queued async copies on client shutdown
>>>
>>>  fs/nfsd/netns.h        |   8 ++
>>>  fs/nfsd/nfs4callback.c |  97 +++++++++++++++
>>>  fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
>>>  fs/nfsd/nfs4state.c    |  77 +++++++++++-
>>>  fs/nfsd/nfs4xdr.c      |  50 ++++++--
>>>  fs/nfsd/nfsctl.c       |   1 +
>>>  fs/nfsd/state.h        |  21 +++-
>>>  fs/nfsd/xdr4.h         |  28 +++++
>>>  fs/nfsd/xdr4cb.h       |  10 ++
>>>  9 files changed, 576 insertions(+), 37 deletions(-)
>>>
>>> --
>>> 1.8.3.1
>>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Mauricio Tavares]

On Mon, Oct 16, 2017 at 9:13 AM, Anna Schumaker
<Anna.Schumaker@netapp.com> wrote:
>
>
> On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
>> On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>>> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>>>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>>>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>>>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>>>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>>>> are allocated on the stack are copies for the kthread.
>>>>
>>>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>>>> we keep the 4MB chunk and requested size for the async copy). If error is
>>>> encountered it's saved but also we save the amount of data copied so far.
>>>> Once done, the results are queued for the callback workqueue and sent via
>>>> CB_OFFLOAD.
>>>>
>>>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>>>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>>>> the ongoing do_splice() and once vfs returns we are choosing not to send
>>>> the CB_OFFLOAD back to the client.
>>>>
>>>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>>>> the copy and will query the i_size_read() of the associated filehandle of
>>>> the destination file and return the result.
>>>
>>> That assumes we're copying into a previously empty file?
>>
>> Sigh. Alright, then it's back to my original solution where I broke
>> everything into 4MB calls and kept track of bytes copies so far.
>
> Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.
>
      Could have that as a parameter defined in some config file perhaps?
>>
>>>
>>> --b.
>>>
>>>>
>>>> v5:
>>>> 1. reimplementing asynchronous copy to use kthreads instead of the workqueue
>>>> 2. store asynchronous copies in the list of the nfs4_client structure
>>>> 3. when copying nfsd4_copy datastructure for the async copy refcount the
>>>> nfs4_client structure as well as struct file src/dst structure
>>>> 4. add refcount to the nfsd4_copy structure to coordinate with offload_status
>>>> and offload_cancel
>>>> 5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to
>>>> interrupt do_splice and calls kthread_stop to wait for the copy to stop
>>>> 6. offload_cancel adds MODIFIES_SOMETHING to flags
>>>> 7. offload_status reports the size of the destination file form i_size_read()
>>>> instead of before 4MB loop updated chunk size.
>>>> 8. for async copy call vfs_copy_file_range() with the whole copy size. still
>>>> keep the loop to do more the MAX_RW_COUNT that do_splice can do.
>>>>
>>>> Olga Kornievskaia (10):
>>>>   NFSD CB_OFFLOAD xdr
>>>>   NFSD OFFLOAD_STATUS xdr
>>>>   NFSD OFFLOAD_CANCEL xdr
>>>>   NFSD xdr callback stateid in async COPY reply
>>>>   NFSD first draft of async copy
>>>>   NFSD return nfs4_stid in nfs4_preprocess_stateid_op
>>>>   NFSD create new stateid for async copy
>>>>   NFSD handle OFFLOAD_CANCEL op
>>>>   NFSD support OFFLOAD_STATUS
>>>>   NFSD stop queued async copies on client shutdown
>>>>
>>>>  fs/nfsd/netns.h        |   8 ++
>>>>  fs/nfsd/nfs4callback.c |  97 +++++++++++++++
>>>>  fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
>>>>  fs/nfsd/nfs4state.c    |  77 +++++++++++-
>>>>  fs/nfsd/nfs4xdr.c      |  50 ++++++--
>>>>  fs/nfsd/nfsctl.c       |   1 +
>>>>  fs/nfsd/state.h        |  21 +++-
>>>>  fs/nfsd/xdr4.h         |  28 +++++
>>>>  fs/nfsd/xdr4cb.h       |  10 ++
>>>>  9 files changed, 576 insertions(+), 37 deletions(-)
>>>>
>>>> --
>>>> 1.8.3.1
>>>>
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[J. Bruce Fields]

On Mon, Oct 16, 2017 at 09:13:20AM -0400, Anna Schumaker wrote:
> 
> 
> On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
> > On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> >> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
> >>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
> >>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
> >>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
> >>> starts the thread, and replies back to the client. nfsd4_copy arguments that
> >>> are allocated on the stack are copies for the kthread.
> >>>
> >>> In the async copy handler, it calls into VFS copy_file_range() (for synch
> >>> we keep the 4MB chunk and requested size for the async copy). If error is
> >>> encountered it's saved but also we save the amount of data copied so far.
> >>> Once done, the results are queued for the callback workqueue and sent via
> >>> CB_OFFLOAD.
> >>>
> >>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
> >>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
> >>> the ongoing do_splice() and once vfs returns we are choosing not to send
> >>> the CB_OFFLOAD back to the client.
> >>>
> >>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
> >>> the copy and will query the i_size_read() of the associated filehandle of
> >>> the destination file and return the result.
> >>
> >> That assumes we're copying into a previously empty file?
> > 
> > Sigh. Alright, then it's back to my original solution where I broke
> > everything into 4MB calls and kept track of bytes copies so far.
> 
> Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Sure, we could.  Do we have reason to believe there's an advantage to
larger sizes?

--b.

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

On Mon, Oct 16, 2017 at 12:49 PM, J. Bruce Fields <bfields@redhat.com> wrote:
> On Mon, Oct 16, 2017 at 09:13:20AM -0400, Anna Schumaker wrote:
>>
>>
>> On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
>> > On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>> >> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>> >>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>> >>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>> >>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>> >>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>> >>> are allocated on the stack are copies for the kthread.
>> >>>
>> >>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>> >>> we keep the 4MB chunk and requested size for the async copy). If error is
>> >>> encountered it's saved but also we save the amount of data copied so far.
>> >>> Once done, the results are queued for the callback workqueue and sent via
>> >>> CB_OFFLOAD.
>> >>>
>> >>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>> >>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>> >>> the ongoing do_splice() and once vfs returns we are choosing not to send
>> >>> the CB_OFFLOAD back to the client.
>> >>>
>> >>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>> >>> the copy and will query the i_size_read() of the associated filehandle of
>> >>> the destination file and return the result.
>> >>
>> >> That assumes we're copying into a previously empty file?
>> >
>> > Sigh. Alright, then it's back to my original solution where I broke
>> > everything into 4MB calls and kept track of bytes copies so far.
>>
>> Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.
>
> Sure, we could.  Do we have reason to believe there's an advantage to
> larger sizes?

I wouldn't think there'd be a large enough performance advantage with
a larger size and there'd be worse OFFLOAD_STATUS information. I'm
sure there is a setup cost for calling into do_splice() and the cost
of doing a function call but I'd like they would be small.

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

Hi Bruce,

You were asking for performance numbers for asynchronous vs
synchronous intra copy. Here's what Jorge reports:

This is using RHEL 7.4 on both the client and server with COMMIT on
the same compound as the COPY for the synchronous case.
In this case, improvement is achieved for copies larger than 16MB.



/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
1KB file: 0.218760585785 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
1KB file: 0.636984395981 seconds
                                                         65.66%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
2KB file: 0.22707760334 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
2KB file: 0.583548688889 seconds
                                                         61.09%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
4KB file: 0.234200882912 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
4KB file: 0.782712388039 seconds
                                                         70.08%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
8KB file: 0.214556503296 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
8KB file: 0.692702102661 seconds
                                                         69.03%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
16KB file: 0.215230226517 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
16KB file: 0.56289691925 seconds
                                                         61.76%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
32KB file: 0.186200523376 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
32KB file: 0.65691485405 seconds
                                                         71.66%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
64KB file: 0.233846497536 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
64KB file: 0.525265741348 seconds
                                                         55.48%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
128KB file: 0.198684954643 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
128KB file: 0.69602959156 seconds
                                                         71.45%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
256KB file: 0.211255192757 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
256KB file: 0.556627941132 seconds
                                                         62.05%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
512KB file: 0.218777489662 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
512KB file: 0.496951031685 seconds
                                                         55.98%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
1MB file: 0.179558849335 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
1MB file: 0.50447602272 seconds
                                                         64.41%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
2MB file: 0.252070856094 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
2MB file: 0.570275163651 seconds
                                                         55.80%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
4MB file: 0.289573478699 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
4MB file: 0.656079149246 seconds
                                                         55.86%
performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
8MB file: 0.50943710804 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
8MB file: 0.696055078506 seconds
                                                         26.81%
performance degradation by async

Performance Improvement:
/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
16MB file: 0.920844507217 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
16MB file: 0.817601919174 seconds
                                                         11.21%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
32MB file: 1.46817543507 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
32MB file: 1.24578406811 seconds
                                                         15.15%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
64MB file: 2.42379112244 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
64MB file: 1.58639280796 seconds
                                                         34.55%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
128MB file: 4.16012530327 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
128MB file: 2.58433949947 seconds
                                                         37.88%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
256MB file: 7.56400749683 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
256MB file: 4.43859291077 seconds
                                                         41.32%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
512MB file: 14.5191983461 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
512MB file: 8.18448216915 seconds
                                                         43.63%
performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
1GB file: 28.7398069143 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
1GB file: 16.1399238825 seconds
                                                         43.84%
performance improvement by async

On Mon, Oct 16, 2017 at 3:25 PM, Olga Kornievskaia <aglo@umich.edu> wrote:
> On Mon, Oct 16, 2017 at 12:49 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>> On Mon, Oct 16, 2017 at 09:13:20AM -0400, Anna Schumaker wrote:
>>>
>>>
>>> On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
>>> > On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>>> >> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>>> >>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>>> >>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>>> >>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>>> >>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>>> >>> are allocated on the stack are copies for the kthread.
>>> >>>
>>> >>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>>> >>> we keep the 4MB chunk and requested size for the async copy). If error is
>>> >>> encountered it's saved but also we save the amount of data copied so far.
>>> >>> Once done, the results are queued for the callback workqueue and sent via
>>> >>> CB_OFFLOAD.
>>> >>>
>>> >>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>>> >>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>>> >>> the ongoing do_splice() and once vfs returns we are choosing not to send
>>> >>> the CB_OFFLOAD back to the client.
>>> >>>
>>> >>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>>> >>> the copy and will query the i_size_read() of the associated filehandle of
>>> >>> the destination file and return the result.
>>> >>
>>> >> That assumes we're copying into a previously empty file?
>>> >
>>> > Sigh. Alright, then it's back to my original solution where I broke
>>> > everything into 4MB calls and kept track of bytes copies so far.
>>>
>>> Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.
>>
>> Sure, we could.  Do we have reason to believe there's an advantage to
>> larger sizes?
>
> I wouldn't think there'd be a large enough performance advantage with
> a larger size and there'd be worse OFFLOAD_STATUS information. I'm
> sure there is a setup cost for calling into do_splice() and the cost
> of doing a function call but I'd like they would be small.

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[Olga Kornievskaia]

Hi Bruce,

Do you have any more comments on the current async patches or should I
post a new version? I plan to do the original OFFLOAD_STATUS looping
and I'm planning to change server to not send the EINVAL error to the
client and instead send a partial copy and 0 when reading past the end
of the file.

Thanks.

On Mon, Oct 23, 2017 at 5:48 PM, Olga Kornievskaia <aglo@umich.edu> wrote:
> Hi Bruce,
>
> You were asking for performance numbers for asynchronous vs
> synchronous intra copy. Here's what Jorge reports:
>
> This is using RHEL 7.4 on both the client and server with COMMIT on
> the same compound as the COPY for the synchronous case.
> In this case, improvement is achieved for copies larger than 16MB.
>
>
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 1KB file: 0.218760585785 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 1KB file: 0.636984395981 seconds
>                                                          65.66%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 2KB file: 0.22707760334 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 2KB file: 0.583548688889 seconds
>                                                          61.09%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 4KB file: 0.234200882912 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 4KB file: 0.782712388039 seconds
>                                                          70.08%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 8KB file: 0.214556503296 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 8KB file: 0.692702102661 seconds
>                                                          69.03%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 16KB file: 0.215230226517 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 16KB file: 0.56289691925 seconds
>                                                          61.76%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 32KB file: 0.186200523376 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 32KB file: 0.65691485405 seconds
>                                                          71.66%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 64KB file: 0.233846497536 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 64KB file: 0.525265741348 seconds
>                                                          55.48%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 128KB file: 0.198684954643 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 128KB file: 0.69602959156 seconds
>                                                          71.45%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 256KB file: 0.211255192757 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 256KB file: 0.556627941132 seconds
>                                                          62.05%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 512KB file: 0.218777489662 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 512KB file: 0.496951031685 seconds
>                                                          55.98%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 1MB file: 0.179558849335 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 1MB file: 0.50447602272 seconds
>                                                          64.41%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 2MB file: 0.252070856094 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 2MB file: 0.570275163651 seconds
>                                                          55.80%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 4MB file: 0.289573478699 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 4MB file: 0.656079149246 seconds
>                                                          55.86%
> performance degradation by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 8MB file: 0.50943710804 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 8MB file: 0.696055078506 seconds
>                                                          26.81%
> performance degradation by async
>
> Performance Improvement:
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 16MB file: 0.920844507217 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 16MB file: 0.817601919174 seconds
>                                                          11.21%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 32MB file: 1.46817543507 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 32MB file: 1.24578406811 seconds
>                                                          15.15%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 64MB file: 2.42379112244 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 64MB file: 1.58639280796 seconds
>                                                          34.55%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 128MB file: 4.16012530327 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 128MB file: 2.58433949947 seconds
>                                                          37.88%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 256MB file: 7.56400749683 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 256MB file: 4.43859291077 seconds
>                                                          41.32%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 512MB file: 14.5191983461 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 512MB file: 8.18448216915 seconds
>                                                          43.63%
> performance improvement by async
>
> /home/mora/logs/nfstest_ssc_20171022201303.log:    PASS: SSC copy of
> 1GB file: 28.7398069143 seconds
> /home/mora/logs/nfstest_ssc_20171015171323.log:    PASS: SSC copy of
> 1GB file: 16.1399238825 seconds
>                                                          43.84%
> performance improvement by async
>
> On Mon, Oct 16, 2017 at 3:25 PM, Olga Kornievskaia <aglo@umich.edu> wrote:
>> On Mon, Oct 16, 2017 at 12:49 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>>> On Mon, Oct 16, 2017 at 09:13:20AM -0400, Anna Schumaker wrote:
>>>>
>>>>
>>>> On 10/13/2017 08:09 PM, Olga Kornievskaia wrote:
>>>> > On Fri, Oct 13, 2017 at 5:26 PM, J. Bruce Fields <bfields@redhat.com> wrote:
>>>> >> On Fri, Oct 13, 2017 at 04:54:02PM -0400, Olga Kornievskaia wrote:
>>>> >>> To do asynchronous copies, NFSD creates a new kthread to handle the request.
>>>> >>> Upon receiving the COPY, it generates a unique copy stateid (stored in a
>>>> >>> global list for keeping track of state for OFFLOAD_STATUS to be queried by),
>>>> >>> starts the thread, and replies back to the client. nfsd4_copy arguments that
>>>> >>> are allocated on the stack are copies for the kthread.
>>>> >>>
>>>> >>> In the async copy handler, it calls into VFS copy_file_range() (for synch
>>>> >>> we keep the 4MB chunk and requested size for the async copy). If error is
>>>> >>> encountered it's saved but also we save the amount of data copied so far.
>>>> >>> Once done, the results are queued for the callback workqueue and sent via
>>>> >>> CB_OFFLOAD.
>>>> >>>
>>>> >>> When the server received an OFFLOAD_CANCEL, it will find the kthread running
>>>> >>> the copy and will send a SIGPENDING and kthread_stop() and it will interrupt
>>>> >>> the ongoing do_splice() and once vfs returns we are choosing not to send
>>>> >>> the CB_OFFLOAD back to the client.
>>>> >>>
>>>> >>> When the server receives an OFFLOAD_STATUS, it will find the kthread running
>>>> >>> the copy and will query the i_size_read() of the associated filehandle of
>>>> >>> the destination file and return the result.
>>>> >>
>>>> >> That assumes we're copying into a previously empty file?
>>>> >
>>>> > Sigh. Alright, then it's back to my original solution where I broke
>>>> > everything into 4MB calls and kept track of bytes copies so far.
>>>>
>>>> Do they have to be 4MB calls?  Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.
>>>
>>> Sure, we could.  Do we have reason to believe there's an advantage to
>>> larger sizes?
>>
>> I wouldn't think there'd be a large enough performance advantage with
>> a larger size and there'd be worse OFFLOAD_STATUS information. I'm
>> sure there is a setup cost for calling into do_splice() and the cost
>> of doing a function call but I'd like they would be small.

Re: [PATCH v5 00/10] NFSD support for asynchronous COPY

[J. Bruce Fields]

On Mon, Oct 23, 2017 at 06:39:54PM -0400, Olga Kornievskaia wrote:
> Do you have any more comments on the current async patches or should I
> post a new version? I plan to do the original OFFLOAD_STATUS looping
> and I'm planning to change server to not send the EINVAL error to the
> client and instead send a partial copy and 0 when reading past the end
> of the file.

That sounds good, I'm afraid I don't have any pending comments, so
resending is fine.

--b.