From: Olga Kornievskaia <olga.kornievskaia@gmail.com>
To: trond.myklebust@hammerspace.com, anna.schumaker@netapp.com,
bfields@redhat.com
Cc: linux-nfs@vger.kernel.org
Subject: [PATCH v6 09/19] NFS: handle source server reboot
Date: Fri, 6 Sep 2019 15:46:21 -0400 [thread overview]
Message-ID: <20190906194631.3216-10-olga.kornievskaia@gmail.com> (raw)
In-Reply-To: <20190906194631.3216-1-olga.kornievskaia@gmail.com>
From: Olga Kornievskaia <kolga@netapp.com>
When the source server reboots after a server-to-server copy was
issued, we need to retry the copy from COPY_NOTIFY. We need to
detect that the source server rebooted and there is a copy waiting
on a destination server and wake it up.
Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
fs/nfs/nfs42proc.c | 68 ++++++++++++++++++++++++++++++++++----------------
fs/nfs/nfs4_fs.h | 1 +
fs/nfs/nfs4file.c | 3 +++
fs/nfs/nfs4state.c | 15 ++++++++---
include/linux/nfs_fs.h | 3 ++-
5 files changed, 64 insertions(+), 26 deletions(-)
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index 705fe69..918dd78 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -153,22 +153,26 @@ int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len)
}
static int handle_async_copy(struct nfs42_copy_res *res,
- struct nfs_server *server,
+ struct nfs_server *dst_server,
+ struct nfs_server *src_server,
struct file *src,
struct file *dst,
- nfs4_stateid *src_stateid)
+ nfs4_stateid *src_stateid,
+ bool *restart)
{
struct nfs4_copy_state *copy, *tmp_copy;
int status = NFS4_OK;
bool found_pending = false;
- struct nfs_open_context *ctx = nfs_file_open_context(dst);
+ struct nfs_open_context *dst_ctx = nfs_file_open_context(dst);
+ struct nfs_open_context *src_ctx = nfs_file_open_context(src);
copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS);
if (!copy)
return -ENOMEM;
- spin_lock(&server->nfs_client->cl_lock);
- list_for_each_entry(tmp_copy, &server->nfs_client->pending_cb_stateids,
+ spin_lock(&dst_server->nfs_client->cl_lock);
+ list_for_each_entry(tmp_copy,
+ &dst_server->nfs_client->pending_cb_stateids,
copies) {
if (memcmp(&res->write_res.stateid, &tmp_copy->stateid,
NFS4_STATEID_SIZE))
@@ -178,7 +182,7 @@ static int handle_async_copy(struct nfs42_copy_res *res,
break;
}
if (found_pending) {
- spin_unlock(&server->nfs_client->cl_lock);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
kfree(copy);
copy = tmp_copy;
goto out;
@@ -186,19 +190,32 @@ static int handle_async_copy(struct nfs42_copy_res *res,
memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE);
init_completion(©->completion);
- copy->parent_state = ctx->state;
+ copy->parent_dst_state = dst_ctx->state;
+ copy->parent_src_state = src_ctx->state;
+
+ list_add_tail(©->copies, &dst_server->ss_copies);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
- list_add_tail(©->copies, &server->ss_copies);
- spin_unlock(&server->nfs_client->cl_lock);
+ if (dst_server != src_server) {
+ spin_lock(&src_server->nfs_client->cl_lock);
+ list_add_tail(©->copies, &src_server->ss_copies);
+ spin_unlock(&src_server->nfs_client->cl_lock);
+ }
status = wait_for_completion_interruptible(©->completion);
- spin_lock(&server->nfs_client->cl_lock);
+ spin_lock(&dst_server->nfs_client->cl_lock);
list_del_init(©->copies);
- spin_unlock(&server->nfs_client->cl_lock);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
+ if (dst_server != src_server) {
+ spin_lock(&src_server->nfs_client->cl_lock);
+ list_del_init(©->copies);
+ spin_unlock(&src_server->nfs_client->cl_lock);
+ }
if (status == -ERESTARTSYS) {
goto out_cancel;
- } else if (copy->flags) {
+ } else if (copy->flags && copy->error) {
status = -EAGAIN;
+ *restart = true;
goto out_cancel;
}
out:
@@ -247,7 +264,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
struct nfs42_copy_args *args,
struct nfs42_copy_res *res,
struct nl4_server *nss,
- nfs4_stateid *cnr_stateid)
+ nfs4_stateid *cnr_stateid,
+ bool *restart)
{
struct rpc_message msg = {
.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY],
@@ -255,7 +273,9 @@ static ssize_t _nfs42_proc_copy(struct file *src,
.rpc_resp = res,
};
struct inode *dst_inode = file_inode(dst);
- struct nfs_server *server = NFS_SERVER(dst_inode);
+ struct inode *src_inode = file_inode(src);
+ struct nfs_server *dst_server = NFS_SERVER(dst_inode);
+ struct nfs_server *src_server = NFS_SERVER(src_inode);
loff_t pos_src = args->src_pos;
loff_t pos_dst = args->dst_pos;
size_t count = args->count;
@@ -291,13 +311,15 @@ static ssize_t _nfs42_proc_copy(struct file *src,
if (!res->commit_res.verf)
return -ENOMEM;
}
+ set_bit(NFS_CLNT_SRC_SSC_COPY_STATE,
+ &src_lock->open_context->state->flags);
set_bit(NFS_CLNT_DST_SSC_COPY_STATE,
&dst_lock->open_context->state->flags);
- status = nfs4_call_sync(server->client, server, &msg,
+ status = nfs4_call_sync(dst_server->client, dst_server, &msg,
&args->seq_args, &res->seq_res, 0);
if (status == -ENOTSUPP)
- server->caps &= ~NFS_CAP_COPY;
+ dst_server->caps &= ~NFS_CAP_COPY;
if (status)
goto out;
@@ -309,8 +331,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
}
if (!res->synchronous) {
- status = handle_async_copy(res, server, src, dst,
- &args->src_stateid);
+ status = handle_async_copy(res, dst_server, src_server, src,
+ dst, &args->src_stateid, restart);
if (status)
return status;
}
@@ -358,6 +380,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
.stateid = &args.dst_stateid,
};
ssize_t err, err2;
+ bool restart = false;
src_lock = nfs_get_lock_context(nfs_file_open_context(src));
if (IS_ERR(src_lock))
@@ -378,7 +401,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
err = _nfs42_proc_copy(src, src_lock,
dst, dst_lock,
&args, &res,
- nss, cnr_stateid);
+ nss, cnr_stateid, &restart);
inode_unlock(file_inode(dst));
if (err >= 0)
@@ -387,8 +410,11 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
err = -EOPNOTSUPP;
break;
} else if (err == -EAGAIN) {
- dst_exception.retry = 1;
- continue;
+ if (!restart) {
+ dst_exception.retry = 1;
+ continue;
+ }
+ break;
} else if (err == -NFS4ERR_OFFLOAD_NO_REQS && !args.sync) {
args.sync = true;
dst_exception.retry = 1;
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 814674f..2122748 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -168,6 +168,7 @@ enum {
NFS_STATE_CHANGE_WAIT, /* A state changing operation is outstanding */
#ifdef CONFIG_NFS_V4_2
NFS_CLNT_DST_SSC_COPY_STATE, /* dst server open state on client*/
+ NFS_CLNT_SRC_SSC_COPY_STATE, /* src server open state on client*/
NFS_SRV_SSC_COPY_STATE, /* ssc state on the dst server */
#endif /* CONFIG_NFS_V4_2 */
};
diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
index a932fc9..2af30b7 100644
--- a/fs/nfs/nfs4file.c
+++ b/fs/nfs/nfs4file.c
@@ -146,6 +146,7 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
return -EOPNOTSUPP;
if (file_inode(file_in) == file_inode(file_out))
return -EOPNOTSUPP;
+retry:
if (!nfs42_files_from_same_server(file_in, file_out)) {
cn_resp = kzalloc(sizeof(struct nfs42_copy_notify_res),
GFP_NOFS);
@@ -164,6 +165,8 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
nss, cnrs);
out:
kfree(cn_resp);
+ if (ret == -EAGAIN)
+ goto retry;
return ret;
}
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index f205037..ad8bc31 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1556,15 +1556,22 @@ static void nfs42_complete_copies(struct nfs4_state_owner *sp, struct nfs4_state
{
struct nfs4_copy_state *copy;
- if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags))
+ if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
+ !test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags))
return;
spin_lock(&sp->so_server->nfs_client->cl_lock);
list_for_each_entry(copy, &sp->so_server->ss_copies, copies) {
- if (!nfs4_stateid_match_other(&state->stateid, ©->parent_state->stateid))
- continue;
+ if ((test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags) &&
+ !nfs4_stateid_match_other(&state->stateid,
+ ©->parent_src_state->stateid)) ||
+ (test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
+ !nfs4_stateid_match_other(&state->stateid,
+ ©->parent_dst_state->stateid)))
+ continue;
copy->flags = 1;
- complete(©->completion);
+ if (test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags))
+ complete(©->completion);
break;
}
spin_unlock(&sp->so_server->nfs_client->cl_lock);
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 0a11712..6ff1ec7 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -195,7 +195,8 @@ struct nfs4_copy_state {
struct nfs_writeverf verf;
int error;
int flags;
- struct nfs4_state *parent_state;
+ struct nfs4_state *parent_src_state;
+ struct nfs4_state *parent_dst_state;
};
/*
--
1.8.3.1
next prev parent reply other threads:[~2019-09-06 19:46 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-09-06 19:46 [PATCH v6 00/19] client and server support for "inter" SSC copy Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 01/19] NFS NFSD: defining nl4_servers structure needed by both Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 02/19] NFS: add COPY_NOTIFY operation Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 03/19] NFS: add ca_source_server<> to COPY Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 04/19] NFS: also send OFFLOAD_CANCEL to source server Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 05/19] NFS: inter ssc open Olga Kornievskaia
2019-09-12 20:23 ` J. Bruce Fields
2019-09-12 22:51 ` Olga Kornievskaia
2019-09-13 0:29 ` J. Bruce Fields
2019-09-06 19:46 ` [PATCH v6 06/19] NFS: skip recovery of copy open on dest server Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 07/19] NFS: for "inter" copy treat ESTALE as ENOTSUPP Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 08/19] NFS: COPY handle ERR_OFFLOAD_DENIED Olga Kornievskaia
2019-09-06 19:46 ` Olga Kornievskaia [this message]
2019-09-06 19:46 ` [PATCH v6 10/19] NFS: replace cross device check in copy_file_range Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 11/19] NFSD fill-in netloc4 structure Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 12/19] NFSD add ca_source_server<> to COPY Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 13/19] NFSD return nfs4_stid in nfs4_preprocess_stateid_op Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 14/19] NFSD COPY_NOTIFY xdr Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 15/19] NFSD add COPY_NOTIFY operation Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 16/19] NFSD check stateids against copy stateids Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 17/19] NFSD generalize nfsd4_compound_state flag names Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 18/19] NFSD: allow inter server COPY to have a STALE source server fh Olga Kornievskaia
2019-09-06 19:46 ` [PATCH v6 19/19] NFSD add nfs4 inter ssc to nfsd4_copy Olga Kornievskaia
2019-09-06 20:27 ` [PATCH v6 00/19] client and server support for "inter" SSC copy Olga Kornievskaia
2019-09-12 20:26 ` J. Bruce Fields
2019-09-12 22:52 ` Olga Kornievskaia
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190906194631.3216-10-olga.kornievskaia@gmail.com \
--to=olga.kornievskaia@gmail.com \
--cc=anna.schumaker@netapp.com \
--cc=bfields@redhat.com \
--cc=linux-nfs@vger.kernel.org \
--cc=trond.myklebust@hammerspace.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).