From: Olga Kornievskaia <olga.kornievskaia@gmail.com>
To: trond.myklebust@hammerspace.com, anna.schumaker@netapp.com
Cc: linux-nfs@vger.kernel.org
Subject: [PATCH v9 10/11] NFS: handle source server reboot
Date: Fri, 14 Jun 2019 16:00:15 -0400 [thread overview]
Message-ID: <20190614200016.12348-11-olga.kornievskaia@gmail.com> (raw)
In-Reply-To: <20190614200016.12348-1-olga.kornievskaia@gmail.com>
From: Olga Kornievskaia <kolga@netapp.com>
When the source server reboots after a server-to-server copy was
issued, we need to retry the copy from COPY_NOTIFY. We need to
detect that the source server rebooted and there is a copy waiting
on a destination server and wake it up.
Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
---
fs/nfs/nfs42proc.c | 68 ++++++++++++++++++++++++++++++++++----------------
fs/nfs/nfs4_fs.h | 1 +
fs/nfs/nfs4file.c | 3 +++
fs/nfs/nfs4state.c | 26 +++++++++++++++----
include/linux/nfs_fs.h | 4 ++-
5 files changed, 75 insertions(+), 27 deletions(-)
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index 705fe69..a21f01a 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -153,22 +153,26 @@ int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len)
}
static int handle_async_copy(struct nfs42_copy_res *res,
- struct nfs_server *server,
+ struct nfs_server *dst_server,
+ struct nfs_server *src_server,
struct file *src,
struct file *dst,
- nfs4_stateid *src_stateid)
+ nfs4_stateid *src_stateid,
+ bool *restart)
{
struct nfs4_copy_state *copy, *tmp_copy;
int status = NFS4_OK;
bool found_pending = false;
- struct nfs_open_context *ctx = nfs_file_open_context(dst);
+ struct nfs_open_context *dst_ctx = nfs_file_open_context(dst);
+ struct nfs_open_context *src_ctx = nfs_file_open_context(src);
copy = kzalloc(sizeof(struct nfs4_copy_state), GFP_NOFS);
if (!copy)
return -ENOMEM;
- spin_lock(&server->nfs_client->cl_lock);
- list_for_each_entry(tmp_copy, &server->nfs_client->pending_cb_stateids,
+ spin_lock(&dst_server->nfs_client->cl_lock);
+ list_for_each_entry(tmp_copy,
+ &dst_server->nfs_client->pending_cb_stateids,
copies) {
if (memcmp(&res->write_res.stateid, &tmp_copy->stateid,
NFS4_STATEID_SIZE))
@@ -178,7 +182,7 @@ static int handle_async_copy(struct nfs42_copy_res *res,
break;
}
if (found_pending) {
- spin_unlock(&server->nfs_client->cl_lock);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
kfree(copy);
copy = tmp_copy;
goto out;
@@ -186,19 +190,32 @@ static int handle_async_copy(struct nfs42_copy_res *res,
memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE);
init_completion(©->completion);
- copy->parent_state = ctx->state;
+ copy->parent_dst_state = dst_ctx->state;
+ copy->parent_src_state = src_ctx->state;
+
+ list_add_tail(©->copies, &dst_server->ss_copies);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
- list_add_tail(©->copies, &server->ss_copies);
- spin_unlock(&server->nfs_client->cl_lock);
+ if (dst_server != src_server) {
+ spin_lock(&src_server->nfs_client->cl_lock);
+ list_add_tail(©->src_copies, &src_server->ss_copies);
+ spin_unlock(&src_server->nfs_client->cl_lock);
+ }
status = wait_for_completion_interruptible(©->completion);
- spin_lock(&server->nfs_client->cl_lock);
+ spin_lock(&dst_server->nfs_client->cl_lock);
list_del_init(©->copies);
- spin_unlock(&server->nfs_client->cl_lock);
+ spin_unlock(&dst_server->nfs_client->cl_lock);
+ if (dst_server != src_server) {
+ spin_lock(&src_server->nfs_client->cl_lock);
+ list_del_init(©->src_copies);
+ spin_unlock(&src_server->nfs_client->cl_lock);
+ }
if (status == -ERESTARTSYS) {
goto out_cancel;
- } else if (copy->flags) {
+ } else if (copy->flags || copy->error == NFS4ERR_PARTNER_NO_AUTH) {
status = -EAGAIN;
+ *restart = true;
goto out_cancel;
}
out:
@@ -247,7 +264,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
struct nfs42_copy_args *args,
struct nfs42_copy_res *res,
struct nl4_server *nss,
- nfs4_stateid *cnr_stateid)
+ nfs4_stateid *cnr_stateid,
+ bool *restart)
{
struct rpc_message msg = {
.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY],
@@ -255,7 +273,9 @@ static ssize_t _nfs42_proc_copy(struct file *src,
.rpc_resp = res,
};
struct inode *dst_inode = file_inode(dst);
- struct nfs_server *server = NFS_SERVER(dst_inode);
+ struct inode *src_inode = file_inode(src);
+ struct nfs_server *dst_server = NFS_SERVER(dst_inode);
+ struct nfs_server *src_server = NFS_SERVER(src_inode);
loff_t pos_src = args->src_pos;
loff_t pos_dst = args->dst_pos;
size_t count = args->count;
@@ -291,13 +311,15 @@ static ssize_t _nfs42_proc_copy(struct file *src,
if (!res->commit_res.verf)
return -ENOMEM;
}
+ set_bit(NFS_CLNT_SRC_SSC_COPY_STATE,
+ &src_lock->open_context->state->flags);
set_bit(NFS_CLNT_DST_SSC_COPY_STATE,
&dst_lock->open_context->state->flags);
- status = nfs4_call_sync(server->client, server, &msg,
+ status = nfs4_call_sync(dst_server->client, dst_server, &msg,
&args->seq_args, &res->seq_res, 0);
if (status == -ENOTSUPP)
- server->caps &= ~NFS_CAP_COPY;
+ dst_server->caps &= ~NFS_CAP_COPY;
if (status)
goto out;
@@ -309,8 +331,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
}
if (!res->synchronous) {
- status = handle_async_copy(res, server, src, dst,
- &args->src_stateid);
+ status = handle_async_copy(res, dst_server, src_server, src,
+ dst, &args->src_stateid, restart);
if (status)
return status;
}
@@ -358,6 +380,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
.stateid = &args.dst_stateid,
};
ssize_t err, err2;
+ bool restart = false;
src_lock = nfs_get_lock_context(nfs_file_open_context(src));
if (IS_ERR(src_lock))
@@ -378,7 +401,7 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
err = _nfs42_proc_copy(src, src_lock,
dst, dst_lock,
&args, &res,
- nss, cnr_stateid);
+ nss, cnr_stateid, &restart);
inode_unlock(file_inode(dst));
if (err >= 0)
@@ -387,8 +410,11 @@ ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src,
err = -EOPNOTSUPP;
break;
} else if (err == -EAGAIN) {
- dst_exception.retry = 1;
- continue;
+ if (!restart) {
+ dst_exception.retry = 1;
+ continue;
+ }
+ break;
} else if (err == -NFS4ERR_OFFLOAD_NO_REQS && !args.sync) {
args.sync = true;
dst_exception.retry = 1;
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index d49fc19..2cb5f50 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -168,6 +168,7 @@ enum {
NFS_STATE_CHANGE_WAIT, /* A state changing operation is outstanding */
#ifdef CONFIG_NFS_V4_2
NFS_CLNT_DST_SSC_COPY_STATE, /* dst server open state on client*/
+ NFS_CLNT_SRC_SSC_COPY_STATE, /* src server open state on client*/
NFS_SRV_SSC_COPY_STATE, /* ssc state on the dst server */
#endif /* CONFIG_NFS_V4_2 */
};
diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
index 5ef3c12..3bfa041 100644
--- a/fs/nfs/nfs4file.c
+++ b/fs/nfs/nfs4file.c
@@ -146,6 +146,7 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
return -EOPNOTSUPP;
if (file_inode(file_in) == file_inode(file_out))
return -EOPNOTSUPP;
+retry:
if (!nfs42_files_from_same_server(file_in, file_out)) {
cn_resp = kzalloc(sizeof(struct nfs42_copy_notify_res),
GFP_NOFS);
@@ -164,6 +165,8 @@ static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
nss, cnrs);
out:
kfree(cn_resp);
+ if (ret == -EAGAIN)
+ goto retry;
return ret;
}
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 045af56..329e3ff 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1555,16 +1555,32 @@ static void nfs42_complete_copies(struct nfs4_state_owner *sp, struct nfs4_state
{
struct nfs4_copy_state *copy;
- if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags))
+ if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
+ !test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags))
return;
spin_lock(&sp->so_server->nfs_client->cl_lock);
list_for_each_entry(copy, &sp->so_server->ss_copies, copies) {
- if (!nfs4_stateid_match_other(&state->stateid, ©->parent_state->stateid))
- continue;
+ if ((test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) &&
+ !nfs4_stateid_match_other(&state->stateid,
+ ©->parent_dst_state->stateid)))
+ continue;
copy->flags = 1;
- complete(©->completion);
- break;
+ if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
+ &state->flags)) {
+ clear_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags);
+ complete(©->completion);
+ }
+ }
+ list_for_each_entry(copy, &sp->so_server->ss_copies, src_copies) {
+ if ((test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags) &&
+ !nfs4_stateid_match_other(&state->stateid,
+ ©->parent_src_state->stateid)))
+ continue;
+ copy->flags = 1;
+ if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE,
+ &state->flags))
+ complete(©->completion);
}
spin_unlock(&sp->so_server->nfs_client->cl_lock);
}
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index d363d57..8df5130 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -189,13 +189,15 @@ struct nfs_inode {
struct nfs4_copy_state {
struct list_head copies;
+ struct list_head src_copies;
nfs4_stateid stateid;
struct completion completion;
uint64_t count;
struct nfs_writeverf verf;
int error;
int flags;
- struct nfs4_state *parent_state;
+ struct nfs4_state *parent_src_state;
+ struct nfs4_state *parent_dst_state;
};
/*
--
1.8.3.1
next prev parent reply other threads:[~2019-06-14 20:00 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-06-14 20:00 [PATCH v9 00/11] client-side support for "inter" SSC copy Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 01/11] NFS NFSD: defining nl4_servers structure needed by both Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 02/11] NFS: add COPY_NOTIFY operation Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 03/11] NFS: add ca_source_server<> to COPY Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 04/11] NFS: also send OFFLOAD_CANCEL to source server Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 05/11] NFS: inter ssc open Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 06/11] NFS: skip recovery of copy open on dest server Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 07/11] NFS: for "inter" copy treat ESTALE as ENOTSUPP Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 08/11] NFS: COPY handle ERR_OFFLOAD_DENIED Olga Kornievskaia
2019-06-14 20:00 ` [PATCH v9 09/11] NFS handle NFS4ERR_PARTNER_NO_AUTH error Olga Kornievskaia
2019-06-14 20:00 ` Olga Kornievskaia [this message]
2019-06-14 20:00 ` [PATCH v9 11/11] NFS: replace cross device check in copy_file_range Olga Kornievskaia
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190614200016.12348-11-olga.kornievskaia@gmail.com \
--to=olga.kornievskaia@gmail.com \
--cc=anna.schumaker@netapp.com \
--cc=linux-nfs@vger.kernel.org \
--cc=trond.myklebust@hammerspace.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).