From: Chuck Lever <chuck.lever@oracle.com>
To: trond.myklebust@netapp.com
Cc: linux-nfs@vger.kernel.org
Subject: [PATCH 13/16] NFS: Add basic migration support to state manager thread
Date: Mon, 09 May 2011 15:38:23 -0400 [thread overview]
Message-ID: <20110509193822.16568.2005.stgit@matisse.1015granger.net> (raw)
In-Reply-To: <20110509192522.16568.59082.stgit@matisse.1015granger.net>
Migration recovery will be handled separately from the normal
synchronous and asynchronous NFS processes, much like the existing
state manager thread. In fact state and migration recovery will
have to be serialized.
Therefore add migration recovery support to the existing state manager
infrastructure, reusing its rendevous mechanism and finite state
machine.
Additional debugging is added so that, while we continue to shape our
migration recovery implementation, the operation of the state manager
is visible. If the extra clutter is objectionable, it can be removed
once we are confident of the migration recovery implementation.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
fs/nfs/nfs4_fs.h | 3 +
fs/nfs/nfs4state.c | 149 ++++++++++++++++++++++++++++++++++++++++++++-
include/linux/nfs_fs_sb.h | 3 +
3 files changed, 153 insertions(+), 2 deletions(-)
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 1832fd6..c3e8641 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -49,6 +49,8 @@ enum nfs4_client_state {
NFS4CLNT_RECALL_SLOT,
NFS4CLNT_LEASE_CONFIRM,
NFS4CLNT_UPDATE_CALLBACK,
+ NFS4CLNT_CLONED_CLIENT,
+ NFS4CLNT_MOVED,
};
enum nfs4_session_state {
@@ -348,6 +350,7 @@ extern void nfs4_close_state(struct path *, struct nfs4_state *, fmode_t);
extern void nfs4_close_sync(struct path *, struct nfs4_state *, fmode_t);
extern void nfs4_state_set_mode_locked(struct nfs4_state *, fmode_t);
extern void nfs4_schedule_lease_recovery(struct nfs_client *);
+extern void nfs4_schedule_migration_recovery(struct nfs_server *);
extern void nfs4_schedule_state_manager(struct nfs_client *);
extern void nfs4_schedule_stateid_recovery(const struct nfs_server *, struct nfs4_state *);
extern void nfs41_handle_sequence_flag_errors(struct nfs_client *clp, u32 flags);
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 3285e40..c7b414a 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -56,6 +56,8 @@
#include "internal.h"
#include "pnfs.h"
+#define NFSDBG_FACILITY NFSDBG_CLIENT
+
#define OPENOWNER_POOL_SIZE 8
const nfs4_stateid zero_stateid;
@@ -1041,9 +1043,32 @@ void nfs4_schedule_lease_recovery(struct nfs_client *clp)
{
if (!clp)
return;
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
nfs4_schedule_state_manager(clp);
+ dprintk("<-- %s\n", __func__);
+}
+
+/**
+ * nfs4_schedule_migration_recovery - start background migration recovery
+ *
+ * @server: nfs_server representing remote file system that is migrating
+ *
+ */
+void nfs4_schedule_migration_recovery(struct nfs_server *server)
+{
+ struct nfs_client *clp = server->nfs_client;
+
+ dprintk("--> %s(%llx:%llx)\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor);
+ if (test_and_set_bit(NFS4CLNT_MOVED, &clp->cl_state) == 0) {
+ clp->cl_moved_server = server;
+ nfs4_schedule_state_manager(clp);
+ }
+ dprintk("<-- %s\n", __func__);
}
static int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
@@ -1393,6 +1418,9 @@ static int nfs4_do_reclaim(struct nfs_client *clp, const struct nfs4_state_recov
struct rb_node *pos;
int status = 0;
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
restart:
rcu_read_lock();
list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
@@ -1422,6 +1450,7 @@ restart:
spin_unlock(&clp->cl_lock);
}
rcu_read_unlock();
+ dprintk("<-- %s: %d\n", __func__, status);
return status;
}
@@ -1432,6 +1461,9 @@ static int nfs4_check_lease(struct nfs_client *clp)
clp->cl_mvops->state_renewal_ops;
int status = -NFS4ERR_EXPIRED;
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
/* Is the client already known to have an expired lease? */
if (test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state))
return 0;
@@ -1446,7 +1478,9 @@ static int nfs4_check_lease(struct nfs_client *clp)
status = ops->renew_lease(clp, cred);
put_rpccred(cred);
out:
- return nfs4_recovery_handle_error(clp, status);
+ status = nfs4_recovery_handle_error(clp, status);
+ dprintk("<-- %s: %d\n", __func__, status);
+ return status;
}
static int nfs4_reclaim_lease(struct nfs_client *clp)
@@ -1456,6 +1490,9 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
clp->cl_mvops->reboot_recovery_ops;
int status = -ENOENT;
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
cred = ops->get_clid_cred(clp);
if (cred != NULL) {
status = ops->establish_clid(clp, cred);
@@ -1468,9 +1505,98 @@ static int nfs4_reclaim_lease(struct nfs_client *clp)
if (status == -NFS4ERR_MINOR_VERS_MISMATCH)
status = -EPROTONOSUPPORT;
}
+ dprintk("<-- %s: %d\n", __func__, status);
+ return status;
+}
+
+/*
+ * If cloning got us a shiny new nfs_client, a RENEW/SETCLIENTID sequence
+ * is needed. Kick off a state manager thread for the new nfs_client to
+ * handle this, and wait for it to finish.
+ */
+static int nfs4_init_cloned_client(struct nfs_client *clp)
+{
+ int status = 0;
+
+ dprintk("--> %s: \"%s\" (client ID %llx)\n",
+ __func__, clp->cl_hostname, clp->cl_clientid);
+
+ if (test_and_set_bit(NFS4CLNT_CLONED_CLIENT, &clp->cl_state) == 0) {
+ clear_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
+ nfs4_schedule_state_manager(clp);
+ status = wait_on_bit(&clp->cl_state, NFS4CLNT_MANAGER_RUNNING,
+ nfs_wait_bit_killable, TASK_KILLABLE);
+ }
+
+ dprintk("<-- %s: %d\n", __func__, status);
return status;
}
+/*
+ * Try remote migration of one FSID from a source server to a
+ * destination server. The source server provides a list of
+ * potential destinations.
+ */
+static void nfs4_try_migration(struct nfs_server *server)
+{
+ struct nfs_client *clp = server->nfs_client;
+ struct nfs4_fs_locations *locations = NULL;
+ struct page *page;
+ int status;
+
+ dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__,
+ (unsigned long long)server->fsid.major,
+ (unsigned long long)server->fsid.minor,
+ clp->cl_hostname);
+
+ status = -ENOMEM;
+ page = alloc_page(GFP_KERNEL);
+ locations = kmalloc(sizeof(struct nfs4_fs_locations), GFP_KERNEL);
+ if (page == NULL || locations == NULL) {
+ dprintk("<-- %s: no memory\n", __func__);
+ goto out_err;
+ }
+
+ status = nfs4_proc_get_mig_status(server, locations, page);
+ if (status != 0) {
+ dprintk("<-- %s: get migration status: %d\n",
+ __func__, status);
+ goto out_err;
+ }
+ if (!(locations->fattr.valid & NFS_ATTR_FATTR_V4_LOCATIONS)) {
+ dprintk("<-- %s: No fs_locations data available, "
+ "migration skipped\n", __func__);
+ goto out_err;
+ }
+
+ /* NB: if successful, nfs4_replace_transport() replaces
+ * server->nfs_client with the cloned nfs_client */
+ status = nfs4_replace_transport(server, locations);
+ if (status != 0) {
+ dprintk("<-- %s: failed to replace transport: %d\n",
+ __func__, status);
+ goto out_err;
+ }
+
+ if (server->nfs_client->cl_clientid == 0) {
+ server->nfs_client->cl_clientid = clp->cl_clientid;
+
+ status = nfs4_init_cloned_client(server->nfs_client);
+ if (status != 0) {
+ dprintk("<-- %s: failed to init nfs_client: %d\n",
+ __func__, status);
+ goto out_err;
+ }
+ }
+
+ dprintk("<-- %s: migration succeeded\n", __func__);
+
+out_err:
+ if (page != NULL)
+ __free_page(page);
+ kfree(locations);
+}
+
#ifdef CONFIG_NFS_V4_1
void nfs4_schedule_session_recovery(struct nfs4_session *session)
{
@@ -1631,8 +1757,22 @@ static void nfs4_state_manager(struct nfs_client *clp)
{
int status = 0;
+ dprintk("--> %s: \"%s\" (client ID %llx) state: %08lx\n",
+ __func__, clp->cl_hostname, clp->cl_clientid, clp->cl_state);
+
/* Ensure exclusive access to NFSv4 state */
do {
+ if (test_and_clear_bit(NFS4CLNT_CLONED_CLIENT,
+ &clp->cl_state)) {
+ /* If the server still recognizes the short-form
+ * client ID, ensure that the next SETCLIENTID doesn't
+ * cause the server to drop all that state */
+ if (nfs4_check_lease(clp) == 0)
+ set_bit(NFS4CLNT_UPDATE_CALLBACK,
+ &clp->cl_state);
+ set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state);
+ }
+
if (test_and_clear_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) {
/* We're going to have to re-establish a clientid */
status = nfs4_reclaim_lease(clp);
@@ -1670,6 +1810,11 @@ static void nfs4_state_manager(struct nfs_client *clp)
goto out_error;
}
+ if (test_and_clear_bit(NFS4CLNT_MOVED, &clp->cl_state)) {
+ nfs4_try_migration(clp->cl_moved_server);
+ continue;
+ }
+
/* First recover reboot state... */
if (test_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state)) {
status = nfs4_do_reclaim(clp,
@@ -1710,7 +1855,6 @@ static void nfs4_state_manager(struct nfs_client *clp)
continue;
}
-
nfs4_clear_state_manager_bit(clp);
/* Did we race with an attempt to give us more work? */
if (clp->cl_state == 0)
@@ -1718,6 +1862,7 @@ static void nfs4_state_manager(struct nfs_client *clp)
if (test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) != 0)
break;
} while (atomic_read(&clp->cl_count) > 1);
+ dprintk("<-- %s\n", __func__);
return;
out_error:
printk(KERN_WARNING "Error: state manager failed on NFSv4 server %s"
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index d0554c4..091abf0 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -58,6 +58,9 @@ struct nfs_client {
struct rpc_wait_queue cl_rpcwaitq;
+ /* accessed only when NFS4CLNT_MOVED bit is set */
+ struct nfs_server * cl_moved_server;
+
/* used for the setclientid verifier */
struct timespec cl_boot_time;
next prev parent reply other threads:[~2011-05-09 19:38 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2011-05-09 19:36 [PATCH 00/16] Client-side migration support for 2.6.40 [take 3] Chuck Lever
2011-05-09 19:36 ` [PATCH 01/16] SUNRPC: Allow temporary blocking of an rpc client Chuck Lever
2011-05-09 19:36 ` [PATCH 02/16] SUNRPC: Use RCU to dereference the rpc_clnt.cl_xprt field Chuck Lever
2011-05-09 19:36 ` [PATCH 03/16] SUNRPC: Move clnt->cl_server into struct rpc_xprt Chuck Lever
2011-05-09 19:36 ` [PATCH 04/16] SUNRPC: Add a helper to switch the transport of the rpc_client Chuck Lever
2011-05-09 19:37 ` [PATCH 05/16] SUNRPC: Add API to acquire source address Chuck Lever
2011-05-09 19:37 ` [PATCH 06/16] NFS: Add a client-side function to display file handles Chuck Lever
2011-05-09 19:37 ` [PATCH 07/16] NFS: Save root file handle in nfs_server Chuck Lever
2011-05-09 19:37 ` [PATCH 08/16] NFS: Introduce NFS_ATTR_FATTR_V4_LOCATIONS Chuck Lever
2011-05-09 19:37 ` [PATCH 09/16] NFS: Introduce nfs4_proc_get_mig_status() Chuck Lever
2011-05-09 19:37 ` [PATCH 10/16] NFS: Add infrastructure for updating callback data Chuck Lever
2011-05-09 19:38 ` [PATCH 11/16] NFS: Add an API for cloning an nfs_client Chuck Lever
2011-05-12 17:30 ` Chuck Lever
2011-05-12 19:30 ` Trond Myklebust
2011-05-09 19:38 ` [PATCH 12/16] NFS: Add functions to swap transports during migration recovery Chuck Lever
2011-05-09 19:38 ` Chuck Lever [this message]
2011-05-09 19:38 ` [PATCH 14/16] NFS: Remove "const" from "struct nfs_server *" fields Chuck Lever
2011-05-09 19:38 ` [PATCH 15/16] NFS: Add migration recovery callouts in nfs4proc.c Chuck Lever
2011-05-09 19:38 ` [PATCH 16/16] NFS: Implement support for NFS4ERR_LEASE_MOVED Chuck Lever
2011-05-09 22:48 ` Chuck Lever
2011-05-11 0:20 ` Tom Haynes
[not found] ` <4DC9D636.3050307-8AdZ+HgO7noAvxtiuMwx3w@public.gmane.org>
2011-05-11 14:04 ` Chuck Lever
2011-05-12 15:37 ` Chuck Lever
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20110509193822.16568.2005.stgit@matisse.1015granger.net \
--to=chuck.lever@oracle.com \
--cc=linux-nfs@vger.kernel.org \
--cc=trond.myklebust@netapp.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.