From: andros@netapp.com
To: bhalevy@panasas.com
Cc: linux-nfs@vger.kernel.org, Andy Adamson <andros@netapp.com>
Subject: [PATCH 36/40] pnfs-submit wave3 filelayout read done
Date: Fri, 4 Feb 2011 16:33:58 -0500 [thread overview]
Message-ID: <1296855242-2592-37-git-send-email-andros@netapp.com> (raw)
In-Reply-To: <1296855242-2592-36-git-send-email-andros@netapp.com>
From: Andy Adamson <andros@netapp.com>
Use our own async error handler.
Mark the layout as failed and retry i/o through the MDS on specified errors.
Signed-off-by: Andy Adamson <andros@netapp.com>
---
fs/nfs/internal.h | 1 +
fs/nfs/nfs4filelayout.c | 86 +++++++++++++++++++++++++++++++++++++++++++
fs/nfs/nfs4proc.c | 44 +++++++++++++--------
fs/nfs/nfs4state.c | 1 +
fs/nfs/pnfs.h | 1 -
include/linux/nfs_xdr.h | 1 +
include/linux/sunrpc/clnt.h | 1 +
net/sunrpc/clnt.c | 8 ++++
8 files changed, 125 insertions(+), 18 deletions(-)
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 5518d61..f69a322 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -281,6 +281,7 @@ extern int nfs_migrate_page(struct address_space *,
#endif
/* nfs4proc.c */
+extern void nfs4_reset_read(struct rpc_task *task, struct nfs_read_data *data);
extern int _nfs4_call_sync(struct nfs_server *server,
struct rpc_message *msg,
struct nfs4_sequence_args *args,
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 5fd8ed3..777d78b 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -40,6 +40,8 @@ MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dean Hildebrand <dhildebz@umich.edu>");
MODULE_DESCRIPTION("The NFSv4 file layout driver");
+#define FILELAYOUT_POLL_RETRY_MAX (15*HZ)
+
static int
filelayout_set_layoutdriver(struct nfs_server *nfss)
{
@@ -95,6 +97,88 @@ filelayout_get_dserver_offset(struct pnfs_layout_segment *lseg, loff_t offset)
BUG();
}
+/* For data server errors we don't recover from */
+static void
+filelayout_set_lo_fail(struct pnfs_layout_segment *lseg, fmode_t mode)
+{
+ if (mode & FMODE_WRITE) {
+ dprintk("%s Setting layout IOMODE_RW fail bit\n", __func__);
+ set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);
+ } else if (mode & FMODE_READ) {
+ dprintk("%s Setting layout IOMODE_READ fail bit\n", __func__);
+ set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);
+ }
+}
+
+/*
+ * Async I/O error handler.
+ *
+ * NFS4ERR_OLD_STATEID can not occur with a zero stateid seqid.
+ */
+static int filelayout_async_handle_error(struct rpc_task *task,
+ struct nfs4_state *state,
+ struct nfs_client *clp,
+ int *reset)
+{
+ if (task->tk_status >= 0)
+ return 0;
+ switch (task->tk_status) {
+ case -NFS4ERR_BADSESSION:
+ case -NFS4ERR_BADSLOT:
+ case -NFS4ERR_BAD_HIGH_SLOT:
+ case -NFS4ERR_DEADSESSION:
+ case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
+ case -NFS4ERR_SEQ_FALSE_RETRY:
+ case -NFS4ERR_SEQ_MISORDERED:
+ dprintk("%s ERROR %d, Reset session. Exchangeid "
+ "flags 0x%x\n", __func__, task->tk_status,
+ clp->cl_exchange_flags);
+ nfs4_schedule_state_recovery(clp);
+ task->tk_status = 0;
+ return -EAGAIN;
+ case -NFS4ERR_DELAY:
+ case -NFS4ERR_GRACE:
+ case -EKEYEXPIRED:
+ rpc_delay(task, FILELAYOUT_POLL_RETRY_MAX);
+ task->tk_status = 0;
+ return -EAGAIN;
+ default:
+ dprintk("%s DS error %d\n", __func__, task->tk_status);
+ /* Layout marked as failed by pnfs_check_io_status.
+ * Retry I/O through the MDS */
+ *reset = 1;
+ task->tk_status = 0;
+ return -EAGAIN;
+ }
+}
+
+/* NFS_PROTO call done callback routines */
+
+static int filelayout_read_done_cb(struct rpc_task *task,
+ struct nfs_read_data *data)
+{
+ struct nfs_client *clp = data->ds_clp;
+ int reset = 0;
+
+ dprintk("%s DS read\n", __func__);
+
+ if (filelayout_async_handle_error(task, data->args.context->state,
+ data->ds_clp, &reset) == -EAGAIN) {
+ dprintk("%s calling restart ds_clp %p ds_clp->cl_session %p\n",
+ __func__, data->ds_clp, data->ds_clp->cl_session);
+ if (reset) {
+ nfs4_reset_read(task, data);
+ filelayout_set_lo_fail(data->lseg,
+ data->args.context->state->state);
+ clp = NFS_SERVER(data->inode)->nfs_client;
+ }
+ nfs_restart_rpc(task, clp);
+ return -EAGAIN;
+ }
+
+ return 0;
+}
+
/*
* Call ops for the async read/write cases
* In the case of dense layouts, the offset needs to be reset to its
@@ -104,6 +188,8 @@ static void filelayout_read_prepare(struct rpc_task *task, void *data)
{
struct nfs_read_data *rdata = (struct nfs_read_data *)data;
+ rdata->read_done_cb = filelayout_read_done_cb;
+
if (nfs41_setup_sequence(rdata->ds_clp->cl_session,
&rdata->args.seq_args, &rdata->res.seq_res,
0, task))
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 3fcf756..9dee49d 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -3075,41 +3075,51 @@ static int nfs4_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle,
return err;
}
-static int nfs4_read_done(struct rpc_task *task, struct nfs_read_data *data)
+static int nfs4_read_done_cb(struct rpc_task *task, struct nfs_read_data *data)
{
struct nfs_server *server = NFS_SERVER(data->inode);
- struct nfs_client *clp = server->nfs_client;
-
- dprintk("--> %s\n", __func__);
-
-#ifdef CONFIG_NFS_V4_1
- /* Is this a DS session */
- if (data->ds_clp) {
- dprintk("%s DS read\n", __func__);
- clp = data->ds_clp;
- }
-#endif /* CONFIG_NFS_V4_1 */
-
- if (!nfs4_sequence_done(task, &data->res.seq_res))
- return -EAGAIN;
if (nfs4_async_handle_error(task, server, data->args.context->state) == -EAGAIN) {
- nfs_restart_rpc(task, client);
+ nfs_restart_rpc(task, server->nfs_client);
return -EAGAIN;
}
nfs_invalidate_atime(data->inode);
- if (task->tk_status > 0 && !data->ds_clp)
+ if (task->tk_status > 0)
renew_lease(server, data->timestamp);
return 0;
}
+static int nfs4_read_done(struct rpc_task *task, struct nfs_read_data *data)
+{
+
+ dprintk("--> %s\n", __func__);
+
+ if (!nfs4_sequence_done(task, &data->res.seq_res))
+ return -EAGAIN;
+
+ return data->read_done_cb(task, data);
+}
+
static void nfs4_proc_read_setup(struct nfs_read_data *data, struct rpc_message *msg)
{
data->timestamp = jiffies;
+ data->read_done_cb = nfs4_read_done_cb;
msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ];
}
+/* Reset the the nfs_read_data to send the read to another server. */
+void nfs4_reset_read(struct rpc_task *task, struct nfs_read_data *data)
+{
+ dprintk("%s Reset task for i/o through \n", __func__);
+ data->ds_clp = NULL;
+ data->args.fh = NFS_FH(data->inode);
+ data->read_done_cb = nfs4_read_done_cb;
+ task->tk_ops = data->call_ops;
+ rpc_task_reset_client(task, NFS_CLIENT(data->inode));
+}
+EXPORT_SYMBOL_GPL(nfs4_reset_read);
+
static int nfs4_write_done(struct rpc_task *task, struct nfs_write_data *data)
{
struct inode *inode = data->inode;
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 49433aa..346fb97 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1022,6 +1022,7 @@ void nfs4_schedule_state_recovery(struct nfs_client *clp)
set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state);
nfs4_schedule_state_manager(clp);
}
+EXPORT_SYMBOL_GPL(nfs4_schedule_state_recovery);
int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state)
{
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 6a99c33..218cdfe 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -198,7 +198,6 @@ void pnfs_roc_release(struct inode *ino);
void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
-
static inline int lo_fail_bit(u32 iomode)
{
return iomode == IOMODE_RW ?
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 1222aa9..c91f468 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -1020,6 +1020,7 @@ struct nfs_read_data {
struct pnfs_layout_segment *lseg;
struct nfs_client *ds_clp; /* pNFS data server */
const struct rpc_call_ops *call_ops; /* For pNFS recovery to MDS */
+ int (*read_done_cb) (struct rpc_task *task, struct nfs_read_data *data);
__u64 orig_offset; /* Filelayout dense stripe */
struct page *page_array[NFS_PAGEVEC_SIZE];
};
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index ef9476a..db7bcaf 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -129,6 +129,7 @@ struct rpc_create_args {
struct rpc_clnt *rpc_create(struct rpc_create_args *args);
struct rpc_clnt *rpc_bind_new_program(struct rpc_clnt *,
struct rpc_program *, u32);
+void rpc_task_reset_client(struct rpc_task *task, struct rpc_clnt *clnt);
struct rpc_clnt *rpc_clone_client(struct rpc_clnt *);
void rpc_shutdown_client(struct rpc_clnt *);
void rpc_release_client(struct rpc_clnt *);
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 57d344c..5c4df70 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -597,6 +597,14 @@ void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt)
}
}
+void rpc_task_reset_client(struct rpc_task *task, struct rpc_clnt *clnt)
+{
+ rpc_task_release_client(task);
+ rpc_task_set_client(task, clnt);
+}
+EXPORT_SYMBOL_GPL(rpc_task_reset_client);
+
+
static void
rpc_task_set_rpc_message(struct rpc_task *task, const struct rpc_message *msg)
{
--
1.6.6
next prev parent reply other threads:[~2011-02-04 21:34 UTC|newest]
Thread overview: 58+ messages / expand[flat|nested] mbox.gz Atom feed top
2011-02-04 21:33 [PATCH 0/40] Wave3: For pNFS team review, not for kernel submission andros
2011-02-04 21:33 ` [PATCH 01/40] pnfs-submit: wave3: lseg refcounting andros
2011-02-04 21:33 ` [PATCH 02/40] pnfs_submit: add data server session to nfs4_setup_sequence andros
2011-02-04 21:33 ` [PATCH 03/40] pnfs_submit: update nfs4_async_handle_error for data server andros
2011-02-04 21:33 ` [PATCH 04/40] pnfs_submit: update state renewal for data servers andros
2011-02-04 21:33 ` [PATCH 05/40] pnfs_submit: wave3 pageio-helpers andros
2011-02-04 21:33 ` [PATCH 06/40] pnfs_submit: wave3 associate layout segment with nfs_page andros
2011-02-04 21:33 ` [PATCH 07/40] pnfs_submit: filelayout policy operations andros
2011-02-04 21:33 ` [PATCH 08/40] pnfs_submit: filelayout i/o helpers andros
2011-02-04 21:33 ` [PATCH 09/40] pnfs_submit: wave3 generic read andros
2011-02-04 21:33 ` [PATCH 10/40] pnfs_submit: filelayout read andros
2011-02-04 21:33 ` [PATCH 11/40] pnfs_submit: increase NFS_MAX_FILE_IO_SIZE andros
2011-02-04 21:33 ` [PATCH 12/40] pnfs_submit: enforce requested DS only pNFS role andros
2011-02-04 21:33 ` [PATCH 13/40] REVERT pnfs_submit-add-data-server-session-to-nfs4_setup_s.patch andros
2011-02-04 21:33 ` [PATCH 14/40] REVERT: pnfs_submit: update nfs4_async_handle_error for data server andros
2011-02-04 21:33 ` [PATCH 15/40] REVERT pnfs_submit: increase NFS_MAX_FILE_IO_SIZE andros
2011-02-04 21:33 ` [PATCH 16/40] REVERT pnfs_submit: enforce requested DS only pNFS role andros
2011-02-04 21:33 ` [PATCH 17/40] SQUASHME pnfs-submit wave3 remove is_ds_only_session andros
2011-02-04 21:33 ` [PATCH 18/40] SQUASHME pnfs-submit: wave3 make pnfs_initiate_read static andros
2011-02-04 21:33 ` [PATCH 19/40] SQUASHME pnfs-submit wave3 filelayout read pagelist cleanup andros
2011-02-04 21:33 ` [PATCH 20/40] SQUASHME pnfs-submit wave3 remove nr_pages from read_pagelist andros
2011-02-04 21:33 ` [PATCH 21/40] SQUASHME pnfs-submit wave3 add comment to nfs4_fl_prepare_ds_fh andros
2011-02-04 21:33 ` [PATCH 22/40] SQUASHME pnfs-submit wave3 move BUG outside of switch andros
2011-02-04 21:33 ` [PATCH 23/40] SQUASHME pnfs-submit wave3 new function for ds expired lease andros
2011-02-04 21:33 ` [PATCH 24/40] NFS move nfs_client initialization into nfs_get_client andros
2011-02-04 21:33 ` [PATCH 25/40] pnfs-submit: wave3 refactor dataserver client setup andros
2011-02-04 21:33 ` [PATCH 26/40] pnfs-submit: wave3 refactor data server session initialization andros
2011-02-04 21:33 ` [PATCH 27/40] pnfs_submit: wave3 rename nfs4_pnfs_ds_create andros
2011-02-04 21:33 ` [PATCH 28/40] pnfs-submit: wave3 turn off pNFS on ds connection failure andros
2011-02-04 21:33 ` [PATCH 29/40] pnfs-submit: wave3 rewrite read lseg refcounting andros
2011-02-04 21:33 ` [PATCH 30/40] pnfs-submit: wave3 let LAYOUTGET distinguish between read and write calls andros
2011-02-04 21:33 ` [PATCH 31/40] pnfs_submit wave3 remove struct pnfs_fl_call_data andros
2011-02-04 21:33 ` [PATCH 32/40] pnfs_submit: wave3 get rid of pnfs_call_data andros
2011-02-04 21:33 ` [PATCH 33/40] pnfs-submit wave3 remove CONFIG_NFS_V4 and V4_1 from nfs_read_data andros
2011-02-04 21:33 ` [PATCH 34/40] pnfs-submit wave3 don't use nfs_read_prepare for DS andros
2011-02-04 21:33 ` [PATCH 35/40] pnfs_submit wave3 filelayout_read_prepare andros
2011-02-04 21:33 ` andros [this message]
2011-02-04 21:33 ` [PATCH 37/40] pnfs-submit wave3 send zero stateid seqid on v4.1 i/o andros
2011-02-04 21:34 ` [PATCH 38/40] pnfs-submit wave3 new flag for state renewal check andros
2011-02-04 21:34 ` [PATCH 39/40] pnfs-submit wave3 new flag for lease time check andros
2011-02-04 21:34 ` [PATCH 40/40] pnfs-submit wave3 add MDS mount DS only check andros
2011-02-07 17:42 ` [PATCH 37/40] pnfs-submit wave3 send zero stateid seqid on v4.1 i/o Benny Halevy
2011-02-09 17:11 ` William A. (Andy) Adamson
2011-02-08 23:06 ` [PATCH 36/40] pnfs-submit wave3 filelayout read done Fred Isaman
2011-02-09 16:10 ` William A. (Andy) Adamson
2011-02-08 22:09 ` [PATCH 33/40] pnfs-submit wave3 remove CONFIG_NFS_V4 and V4_1 from nfs_read_data Fred Isaman
[not found] ` <AANLkTin_N0rFNr2KzxZ32bpWWUzwJQ4skLnZNVA=W6FQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2011-02-09 16:11 ` William A. (Andy) Adamson
2011-02-04 21:59 ` [PATCH 30/40] pnfs-submit: wave3 let LAYOUTGET distinguish between read and write calls Fred Isaman
2011-02-05 16:47 ` William A. (Andy) Adamson
2011-02-04 21:51 ` [PATCH 23/40] SQUASHME pnfs-submit wave3 new function for ds expired lease Fred Isaman
2011-02-05 16:46 ` William A. (Andy) Adamson
2011-02-06 19:41 ` Fred Isaman
2011-02-07 15:05 ` William A. (Andy) Adamson
2011-02-07 15:29 ` Fred Isaman
2011-02-04 21:44 ` [PATCH 19/40] SQUASHME pnfs-submit wave3 filelayout read pagelist cleanup Fred Isaman
2011-02-05 16:47 ` William A. (Andy) Adamson
2011-02-10 5:59 ` [PATCH 0/40] Wave3: For pNFS team review, not for kernel submission Benny Halevy
2011-02-10 14:17 ` William A. (Andy) Adamson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1296855242-2592-37-git-send-email-andros@netapp.com \
--to=andros@netapp.com \
--cc=bhalevy@panasas.com \
--cc=linux-nfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.