From: andros@netapp.com
To: bhalevy@panasas.com
Cc: linux-nfs@vger.kernel.org, Andy Adamson <andros@netapp.com>
Subject: [PATCH 28/40] pnfs-submit: wave3 turn off pNFS on ds connection failure
Date: Fri, 4 Feb 2011 16:33:50 -0500 [thread overview]
Message-ID: <1296855242-2592-29-git-send-email-andros@netapp.com> (raw)
In-Reply-To: <1296855242-2592-28-git-send-email-andros@netapp.com>
From: Andy Adamson <andros@netapp.com>
If a data server is unavailable, go through MDS.
Mark the deviceid containing the data server as a negative cache entry.
Do not try to connect to any data server on a deviceid marked as a negative
cache entry. Mark any layout that tries to use the marked deviceid as failed.
Inodes with a layout marked as fails will not use the layout for I/O, and will
not perform any more layoutgets.
Inodes without a layout will still do layoutget, but the layout will get
marked immediately.
Signed-off-by: Andy Adamson <andros@netapp.com>
---
fs/nfs/nfs4filelayout.c | 6 ++++--
fs/nfs/nfs4filelayout.h | 3 +++
fs/nfs/nfs4filelayoutdev.c | 39 +++++++++++++++++++++++++++++----------
fs/nfs/pnfs.c | 18 ++++++++++++++----
fs/nfs/pnfs.h | 4 ++++
5 files changed, 54 insertions(+), 16 deletions(-)
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index d925af6..9b9a81c 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -146,7 +146,9 @@ filelayout_read_pagelist(struct nfs_read_data *data)
idx = nfs4_fl_calc_ds_index(lseg, offset);
ds = nfs4_fl_prepare_ds(lseg, idx);
if (!ds) {
- printk(KERN_ERR "%s: prepare_ds failed, use MDS\n", __func__);
+ /* Either layout fh index faulty, or ds connect failed */
+ set_bit(lo_fail_bit(IOMODE_RW), &lseg->pls_layout->plh_flags);
+ set_bit(lo_fail_bit(IOMODE_READ), &lseg->pls_layout->plh_flags);
return PNFS_NOT_ATTEMPTED;
}
dprintk("%s USE DS:ip %x %hu\n", __func__,
@@ -161,7 +163,7 @@ filelayout_read_pagelist(struct nfs_read_data *data)
data->args.offset = filelayout_get_dserver_offset(lseg, offset);
data->fldata.orig_offset = offset;
- /* Perform an asynchronous read */
+ /* Perform an asynchronous read to ds */
nfs_initiate_read(data, ds->ds_clp->cl_rpcclient,
&filelayout_read_call_ops);
return PNFS_ATTEMPTED;
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index f884b0c..7e33bd8 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -96,5 +96,8 @@ extern struct nfs4_file_layout_dsaddr *
nfs4_fl_find_get_deviceid(struct nfs_client *, struct nfs4_deviceid *dev_id);
struct nfs4_file_layout_dsaddr *
get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id);
+void filelayout_mark_devid_negative(struct nfs_client *clp,
+ struct pnfs_deviceid_node *devid,
+ int err, u32 ds_ipaddr);
#endif /* FS_NFS_NFS4FILELAYOUT_H */
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index 9bb13f5..8642109 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -558,27 +558,46 @@ nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, loff_t offset)
i = _nfs4_fl_calc_j_index(lseg, offset);
return flseg->fh_array[i];
}
+void
+filelayout_mark_devid_negative(struct nfs_client *mds_clp,
+ struct pnfs_deviceid_node *devid,
+ int err, u32 ds_addr)
+{
+ u32 *p = (u32 *)&devid->de_id;
+
+ printk(KERN_ERR "NFS: data server %x connection error %d."
+ " Deviceid [%x%x%x%x] marked out of use.\n",
+ ds_addr, err, p[0], p[1], p[2], p[3]);
+
+ pnfs_mark_devid_negative(mds_clp, devid);
+}
struct nfs4_pnfs_ds *
nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
{
- struct nfs4_file_layout_dsaddr *dsaddr;
+ struct nfs4_file_layout_dsaddr *dsaddr = FILELAYOUT_LSEG(lseg)->dsaddr;
+ struct nfs4_pnfs_ds *ds = dsaddr->ds_list[ds_idx];
- dsaddr = FILELAYOUT_LSEG(lseg)->dsaddr;
- if (dsaddr->ds_list[ds_idx] == NULL) {
- printk(KERN_ERR "%s: No data server for device id!\n",
- __func__);
+ if (ds == NULL) {
+ printk(KERN_ERR "%s: No data server for offset index %d\n",
+ __func__, ds_idx);
return NULL;
}
- if (!dsaddr->ds_list[ds_idx]->ds_clp) {
+ if (!ds->ds_clp) {
+ struct nfs_server *s = NFS_SERVER(lseg->pls_layout->plh_inode);
int err;
- err = nfs4_ds_connect(NFS_SERVER(lseg->pls_layout->plh_inode),
- dsaddr->ds_list[ds_idx]);
+ /* Already tried to connect, don't try again */
+ if (dsaddr->deviceid.de_flags & NFS4_DEVICE_ID_NEG_ENTRY) {
+ dprintk("%s Deviceid marked out of use\n", __func__);
+ return NULL;
+ }
+ err = nfs4_ds_connect(s, ds);
if (err) {
- printk(KERN_ERR "%s nfs4_ds_connect error %d\n",
- __func__, err);
+ filelayout_mark_devid_negative(s->nfs_client,
+ &dsaddr->deviceid, err,
+ ntohl(ds->ds_ip_addr));
return NULL;
}
}
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 4c49109..72786ec 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -759,15 +759,16 @@ pnfs_update_layout(struct inode *ino,
dprintk("%s matches recall, use MDS\n", __func__);
goto out_unlock;
}
+
+ /* If LAYOUTGET or pNFS I/O already failed once we don't try again */
+ if (test_bit(lo_fail_bit(iomode), &nfsi->layout->plh_flags))
+ goto out_unlock;
+
/* Check to see if the layout for the given range already exists */
lseg = pnfs_find_lseg(lo, iomode);
if (lseg)
goto out_unlock;
- /* if LAYOUTGET already failed once we don't try again */
- if (test_bit(lo_fail_bit(iomode), &nfsi->layout->plh_flags))
- goto out_unlock;
-
if (pnfs_layoutgets_blocked(lo, NULL, 0))
goto out_unlock;
atomic_inc(&lo->plh_outstanding);
@@ -1089,3 +1090,12 @@ pnfs_put_deviceid_cache(struct nfs_client *clp)
}
}
EXPORT_SYMBOL_GPL(pnfs_put_deviceid_cache);
+
+void
+pnfs_mark_devid_negative(struct nfs_client *clp, struct pnfs_deviceid_node *d)
+{
+ spin_lock(&clp->cl_devid_cache->dc_lock);
+ d->de_flags |= NFS4_DEVICE_ID_NEG_ENTRY;
+ spin_unlock(&clp->cl_devid_cache->dc_lock);
+}
+EXPORT_SYMBOL_GPL(pnfs_mark_devid_negative);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index cbbcdfa..25a4e25 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -135,6 +135,8 @@ struct pnfs_deviceid_node {
struct hlist_node de_node;
struct nfs4_deviceid de_id;
atomic_t de_ref;
+ unsigned long de_flags;
+#define NFS4_DEVICE_ID_NEG_ENTRY 1
};
struct pnfs_deviceid_cache {
@@ -155,6 +157,8 @@ extern struct pnfs_deviceid_node *pnfs_add_deviceid(
struct pnfs_deviceid_node *);
extern void pnfs_put_deviceid(struct pnfs_deviceid_cache *c,
struct pnfs_deviceid_node *devid);
+extern void pnfs_mark_devid_negative(struct nfs_client *clp,
+ struct pnfs_deviceid_node *d);
extern int pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *);
extern void pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *);
--
1.6.6
next prev parent reply other threads:[~2011-02-04 21:34 UTC|newest]
Thread overview: 58+ messages / expand[flat|nested] mbox.gz Atom feed top
2011-02-04 21:33 [PATCH 0/40] Wave3: For pNFS team review, not for kernel submission andros
2011-02-04 21:33 ` [PATCH 01/40] pnfs-submit: wave3: lseg refcounting andros
2011-02-04 21:33 ` [PATCH 02/40] pnfs_submit: add data server session to nfs4_setup_sequence andros
2011-02-04 21:33 ` [PATCH 03/40] pnfs_submit: update nfs4_async_handle_error for data server andros
2011-02-04 21:33 ` [PATCH 04/40] pnfs_submit: update state renewal for data servers andros
2011-02-04 21:33 ` [PATCH 05/40] pnfs_submit: wave3 pageio-helpers andros
2011-02-04 21:33 ` [PATCH 06/40] pnfs_submit: wave3 associate layout segment with nfs_page andros
2011-02-04 21:33 ` [PATCH 07/40] pnfs_submit: filelayout policy operations andros
2011-02-04 21:33 ` [PATCH 08/40] pnfs_submit: filelayout i/o helpers andros
2011-02-04 21:33 ` [PATCH 09/40] pnfs_submit: wave3 generic read andros
2011-02-04 21:33 ` [PATCH 10/40] pnfs_submit: filelayout read andros
2011-02-04 21:33 ` [PATCH 11/40] pnfs_submit: increase NFS_MAX_FILE_IO_SIZE andros
2011-02-04 21:33 ` [PATCH 12/40] pnfs_submit: enforce requested DS only pNFS role andros
2011-02-04 21:33 ` [PATCH 13/40] REVERT pnfs_submit-add-data-server-session-to-nfs4_setup_s.patch andros
2011-02-04 21:33 ` [PATCH 14/40] REVERT: pnfs_submit: update nfs4_async_handle_error for data server andros
2011-02-04 21:33 ` [PATCH 15/40] REVERT pnfs_submit: increase NFS_MAX_FILE_IO_SIZE andros
2011-02-04 21:33 ` [PATCH 16/40] REVERT pnfs_submit: enforce requested DS only pNFS role andros
2011-02-04 21:33 ` [PATCH 17/40] SQUASHME pnfs-submit wave3 remove is_ds_only_session andros
2011-02-04 21:33 ` [PATCH 18/40] SQUASHME pnfs-submit: wave3 make pnfs_initiate_read static andros
2011-02-04 21:33 ` [PATCH 19/40] SQUASHME pnfs-submit wave3 filelayout read pagelist cleanup andros
2011-02-04 21:33 ` [PATCH 20/40] SQUASHME pnfs-submit wave3 remove nr_pages from read_pagelist andros
2011-02-04 21:33 ` [PATCH 21/40] SQUASHME pnfs-submit wave3 add comment to nfs4_fl_prepare_ds_fh andros
2011-02-04 21:33 ` [PATCH 22/40] SQUASHME pnfs-submit wave3 move BUG outside of switch andros
2011-02-04 21:33 ` [PATCH 23/40] SQUASHME pnfs-submit wave3 new function for ds expired lease andros
2011-02-04 21:33 ` [PATCH 24/40] NFS move nfs_client initialization into nfs_get_client andros
2011-02-04 21:33 ` [PATCH 25/40] pnfs-submit: wave3 refactor dataserver client setup andros
2011-02-04 21:33 ` [PATCH 26/40] pnfs-submit: wave3 refactor data server session initialization andros
2011-02-04 21:33 ` [PATCH 27/40] pnfs_submit: wave3 rename nfs4_pnfs_ds_create andros
2011-02-04 21:33 ` andros [this message]
2011-02-04 21:33 ` [PATCH 29/40] pnfs-submit: wave3 rewrite read lseg refcounting andros
2011-02-04 21:33 ` [PATCH 30/40] pnfs-submit: wave3 let LAYOUTGET distinguish between read and write calls andros
2011-02-04 21:33 ` [PATCH 31/40] pnfs_submit wave3 remove struct pnfs_fl_call_data andros
2011-02-04 21:33 ` [PATCH 32/40] pnfs_submit: wave3 get rid of pnfs_call_data andros
2011-02-04 21:33 ` [PATCH 33/40] pnfs-submit wave3 remove CONFIG_NFS_V4 and V4_1 from nfs_read_data andros
2011-02-04 21:33 ` [PATCH 34/40] pnfs-submit wave3 don't use nfs_read_prepare for DS andros
2011-02-04 21:33 ` [PATCH 35/40] pnfs_submit wave3 filelayout_read_prepare andros
2011-02-04 21:33 ` [PATCH 36/40] pnfs-submit wave3 filelayout read done andros
2011-02-04 21:33 ` [PATCH 37/40] pnfs-submit wave3 send zero stateid seqid on v4.1 i/o andros
2011-02-04 21:34 ` [PATCH 38/40] pnfs-submit wave3 new flag for state renewal check andros
2011-02-04 21:34 ` [PATCH 39/40] pnfs-submit wave3 new flag for lease time check andros
2011-02-04 21:34 ` [PATCH 40/40] pnfs-submit wave3 add MDS mount DS only check andros
2011-02-07 17:42 ` [PATCH 37/40] pnfs-submit wave3 send zero stateid seqid on v4.1 i/o Benny Halevy
2011-02-09 17:11 ` William A. (Andy) Adamson
2011-02-08 23:06 ` [PATCH 36/40] pnfs-submit wave3 filelayout read done Fred Isaman
2011-02-09 16:10 ` William A. (Andy) Adamson
2011-02-08 22:09 ` [PATCH 33/40] pnfs-submit wave3 remove CONFIG_NFS_V4 and V4_1 from nfs_read_data Fred Isaman
[not found] ` <AANLkTin_N0rFNr2KzxZ32bpWWUzwJQ4skLnZNVA=W6FQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org>
2011-02-09 16:11 ` William A. (Andy) Adamson
2011-02-04 21:59 ` [PATCH 30/40] pnfs-submit: wave3 let LAYOUTGET distinguish between read and write calls Fred Isaman
2011-02-05 16:47 ` William A. (Andy) Adamson
2011-02-04 21:51 ` [PATCH 23/40] SQUASHME pnfs-submit wave3 new function for ds expired lease Fred Isaman
2011-02-05 16:46 ` William A. (Andy) Adamson
2011-02-06 19:41 ` Fred Isaman
2011-02-07 15:05 ` William A. (Andy) Adamson
2011-02-07 15:29 ` Fred Isaman
2011-02-04 21:44 ` [PATCH 19/40] SQUASHME pnfs-submit wave3 filelayout read pagelist cleanup Fred Isaman
2011-02-05 16:47 ` William A. (Andy) Adamson
2011-02-10 5:59 ` [PATCH 0/40] Wave3: For pNFS team review, not for kernel submission Benny Halevy
2011-02-10 14:17 ` William A. (Andy) Adamson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1296855242-2592-29-git-send-email-andros@netapp.com \
--to=andros@netapp.com \
--cc=bhalevy@panasas.com \
--cc=linux-nfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.