From: trondmy@kernel.org
To: linux-nfs@vger.kernel.org
Subject: [PATCH 12/22] pNFS: Add infrastructure for cleaning up per-layout commit structures
Date: Tue, 24 Mar 2020 19:47:18 -0400 [thread overview]
Message-ID: <20200324234728.8997-13-trondmy@kernel.org> (raw)
In-Reply-To: <20200324234728.8997-12-trondmy@kernel.org>
From: Trond Myklebust <trond.myklebust@hammerspace.com>
Ensure that both the file and flexfiles layout types clean up when
freeing the layout segments.
Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>
---
fs/nfs/filelayout/filelayout.c | 16 +++++
fs/nfs/flexfilelayout/flexfilelayout.c | 11 ++++
fs/nfs/internal.h | 4 +-
fs/nfs/pnfs.c | 1 +
fs/nfs/pnfs.h | 4 ++
fs/nfs/pnfs_nfs.c | 88 +++++++++++++++++++++++++-
include/linux/nfs_xdr.h | 1 +
7 files changed, 121 insertions(+), 4 deletions(-)
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 7bd02efbe19a..1cb043838b14 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -750,11 +750,16 @@ filelayout_free_lseg(struct pnfs_layout_segment *lseg)
/* This assumes a single RW lseg */
if (lseg->pls_range.iomode == IOMODE_RW) {
struct nfs4_filelayout *flo;
+ struct inode *inode;
flo = FILELAYOUT_FROM_HDR(lseg->pls_layout);
+ inode = flo->generic_hdr.plh_inode;
+ spin_lock(&inode->i_lock);
flo->commit_info.nbuckets = 0;
kfree(flo->commit_info.buckets);
flo->commit_info.buckets = NULL;
+ pnfs_generic_ds_cinfo_release_lseg(&flo->commit_info, lseg);
+ spin_unlock(&inode->i_lock);
}
_filelayout_free_lseg(fl);
}
@@ -1163,6 +1168,16 @@ filelayout_get_ds_info(struct inode *inode)
return &FILELAYOUT_FROM_HDR(layout)->commit_info;
}
+static void
+filelayout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo,
+ struct inode *inode)
+{
+ spin_lock(&inode->i_lock);
+ pnfs_generic_ds_cinfo_destroy(fl_cinfo);
+ spin_unlock(&inode->i_lock);
+}
+
+
static struct pnfs_layoutdriver_type filelayout_type = {
.id = LAYOUT_NFSV4_1_FILES,
.name = "LAYOUT_NFSV4_1_FILES",
@@ -1176,6 +1191,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
.pg_read_ops = &filelayout_pg_read_ops,
.pg_write_ops = &filelayout_pg_write_ops,
.get_ds_info = &filelayout_get_ds_info,
+ .release_ds_info = filelayout_release_ds_info,
.mark_request_commit = filelayout_mark_request_commit,
.clear_request_commit = pnfs_generic_clear_request_commit,
.scan_commit_lists = pnfs_generic_scan_commit_lists,
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index c7cccdd746e4..3ddd8080f76d 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -580,6 +580,7 @@ ff_layout_free_lseg(struct pnfs_layout_segment *lseg)
kfree(ffl->commit_info.buckets);
ffl->commit_info.buckets = NULL;
}
+ pnfs_generic_ds_cinfo_release_lseg(&ffl->commit_info, lseg);
spin_unlock(&inode->i_lock);
}
_ff_layout_free_lseg(fls);
@@ -2003,6 +2004,15 @@ ff_layout_get_ds_info(struct inode *inode)
return &FF_LAYOUT_FROM_HDR(layout)->commit_info;
}
+static void
+ff_layout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo,
+ struct inode *inode)
+{
+ spin_lock(&inode->i_lock);
+ pnfs_generic_ds_cinfo_destroy(fl_cinfo);
+ spin_unlock(&inode->i_lock);
+}
+
static void
ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d)
{
@@ -2503,6 +2513,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
.pg_read_ops = &ff_layout_pg_read_ops,
.pg_write_ops = &ff_layout_pg_write_ops,
.get_ds_info = ff_layout_get_ds_info,
+ .release_ds_info = ff_layout_release_ds_info,
.free_deviceid_node = ff_layout_free_deviceid_node,
.mark_request_commit = pnfs_layout_mark_request_commit,
.clear_request_commit = pnfs_generic_clear_request_commit,
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 4a1adad3740f..683146a51599 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -534,9 +534,11 @@ void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo)
pnfs_bucket_clear_pnfs_ds_commit_verifiers(cinfo->buckets,
cinfo->nbuckets);
- list_for_each_entry(array, &cinfo->commits, cinfo_list)
+ rcu_read_lock();
+ list_for_each_entry_rcu(array, &cinfo->commits, cinfo_list)
pnfs_bucket_clear_pnfs_ds_commit_verifiers(array->buckets,
array->nbuckets);
+ rcu_read_unlock();
}
#else
static inline
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 6b25117fca5f..eba18f137fb0 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -506,6 +506,7 @@ pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
{
INIT_LIST_HEAD(&lseg->pls_list);
INIT_LIST_HEAD(&lseg->pls_lc_list);
+ INIT_LIST_HEAD(&lseg->pls_commits);
refcount_set(&lseg->pls_refcount, 1);
set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
lseg->pls_layout = lo;
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 4ff974631206..9742ab184960 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -66,6 +66,7 @@ struct nfs4_pnfs_ds {
struct pnfs_layout_segment {
struct list_head pls_list;
struct list_head pls_lc_list;
+ struct list_head pls_commits;
struct pnfs_layout_range pls_range;
refcount_t pls_refcount;
u32 pls_seq;
@@ -370,6 +371,9 @@ void nfs4_deviceid_purge_client(const struct nfs_client *);
/* pnfs_nfs.c */
struct pnfs_commit_array *pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags);
void pnfs_free_commit_array(struct pnfs_commit_array *p);
+void pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo,
+ struct pnfs_layout_segment *lseg);
+void pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo);
void pnfs_generic_clear_request_commit(struct nfs_page *req,
struct nfs_commit_info *cinfo);
diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
index f895a28b1e26..edad251a6a48 100644
--- a/fs/nfs/pnfs_nfs.c
+++ b/fs/nfs/pnfs_nfs.c
@@ -118,6 +118,67 @@ pnfs_free_commit_array(struct pnfs_commit_array *p)
}
EXPORT_SYMBOL_GPL(pnfs_free_commit_array);
+static void
+pnfs_release_commit_array_locked(struct pnfs_commit_array *array)
+{
+ list_del_rcu(&array->cinfo_list);
+ list_del(&array->lseg_list);
+ pnfs_free_commit_array(array);
+}
+
+static void
+pnfs_put_commit_array_locked(struct pnfs_commit_array *array)
+{
+ if (refcount_dec_and_test(&array->refcount))
+ pnfs_release_commit_array_locked(array);
+}
+
+static void
+pnfs_put_commit_array(struct pnfs_commit_array *array, struct inode *inode)
+{
+ if (refcount_dec_and_lock(&array->refcount, &inode->i_lock)) {
+ pnfs_release_commit_array_locked(array);
+ spin_unlock(&inode->i_lock);
+ }
+}
+
+static struct pnfs_commit_array *
+pnfs_get_commit_array(struct pnfs_commit_array *array)
+{
+ if (refcount_inc_not_zero(&array->refcount))
+ return array;
+ return NULL;
+}
+
+static void
+pnfs_remove_and_free_commit_array(struct pnfs_commit_array *array)
+{
+ array->lseg = NULL;
+ list_del_init(&array->lseg_list);
+ pnfs_put_commit_array_locked(array);
+}
+
+void
+pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo,
+ struct pnfs_layout_segment *lseg)
+{
+ struct pnfs_commit_array *array, *tmp;
+
+ list_for_each_entry_safe(array, tmp, &lseg->pls_commits, lseg_list)
+ pnfs_remove_and_free_commit_array(array);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_release_lseg);
+
+void
+pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo)
+{
+ struct pnfs_commit_array *array, *tmp;
+
+ list_for_each_entry_safe(array, tmp, &fl_cinfo->commits, cinfo_list)
+ pnfs_remove_and_free_commit_array(array);
+}
+EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_destroy);
+
/*
* Locks the nfs_page requests for commit and moves them to
* @bucket->committing.
@@ -177,14 +238,21 @@ int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max)
max -= cnt;
if (!max)
return rv;
- list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) {
+ rcu_read_lock();
+ list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
+ if (!array->lseg || !pnfs_get_commit_array(array))
+ continue;
+ rcu_read_unlock();
cnt = pnfs_bucket_scan_array(cinfo, array->buckets,
array->nbuckets, max);
+ rcu_read_lock();
+ pnfs_put_commit_array(array, cinfo->inode);
rv += cnt;
max -= cnt;
if (!max)
break;
}
+ rcu_read_unlock();
return rv;
}
EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
@@ -230,13 +298,20 @@ void pnfs_generic_recover_commit_reqs(struct list_head *dst,
fl_cinfo->nbuckets,
cinfo);
fl_cinfo->nwritten -= nwritten;
- list_for_each_entry(array, &fl_cinfo->commits, cinfo_list) {
+ rcu_read_lock();
+ list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
+ if (!array->lseg || !pnfs_get_commit_array(array))
+ continue;
+ rcu_read_unlock();
nwritten = pnfs_bucket_recover_commit_reqs(dst,
array->buckets,
array->nbuckets,
cinfo);
+ rcu_read_lock();
+ pnfs_put_commit_array(array, cinfo->inode);
fl_cinfo->nwritten -= nwritten;
}
+ rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
@@ -330,9 +405,16 @@ pnfs_alloc_ds_commits_list(struct list_head *list,
struct pnfs_commit_array *array;
unsigned int ret = 0;
- list_for_each_entry(array, &fl_cinfo->commits, cinfo_list)
+ rcu_read_lock();
+ list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
+ if (!array->lseg || !pnfs_get_commit_array(array))
+ continue;
+ rcu_read_unlock();
ret += pnfs_bucket_alloc_ds_commits(list, array->buckets,
array->nbuckets, cinfo);
+ rcu_read_lock();
+ pnfs_put_commit_array(array, cinfo->inode);
+ }
return ret;
}
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 9946787eda72..33be2ee2a248 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -1275,6 +1275,7 @@ struct pnfs_commit_array {
struct list_head lseg_list;
struct pnfs_layout_segment *lseg;
struct rcu_head rcu;
+ refcount_t refcount;
unsigned int nbuckets;
struct pnfs_commit_bucket buckets[];
};
--
2.25.1
next prev parent reply other threads:[~2020-03-24 23:49 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-24 23:47 [PATCH 00/22] Fix NFS commit to DS trondmy
2020-03-24 23:47 ` [PATCH 01/22] pNFS/flexfiles: Simplify allocation of the mirror array trondmy
2020-03-24 23:47 ` [PATCH 02/22] NFS/pNFS: Refactor pnfs_generic_commit_pagelist() trondmy
2020-03-24 23:47 ` [PATCH 03/22] pNFS: Add a helper to allocate the array of buckets trondmy
2020-03-24 23:47 ` [PATCH 04/22] NFSv4/pnfs: Support a list of commit arrays in struct pnfs_ds_commit_info trondmy
2020-03-24 23:47 ` [PATCH 05/22] NFSv4/pNFS: Scan the full list of commit arrays when committing trondmy
2020-03-24 23:47 ` [PATCH 06/22] pNFS: Support per-layout segment commits in pnfs_generic_recover_commit_reqs() trondmy
2020-03-24 23:47 ` [PATCH 07/22] pNFS: Support per-layout segment commits in pnfs_generic_commit_pagelist() trondmy
2020-03-24 23:47 ` [PATCH 08/22] NFS/pNFS: Allow O_DIRECT to release the DS commitinfo trondmy
2020-03-24 23:47 ` [PATCH 09/22] NFS: commit errors should be fatal trondmy
2020-03-24 23:47 ` [PATCH 10/22] NFS: Fix O_DIRECT commit verifier handling trondmy
2020-03-24 23:47 ` [PATCH 11/22] NFS/pNFS: Support commit arrays in nfs_clear_pnfs_ds_commit_verifiers() trondmy
2020-03-24 23:47 ` trondmy [this message]
2020-03-24 23:47 ` [PATCH 13/22] pNFS: Enable per-layout segment commit structures trondmy
2020-03-24 23:47 ` [PATCH 14/22] NFS/pNFS: Add a helper pnfs_generic_search_commit_reqs() trondmy
2020-03-24 23:47 ` [PATCH 15/22] NFS: Remove bucket array from struct pnfs_ds_commit_info trondmy
2020-03-24 23:47 ` [PATCH 16/22] NFS/pNFS: Clean up pNFS commit operations trondmy
2020-03-24 23:47 ` [PATCH 17/22] NFS/pNFS: Simplify bucket layout segment reference counting trondmy
2020-03-24 23:47 ` [PATCH 18/22] NFS/pNFS: Fix pnfs_layout_mark_request_commit() invalid layout segment handling trondmy
2020-03-24 23:47 ` [PATCH 19/22] pNFS/flexfile: Don't merge layout segments if the mirrors don't match trondmy
2020-03-24 23:47 ` [PATCH 20/22] pNFS/flexfiles: Check the layout segment range before doing I/O trondmy
2020-03-24 23:47 ` [PATCH 21/22] pNFS/flexfiles: remove requirement for whole file layouts trondmy
2020-03-24 23:47 ` [PATCH 22/22] pNFS/flexfiles: Specify the layout segment range in LAYOUTGET trondmy
2020-03-27 6:39 ` [PATCH 16/22] NFS/pNFS: Clean up pNFS commit operations kbuild test robot
2020-03-27 4:20 ` [PATCH 04/22] NFSv4/pnfs: Support a list of commit arrays in struct pnfs_ds_commit_info kbuild test robot
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200324234728.8997-13-trondmy@kernel.org \
--to=trondmy@kernel.org \
--cc=linux-nfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.