From: Chuck Lever III <chuck.lever@oracle.com>
To: Dai Ngo <dai.ngo@oracle.com>
Cc: Jeff Layton <jlayton@kernel.org>,
Linux NFS Mailing List <linux-nfs@vger.kernel.org>
Subject: Re: [PATCH v7 2/2] NFSD: add shrinker to reap courtesy clients on low memory condition
Date: Wed, 14 Sep 2022 18:32:37 +0000 [thread overview]
Message-ID: <0E0CBCE0-0270-4086-91F1-901153A9D2B0@oracle.com> (raw)
In-Reply-To: <1663170866-21524-3-git-send-email-dai.ngo@oracle.com>
> On Sep 14, 2022, at 8:54 AM, Dai Ngo <dai.ngo@oracle.com> wrote:
>
> Add courtesy_client_reaper to react to low memory condition triggered
> by the system memory shrinker.
>
> The delayed_work for the courtesy_client_reaper is scheduled on
> the shrinker's count callback using the laundry_wq.
>
> The shrinker's scan callback is not used for expiring the courtesy
> clients due to potential deadlocks.
>
> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
> ---
> fs/nfsd/netns.h | 2 ++
> fs/nfsd/nfs4state.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++-----
> fs/nfsd/nfsctl.c | 6 ++--
> fs/nfsd/nfsd.h | 7 ++--
> 4 files changed, 97 insertions(+), 12 deletions(-)
>
> diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
> index 55c7006d6109..8c854ba3285b 100644
> --- a/fs/nfsd/netns.h
> +++ b/fs/nfsd/netns.h
> @@ -194,6 +194,8 @@ struct nfsd_net {
> int nfs4_max_clients;
>
> atomic_t nfsd_courtesy_clients;
> + struct shrinker nfsd_client_shrinker;
> + struct delayed_work nfsd_shrinker_work;
> };
>
> /* Simple check to find out if a given net was properly initialized */
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 2827329704ea..62b848bb55df 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -4347,7 +4347,27 @@ nfsd4_init_slabs(void)
> return -ENOMEM;
> }
>
> -void nfsd4_init_leases_net(struct nfsd_net *nn)
> +static unsigned long
> +nfsd_courtesy_client_count(struct shrinker *shrink, struct shrink_control *sc)
> +{
> + int cnt;
> + struct nfsd_net *nn = container_of(shrink,
> + struct nfsd_net, nfsd_client_shrinker);
> +
> + cnt = atomic_read(&nn->nfsd_courtesy_clients);
> + if (cnt > 0)
> + mod_delayed_work(laundry_wq, &nn->nfsd_shrinker_work, 0);
> + return (unsigned long)cnt;
> +}
> +
> +static unsigned long
> +nfsd_courtesy_client_scan(struct shrinker *shrink, struct shrink_control *sc)
> +{
> + return SHRINK_STOP;
> +}
> +
> +int
> +nfsd4_init_leases_net(struct nfsd_net *nn)
> {
> struct sysinfo si;
> u64 max_clients;
> @@ -4368,6 +4388,16 @@ void nfsd4_init_leases_net(struct nfsd_net *nn)
> nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);
>
> atomic_set(&nn->nfsd_courtesy_clients, 0);
> + nn->nfsd_client_shrinker.scan_objects = nfsd_courtesy_client_scan;
> + nn->nfsd_client_shrinker.count_objects = nfsd_courtesy_client_count;
> + nn->nfsd_client_shrinker.seeks = DEFAULT_SEEKS;
> + return register_shrinker(&nn->nfsd_client_shrinker, "nfsd-client");
> +}
> +
> +void
> +nfsd4_leases_net_shutdown(struct nfsd_net *nn)
> +{
> + unregister_shrinker(&nn->nfsd_client_shrinker);
> }
>
> static void init_nfs4_replay(struct nfs4_replay *rp)
> @@ -5909,10 +5939,49 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
> spin_unlock(&nn->client_lock);
> }
>
> +static void
> +nfs4_get_courtesy_client_reaplist(struct nfsd_net *nn,
> + struct list_head *reaplist)
> +{
> + unsigned int maxreap = 0, reapcnt = 0;
> + struct list_head *pos, *next;
> + struct nfs4_client *clp;
> +
> + maxreap = NFSD_CLIENT_MAX_TRIM_PER_RUN;
> + INIT_LIST_HEAD(reaplist);
> +
> + spin_lock(&nn->client_lock);
> + list_for_each_safe(pos, next, &nn->client_lru) {
> + clp = list_entry(pos, struct nfs4_client, cl_lru);
> + if (clp->cl_state == NFSD4_ACTIVE)
> + break;
> + if (reapcnt >= maxreap)
> + break;
> + if (!mark_client_expired_locked(clp)) {
> + list_add(&clp->cl_lru, reaplist);
> + reapcnt++;
> + }
> + }
> + spin_unlock(&nn->client_lock);
> +}
> +
> +static void
> +nfs4_process_client_reaplist(struct list_head *reaplist)
> +{
> + struct list_head *pos, *next;
> + struct nfs4_client *clp;
> +
> + list_for_each_safe(pos, next, reaplist) {
> + clp = list_entry(pos, struct nfs4_client, cl_lru);
> + trace_nfsd_clid_purged(&clp->cl_clientid);
> + list_del_init(&clp->cl_lru);
> + expire_client(clp);
> + }
> +}
> +
> static time64_t
> nfs4_laundromat(struct nfsd_net *nn)
> {
> - struct nfs4_client *clp;
> struct nfs4_openowner *oo;
> struct nfs4_delegation *dp;
> struct nfs4_ol_stateid *stp;
> @@ -5941,12 +6010,8 @@ nfs4_laundromat(struct nfsd_net *nn)
> }
> spin_unlock(&nn->s2s_cp_lock);
> nfs4_get_client_reaplist(nn, &reaplist, <);
> - list_for_each_safe(pos, next, &reaplist) {
> - clp = list_entry(pos, struct nfs4_client, cl_lru);
> - trace_nfsd_clid_purged(&clp->cl_clientid);
> - list_del_init(&clp->cl_lru);
> - expire_client(clp);
> - }
> + nfs4_process_client_reaplist(&reaplist);
> +
> spin_lock(&state_lock);
> list_for_each_safe(pos, next, &nn->del_recall_lru) {
> dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
> @@ -6029,6 +6094,18 @@ laundromat_main(struct work_struct *laundry)
> queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ);
> }
>
> +static void
> +courtesy_client_reaper(struct work_struct *reaper)
> +{
> + struct list_head reaplist;
> + struct delayed_work *dwork = to_delayed_work(reaper);
> + struct nfsd_net *nn = container_of(dwork, struct nfsd_net,
> + nfsd_shrinker_work);
> +
> + nfs4_get_courtesy_client_reaplist(nn, &reaplist);
> + nfs4_process_client_reaplist(&reaplist);
> +}
> +
> static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp)
> {
> if (!fh_match(&fhp->fh_handle, &stp->sc_file->fi_fhandle))
> @@ -7845,6 +7922,7 @@ static int nfs4_state_create_net(struct net *net)
> INIT_LIST_HEAD(&nn->blocked_locks_lru);
>
> INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main);
> + INIT_DELAYED_WORK(&nn->nfsd_shrinker_work, courtesy_client_reaper);
> get_net(net);
>
> return 0;
> diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
> index 917fa1892fd2..597a26ad4183 100644
> --- a/fs/nfsd/nfsctl.c
> +++ b/fs/nfsd/nfsctl.c
> @@ -1481,11 +1481,12 @@ static __net_init int nfsd_init_net(struct net *net)
> goto out_idmap_error;
> nn->nfsd_versions = NULL;
> nn->nfsd4_minorversions = NULL;
> + retval = nfsd4_init_leases_net(nn);
> + if (retval)
> + goto out_drc_error;
> retval = nfsd_reply_cache_init(nn);
> if (retval)
> goto out_drc_error;
> - nfsd4_init_leases_net(nn);
> -
> get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
> seqlock_init(&nn->writeverf_lock);
>
> @@ -1507,6 +1508,7 @@ static __net_exit void nfsd_exit_net(struct net *net)
> nfsd_idmap_shutdown(net);
> nfsd_export_shutdown(net);
> nfsd_netns_free_versions(net_generic(net, nfsd_net_id));
> + nfsd4_leases_net_shutdown(nn);
> }
>
> static struct pernet_operations nfsd_net_ops = {
> diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
> index 57a468ed85c3..cd92f615faa3 100644
> --- a/fs/nfsd/nfsd.h
> +++ b/fs/nfsd/nfsd.h
> @@ -343,6 +343,7 @@ void nfsd_lockd_shutdown(void);
> #define NFSD_COURTESY_CLIENT_TIMEOUT (24 * 60 * 60) /* seconds */
> #define NFSD_CLIENT_MAX_TRIM_PER_RUN 128
> #define NFS4_CLIENTS_PER_GB 1024
> +#define NFSD_CLIENT_SHRINKER_MINTIMEOUT 1 /* seconds */
You don't need this definition any more. I can remove it
when I apply the patch.
Otherwise, these patches look great. I will give a few
more days for more review comments.
> /*
> * The following attributes are currently not supported by the NFSv4 server:
> @@ -498,7 +499,8 @@ extern void unregister_cld_notifier(void);
> extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
> #endif
>
> -extern void nfsd4_init_leases_net(struct nfsd_net *nn);
> +extern int nfsd4_init_leases_net(struct nfsd_net *nn);
> +extern void nfsd4_leases_net_shutdown(struct nfsd_net *nn);
>
> #else /* CONFIG_NFSD_V4 */
> static inline int nfsd4_is_junction(struct dentry *dentry)
> @@ -506,7 +508,8 @@ static inline int nfsd4_is_junction(struct dentry *dentry)
> return 0;
> }
>
> -static inline void nfsd4_init_leases_net(struct nfsd_net *nn) {};
> +static inline int nfsd4_init_leases_net(struct nfsd_net *nn) { return 0; };
> +static inline void nfsd4_leases_net_shutdown(struct nfsd_net *nn) {};
>
> #define register_cld_notifier() 0
> #define unregister_cld_notifier() do { } while(0)
> --
> 2.9.5
>
--
Chuck Lever
next prev parent reply other threads:[~2022-09-14 18:32 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-09-14 15:54 [PATCH v7 0/2] NFSD: memory shrinker for NFSv4 clients Dai Ngo
2022-09-14 15:54 ` [PATCH v7 1/2] NFSD: keep track of the number of courtesy clients in the system Dai Ngo
2022-09-14 15:54 ` [PATCH v7 2/2] NFSD: add shrinker to reap courtesy clients on low memory condition Dai Ngo
2022-09-14 18:32 ` Chuck Lever III [this message]
2022-09-14 18:37 ` dai.ngo
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=0E0CBCE0-0270-4086-91F1-901153A9D2B0@oracle.com \
--to=chuck.lever@oracle.com \
--cc=dai.ngo@oracle.com \
--cc=jlayton@kernel.org \
--cc=linux-nfs@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.