[PATCH v2 0/2] limit the number of v4 clients to 1024 per 1GB of system memory

[PATCH v2 0/2] limit the number of v4 clients to 1024 per 1GB of system memory

[Dai Ngo]

This patch series enforces a limit on the number of v4 clients allowed
in the system. With Courteous server support there are potentially a
lots courtesy clients exist in the system that use up memory resource
preventing them to be used by other components in the system. Also
without a limit on the number of clients, the number of clients can
grow to a very large number even for system with small memory configuration
eventually render the system into an unusable state.

v2:
. move all defines to nfsd.h
. replace unsigned int nfs4_max_client to int
. kick start laundromat in alloc_client when max client reached.
. restyle compute of maxreap in nfs4_get_client_reaplist to oneline.
. redo enforce of maxreap in nfs4_get_client_reaplist for readability
. use bit-wise interger to compute usable memory in nfsd_init_net.
. replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB.
. use all memory, including high mem, to compute max client.

---

Dai Ngo (2):
      NFSD: keep track of the number of v4 clients in the system
      NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

 fs/nfsd/netns.h     |  3 +++
 fs/nfsd/nfs4state.c | 28 ++++++++++++++++++++--------
 fs/nfsd/nfsctl.c    |  8 ++++++++
 fs/nfsd/nfsd.h      |  2 ++
 4 files changed, 33 insertions(+), 8 deletions(-)
--
Dai Ngo

[PATCH v2 1/2] NFSD: keep track of the number of v4 clients in the system

[Dai Ngo]

Add counter nfs4_client_count to keep track of the total number
of v4 clients, including courtesy clients, in the system.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/netns.h     | 2 ++
 fs/nfsd/nfs4state.c | 8 ++++++--
 fs/nfsd/nfsctl.c    | 2 ++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 1b1a962a1804..ce864f001a3e 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -189,6 +189,8 @@ struct nfsd_net {
 	struct nfsd_fcache_disposal *fcache_disposal;
 
 	siphash_key_t		siphash_key;
+
+	atomic_t		nfs4_client_count;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 9409a0dc1b76..30e16d9e8657 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -2053,7 +2053,8 @@ STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn)
  * This type of memory management is somewhat inefficient, but we use it
  * anyway since SETCLIENTID is not a common operation.
  */
-static struct nfs4_client *alloc_client(struct xdr_netobj name)
+static struct nfs4_client *alloc_client(struct xdr_netobj name,
+				struct nfsd_net *nn)
 {
 	struct nfs4_client *clp;
 	int i;
@@ -2076,6 +2077,7 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 	atomic_set(&clp->cl_rpc_users, 0);
 	clp->cl_cb_state = NFSD4_CB_UNKNOWN;
 	clp->cl_state = NFSD4_ACTIVE;
+	atomic_inc(&nn->nfs4_client_count);
 	atomic_set(&clp->cl_delegs_in_recall, 0);
 	INIT_LIST_HEAD(&clp->cl_idhash);
 	INIT_LIST_HEAD(&clp->cl_openowners);
@@ -2187,6 +2189,7 @@ __destroy_client(struct nfs4_client *clp)
 	struct nfs4_openowner *oo;
 	struct nfs4_delegation *dp;
 	struct list_head reaplist;
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
 	INIT_LIST_HEAD(&reaplist);
 	spin_lock(&state_lock);
@@ -2226,6 +2229,7 @@ __destroy_client(struct nfs4_client *clp)
 	nfsd4_shutdown_callback(clp);
 	if (clp->cl_cb_conn.cb_xprt)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
+	atomic_add_unless(&nn->nfs4_client_count, -1, 0);
 	free_client(clp);
 	wake_up_all(&expiry_wq);
 }
@@ -2848,7 +2852,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name,
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	struct dentry *dentries[ARRAY_SIZE(client_files)];
 
-	clp = alloc_client(name);
+	clp = alloc_client(name, nn);
 	if (clp == NULL)
 		return NULL;
 
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 0621c2faf242..547f4c4b9668 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1487,6 +1487,8 @@ static __net_init int nfsd_init_net(struct net *net)
 	get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
 	seqlock_init(&nn->writeverf_lock);
 
+	atomic_set(&nn->nfs4_client_count, 0);
+
 	return 0;
 
 out_drc_error:
-- 
2.9.5

[PATCH v2 2/2] NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

[Dai Ngo]

Currently there is no limit on how many v4 clients are supported
by the system. This can be a problem in systems with small memory
configuration to function properly when a very large number of
clients exist that creates memory shortage conditions.

This patch enforces a limit of 1024 NFSv4 clients, including courtesy
clients, per 1GB of system memory.  When the number of the clients
reaches the limit, requests that create new clients are returned
with NFS4ERR_DELAY and the laundromat is kicked start to trim old
clients. Due to the overhead of the upcall to remove the client
record, the maximun number of clients the laundromat removes on
each run is limited to 128. This is done to ensure the laundromat
can still process the other tasks in a timely manner.

Since there is now a limit of the number of clients, the 24-hr
idle time limit of courtesy client is no longer needed and was
removed.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/netns.h     |  1 +
 fs/nfsd/nfs4state.c | 20 ++++++++++++++------
 fs/nfsd/nfsctl.c    |  6 ++++++
 fs/nfsd/nfsd.h      |  2 ++
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index ce864f001a3e..ffe17743cc74 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -191,6 +191,7 @@ struct nfsd_net {
 	siphash_key_t		siphash_key;
 
 	atomic_t		nfs4_client_count;
+	int			nfs4_max_clients;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 30e16d9e8657..19807f7f618d 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -2059,6 +2059,10 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name,
 	struct nfs4_client *clp;
 	int i;
 
+	if (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) {
+		mod_delayed_work(laundry_wq, &nn->laundromat_work, 0);
+		return NULL;
+	}
 	clp = kmem_cache_zalloc(client_slab, GFP_KERNEL);
 	if (clp == NULL)
 		return NULL;
@@ -5796,9 +5800,12 @@ static void
 nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 				struct laundry_time *lt)
 {
+	unsigned int maxreap, reapcnt = 0;
 	struct list_head *pos, *next;
 	struct nfs4_client *clp;
 
+	maxreap = (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) ?
+			NFSD_CLIENT_MAX_TRIM_PER_RUN : 0;
 	INIT_LIST_HEAD(reaplist);
 	spin_lock(&nn->client_lock);
 	list_for_each_safe(pos, next, &nn->client_lru) {
@@ -5809,14 +5816,15 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 			break;
 		if (!atomic_read(&clp->cl_rpc_users))
 			clp->cl_state = NFSD4_COURTESY;
-		if (!client_has_state(clp) ||
-				ktime_get_boottime_seconds() >=
-				(clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
+		if (!client_has_state(clp))
 			goto exp_client;
-		if (nfs4_anylock_blockers(clp)) {
+		if (!nfs4_anylock_blockers(clp))
+			if (reapcnt >= maxreap)
+				continue;
 exp_client:
-			if (!mark_client_expired_locked(clp))
-				list_add(&clp->cl_lru, reaplist);
+		if (!mark_client_expired_locked(clp)) {
+			list_add(&clp->cl_lru, reaplist);
+			reapcnt++;
 		}
 	}
 	spin_unlock(&nn->client_lock);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 547f4c4b9668..bbd251da86e4 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -1463,6 +1463,8 @@ static __net_init int nfsd_init_net(struct net *net)
 {
 	int retval;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+	u64 max_clients;
+	struct sysinfo si;
 
 	retval = nfsd_export_init(net);
 	if (retval)
@@ -1488,6 +1490,10 @@ static __net_init int nfsd_init_net(struct net *net)
 	seqlock_init(&nn->writeverf_lock);
 
 	atomic_set(&nn->nfs4_client_count, 0);
+	si_meminfo(&si);
+	max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024);
+	max_clients *= NFS4_CLIENTS_PER_GB;
+	nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);
 
 	return 0;
 
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 847b482155ae..bbada18225b1 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -341,6 +341,8 @@ void		nfsd_lockd_shutdown(void);
 
 #define NFSD_LAUNDROMAT_MINTIMEOUT      1   /* seconds */
 #define	NFSD_COURTESY_CLIENT_TIMEOUT	(24 * 60 * 60)	/* seconds */
+#define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
+#define	NFS4_CLIENTS_PER_GB		1024
 
 /*
  * The following attributes are currently not supported by the NFSv4 server:
-- 
2.9.5

Re: [PATCH v2 0/2] limit the number of v4 clients to 1024 per 1GB of system memory

[Chuck Lever III]

> On Jul 14, 2022, at 12:17 PM, Dai Ngo <dai.ngo@oracle.com> wrote:
> 
> This patch series enforces a limit on the number of v4 clients allowed
> in the system. With Courteous server support there are potentially a
> lots courtesy clients exist in the system that use up memory resource
> preventing them to be used by other components in the system. Also
> without a limit on the number of clients, the number of clients can
> grow to a very large number even for system with small memory configuration
> eventually render the system into an unusable state.
> 
> v2:
> . move all defines to nfsd.h
> . replace unsigned int nfs4_max_client to int
> . kick start laundromat in alloc_client when max client reached.
> . restyle compute of maxreap in nfs4_get_client_reaplist to oneline.
> . redo enforce of maxreap in nfs4_get_client_reaplist for readability
> . use bit-wise interger to compute usable memory in nfsd_init_net.
> . replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB.
> . use all memory, including high mem, to compute max client.

Hello Dai, I applied these two to NFSD's for-next branch for early
adopters and other testing and review.


> ---
> 
> Dai Ngo (2):
>      NFSD: keep track of the number of v4 clients in the system
>      NFSD: limit the number of v4 clients to 1024 per 1GB of system memory
> 
> fs/nfsd/netns.h     |  3 +++
> fs/nfsd/nfs4state.c | 28 ++++++++++++++++++++--------
> fs/nfsd/nfsctl.c    |  8 ++++++++
> fs/nfsd/nfsd.h      |  2 ++
> 4 files changed, 33 insertions(+), 8 deletions(-)
> --
> Dai Ngo
> 

--
Chuck Lever

Re: [PATCH v2 0/2] limit the number of v4 clients to 1024 per 1GB of system memory

[dai.ngo]

On 7/14/22 10:18 AM, Chuck Lever III wrote:
>
>> On Jul 14, 2022, at 12:17 PM, Dai Ngo <dai.ngo@oracle.com> wrote:
>>
>> This patch series enforces a limit on the number of v4 clients allowed
>> in the system. With Courteous server support there are potentially a
>> lots courtesy clients exist in the system that use up memory resource
>> preventing them to be used by other components in the system. Also
>> without a limit on the number of clients, the number of clients can
>> grow to a very large number even for system with small memory configuration
>> eventually render the system into an unusable state.
>>
>> v2:
>> . move all defines to nfsd.h
>> . replace unsigned int nfs4_max_client to int
>> . kick start laundromat in alloc_client when max client reached.
>> . restyle compute of maxreap in nfs4_get_client_reaplist to oneline.
>> . redo enforce of maxreap in nfs4_get_client_reaplist for readability
>> . use bit-wise interger to compute usable memory in nfsd_init_net.
>> . replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB.
>> . use all memory, including high mem, to compute max client.
> Hello Dai, I applied these two to NFSD's for-next branch for early
> adopters and other testing and review.

Thank you Chuck,

-Dai

>
>
>> ---
>>
>> Dai Ngo (2):
>>       NFSD: keep track of the number of v4 clients in the system
>>       NFSD: limit the number of v4 clients to 1024 per 1GB of system memory
>>
>> fs/nfsd/netns.h     |  3 +++
>> fs/nfsd/nfs4state.c | 28 ++++++++++++++++++++--------
>> fs/nfsd/nfsctl.c    |  8 ++++++++
>> fs/nfsd/nfsd.h      |  2 ++
>> 4 files changed, 33 insertions(+), 8 deletions(-)
>> --
>> Dai Ngo
>>
> --
> Chuck Lever
>
>
>

Re: [PATCH v2 0/2] limit the number of v4 clients to 1024 per 1GB of system memory

[dai.ngo]

On 7/14/22 10:56 AM, dai.ngo@oracle.com wrote:
>
> On 7/14/22 10:18 AM, Chuck Lever III wrote:
>>
>>> On Jul 14, 2022, at 12:17 PM, Dai Ngo <dai.ngo@oracle.com> wrote:
>>>
>>> This patch series enforces a limit on the number of v4 clients allowed
>>> in the system. With Courteous server support there are potentially a
>>> lots courtesy clients exist in the system that use up memory resource
>>> preventing them to be used by other components in the system. Also
>>> without a limit on the number of clients, the number of clients can
>>> grow to a very large number even for system with small memory 
>>> configuration
>>> eventually render the system into an unusable state.
>>>
>>> v2:
>>> . move all defines to nfsd.h
>>> . replace unsigned int nfs4_max_client to int
>>> . kick start laundromat in alloc_client when max client reached.
>>> . restyle compute of maxreap in nfs4_get_client_reaplist to oneline.
>>> . redo enforce of maxreap in nfs4_get_client_reaplist for readability
>>> . use bit-wise interger to compute usable memory in nfsd_init_net.
>>> . replace NFS4_MAX_CLIENTS_PER_4GB to NFS4_CLIENTS_PER_GB.
>>> . use all memory, including high mem, to compute max client.
>> Hello Dai, I applied these two to NFSD's for-next branch for early
>> adopters and other testing and review.
>
> Thank you Chuck,

I forgot to mention that I will prepare the patches for
pynfs to deal with NFS4ERR_DELAY on SETCLIENIID and
EXCHANGE_ID.

-Dai

>
> -Dai
>
>>
>>
>>> ---
>>>
>>> Dai Ngo (2):
>>>       NFSD: keep track of the number of v4 clients in the system
>>>       NFSD: limit the number of v4 clients to 1024 per 1GB of system 
>>> memory
>>>
>>> fs/nfsd/netns.h     |  3 +++
>>> fs/nfsd/nfs4state.c | 28 ++++++++++++++++++++--------
>>> fs/nfsd/nfsctl.c    |  8 ++++++++
>>> fs/nfsd/nfsd.h      |  2 ++
>>> 4 files changed, 33 insertions(+), 8 deletions(-)
>>> -- 
>>> Dai Ngo
>>>
>> -- 
>> Chuck Lever
>>
>>
>>

Re: [PATCH v2 2/2] NFSD: limit the number of v4 clients to 1024 per 1GB of system memory

[Geert Uytterhoeven]

 	Hi Dai,

On Thu, 14 Jul 2022, Dai Ngo wrote:
> Currently there is no limit on how many v4 clients are supported
> by the system. This can be a problem in systems with small memory
> configuration to function properly when a very large number of
> clients exist that creates memory shortage conditions.
>
> This patch enforces a limit of 1024 NFSv4 clients, including courtesy
> clients, per 1GB of system memory.  When the number of the clients
> reaches the limit, requests that create new clients are returned
> with NFS4ERR_DELAY and the laundromat is kicked start to trim old
> clients. Due to the overhead of the upcall to remove the client
> record, the maximun number of clients the laundromat removes on
> each run is limited to 128. This is done to ensure the laundromat
> can still process the other tasks in a timely manner.
>
> Since there is now a limit of the number of clients, the 24-hr
> idle time limit of courtesy client is no longer needed and was
> removed.
>
> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>

Thanks for your patch, which is now commit 05eaba9bd8c06580 ("NFSD:
limit the number of v4 clients to 1024 per 1GB of system memory")
in next-20220715.

noreply@ellerman.id.au reports:

     fs/nfsd/nfsctl.c:1504:24: error: 'NFS4_CLIENTS_PER_GB' undeclared (first use in this function)

> --- a/fs/nfsd/nfsctl.c
> +++ b/fs/nfsd/nfsctl.c
> @@ -1463,6 +1463,8 @@ static __net_init int nfsd_init_net(struct net *net)
> {
> 	int retval;
> 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
> +	u64 max_clients;
> +	struct sysinfo si;
>
> 	retval = nfsd_export_init(net);
> 	if (retval)
> @@ -1488,6 +1490,10 @@ static __net_init int nfsd_init_net(struct net *net)

Not protected by #ifdef CONFIG_NFSD_V4:

> 	seqlock_init(&nn->writeverf_lock);
>
> 	atomic_set(&nn->nfs4_client_count, 0);
> +	si_meminfo(&si);
> +	max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024);
> +	max_clients *= NFS4_CLIENTS_PER_GB;
> +	nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);
>
> 	return 0;
>
> diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
> index 847b482155ae..bbada18225b1 100644
> --- a/fs/nfsd/nfsd.h
> +++ b/fs/nfsd/nfsd.h
> @@ -341,6 +341,8 @@ void		nfsd_lockd_shutdown(void);

Protected by #ifdef CONFIG_NFSD_V4:

>
> #define NFSD_LAUNDROMAT_MINTIMEOUT      1   /* seconds */
> #define	NFSD_COURTESY_CLIENT_TIMEOUT	(24 * 60 * 60)	/* seconds */
> +#define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
> +#define	NFS4_CLIENTS_PER_GB		1024
>
> /*
>  * The following attributes are currently not supported by the NFSv4 server:
> -- 
> 2.9.5

Gr{oetje,eeting}s,

 						Geert

--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert@linux-m68k.org

In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
 							    -- Linus Torvalds