[PATCH v3 0/2] NFSD: recall write delegation on GETATTR conflict

[PATCH v3 0/2] NFSD: recall write delegation on GETATTR conflict

[Dai Ngo]

This patch series adds the recall of write delegation when there is
conflict with a GETATTR and a counter in /proc/net/rpc/nfsd to keep
count of this recall.

Changes from v1:

- add comment for nfsd4_deleg_getattr_conflict
- only wait 30ms for delegation to be returned before returing
  NFS4ERR_DELAY
- fix test robot undeclared NFSD_STATS_WDELEG_GETATTR error

Changes from v2:
- call nfsd_open_break_lease for non-nfs lease with F_WRLCK

[PATCH v3 1/2] NFSD: handle GETATTR conflict with write delegation

[Dai Ngo]

If the GETATTR request on a file that has write delegation in effect
and the request attributes include the change info and size attribute
then the write delegation is recalled. If the delegation is returned
within 30ms then the GETATTR is serviced as normal otherwise the
NFS4ERR_DELAY error is returned for the GETATTR.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/nfs4state.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfs4xdr.c   |  5 +++++
 fs/nfsd/state.h     |  3 +++
 3 files changed, 68 insertions(+)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index b90b74a5e66e..fea78d90ecf7 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -8353,3 +8353,63 @@ nfsd4_get_writestateid(struct nfsd4_compound_state *cstate,
 {
 	get_stateid(cstate, &u->write.wr_stateid);
 }
+
+/**
+ * nfsd4_deleg_getattr_conflict - Trigger recall if GETATTR causes conflict
+ * @rqstp: RPC transaction context
+ * @inode: file to be checked for a conflict
+ *
+ * This function is called when there is a conflict between a write
+ * delegation and a change/size GETATR from another client. The server
+ * must either use the CB_GETATTR to get the current values of the
+ * attributes from the client that hold the delegation or recall the
+ * delegation before replying to the GETATTR. See RFC 8881 section
+ * 18.7.4.
+ *
+ * Returns 0 if there is no conflict; otherwise an nfs_stat
+ * code is returned.
+ */
+__be32
+nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct inode *inode)
+{
+	__be32 status;
+	struct file_lock_context *ctx;
+	struct file_lock *fl;
+	struct nfs4_delegation *dp;
+
+	ctx = locks_inode_context(inode);
+	if (!ctx)
+		return 0;
+	spin_lock(&ctx->flc_lock);
+	list_for_each_entry(fl, &ctx->flc_lease, fl_list) {
+		if (fl->fl_flags == FL_LAYOUT)
+			continue;
+		if (fl->fl_lmops != &nfsd_lease_mng_ops) {
+			/*
+			 * non-nfs lease, if it's a lease with F_RDLCK then
+			 * we are done; there isn't any write delegation
+			 * on this inode
+			 */
+			if (fl->fl_type == F_RDLCK)
+				break;
+			goto break_lease;
+		}
+		if (fl->fl_type == F_WRLCK) {
+			dp = fl->fl_owner;
+			if (dp->dl_recall.cb_clp == *(rqstp->rq_lease_breaker)) {
+				spin_unlock(&ctx->flc_lock);
+				return 0;
+			}
+break_lease:
+			spin_unlock(&ctx->flc_lock);
+			status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ));
+			if (status != nfserr_jukebox ||
+					!nfsd_wait_for_delegreturn(rqstp, inode))
+				return status;
+			return 0;
+		}
+		break;
+	}
+	spin_unlock(&ctx->flc_lock);
+	return 0;
+}
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index b83954fc57e3..4590b893dbc8 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -2970,6 +2970,11 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
 		if (status)
 			goto out;
 	}
+	if (bmval0 & (FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE)) {
+		status = nfsd4_deleg_getattr_conflict(rqstp, d_inode(dentry));
+		if (status)
+			goto out;
+	}
 
 	err = vfs_getattr(&path, &stat,
 			  STATX_BASIC_STATS | STATX_BTIME | STATX_CHANGE_COOKIE,
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index d49d3060ed4f..cbddcf484dba 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -732,4 +732,7 @@ static inline bool try_to_expire_client(struct nfs4_client *clp)
 	cmpxchg(&clp->cl_state, NFSD4_COURTESY, NFSD4_EXPIRABLE);
 	return clp->cl_state == NFSD4_EXPIRABLE;
 }
+
+extern __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp,
+				struct inode *inode);
 #endif   /* NFSD4_STATE_H */
-- 
2.9.5

[PATCH v3 2/2] NFSD: add counter for write delegation recall due to conflict with GETATTR

[Dai Ngo]

Add counter to keep track of how many times write delegations are
recalled due to conflict with GETATTR.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
---
 fs/nfsd/nfs4state.c | 1 +
 fs/nfsd/stats.c     | 2 ++
 fs/nfsd/stats.h     | 7 +++++++
 3 files changed, 10 insertions(+)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 29ed2e72b665..cba27dfa39e8 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -8402,6 +8402,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct inode *inode)
 			}
 break_lease:
 			spin_unlock(&ctx->flc_lock);
+			nfsd_stats_wdeleg_getattr_inc();
 			status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ));
 			if (status != nfserr_jukebox ||
 					!nfsd_wait_for_delegreturn(rqstp, inode))
diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c
index 777e24e5da33..63797635e1c3 100644
--- a/fs/nfsd/stats.c
+++ b/fs/nfsd/stats.c
@@ -65,6 +65,8 @@ static int nfsd_show(struct seq_file *seq, void *v)
 		seq_printf(seq, " %lld",
 			   percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_NFS4_OP(i)]));
 	}
+	seq_printf(seq, "\nwdeleg_getattr %lld",
+		percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]));
 
 	seq_putc(seq, '\n');
 #endif
diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h
index 9b43dc3d9991..cf5524e7ca06 100644
--- a/fs/nfsd/stats.h
+++ b/fs/nfsd/stats.h
@@ -22,6 +22,7 @@ enum {
 	NFSD_STATS_FIRST_NFS4_OP,	/* count of individual nfsv4 operations */
 	NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP,
 #define NFSD_STATS_NFS4_OP(op)	(NFSD_STATS_FIRST_NFS4_OP + (op))
+	NFSD_STATS_WDELEG_GETATTR,	/* count of getattr conflict with wdeleg */
 #endif
 	NFSD_STATS_COUNTERS_NUM
 };
@@ -93,4 +94,10 @@ static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount)
 	percpu_counter_sub(&nn->counter[NFSD_NET_DRC_MEM_USAGE], amount);
 }
 
+#ifdef CONFIG_NFSD_V4
+static inline void nfsd_stats_wdeleg_getattr_inc(void)
+{
+	percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]);
+}
+#endif
 #endif /* _NFSD_STATS_H */
-- 
2.9.5

Re: [PATCH v3 1/2] NFSD: handle GETATTR conflict with write delegation

[Jeff Layton]

On Tue, 2023-05-30 at 19:35 -0700, Dai Ngo wrote:
> If the GETATTR request on a file that has write delegation in effect
> and the request attributes include the change info and size attribute
> then the write delegation is recalled. If the delegation is returned
> within 30ms then the GETATTR is serviced as normal otherwise the
> NFS4ERR_DELAY error is returned for the GETATTR.
> 
> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
> ---
>  fs/nfsd/nfs4state.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  fs/nfsd/nfs4xdr.c   |  5 +++++
>  fs/nfsd/state.h     |  3 +++
>  3 files changed, 68 insertions(+)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index b90b74a5e66e..fea78d90ecf7 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -8353,3 +8353,63 @@ nfsd4_get_writestateid(struct nfsd4_compound_state *cstate,
>  {
>  	get_stateid(cstate, &u->write.wr_stateid);
>  }
> +
> +/**
> + * nfsd4_deleg_getattr_conflict - Trigger recall if GETATTR causes conflict
> + * @rqstp: RPC transaction context
> + * @inode: file to be checked for a conflict
> + *
> + * This function is called when there is a conflict between a write
> + * delegation and a change/size GETATR from another client. The server
> + * must either use the CB_GETATTR to get the current values of the
> + * attributes from the client that hold the delegation or recall the
> + * delegation before replying to the GETATTR. See RFC 8881 section
> + * 18.7.4.
> + *
> + * Returns 0 if there is no conflict; otherwise an nfs_stat
> + * code is returned.
> + */
> +__be32
> +nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct inode *inode)
> +{
> +	__be32 status;
> +	struct file_lock_context *ctx;
> +	struct file_lock *fl;
> +	struct nfs4_delegation *dp;
> +
> +	ctx = locks_inode_context(inode);
> +	if (!ctx)
> +		return 0;
> +	spin_lock(&ctx->flc_lock);
> +	list_for_each_entry(fl, &ctx->flc_lease, fl_list) {
> +		if (fl->fl_flags == FL_LAYOUT)
> +			continue;
> +		if (fl->fl_lmops != &nfsd_lease_mng_ops) {
> +			/*
> +			 * non-nfs lease, if it's a lease with F_RDLCK then
> +			 * we are done; there isn't any write delegation
> +			 * on this inode
> +			 */
> +			if (fl->fl_type == F_RDLCK)
> +				break;
> +			goto break_lease;
> +		}
> +		if (fl->fl_type == F_WRLCK) {
> +			dp = fl->fl_owner;
> +			if (dp->dl_recall.cb_clp == *(rqstp->rq_lease_breaker)) {
> +				spin_unlock(&ctx->flc_lock);
> +				return 0;
> +			}
> +break_lease:
> +			spin_unlock(&ctx->flc_lock);
> +			status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ));
> +			if (status != nfserr_jukebox ||
> +					!nfsd_wait_for_delegreturn(rqstp, inode))
> +				return status;
> +			return 0;
> +		}
> +		break;
> +	}
> +	spin_unlock(&ctx->flc_lock);
> +	return 0;
> +}

I think you have the logic right now, but this function kind of
resembles spaghetti. I'll add a reviewed-by, but it might be good to
simplify this function somehow later.

> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index b83954fc57e3..4590b893dbc8 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -2970,6 +2970,11 @@ nfsd4_encode_fattr(struct xdr_stream *xdr, struct svc_fh *fhp,
>  		if (status)
>  			goto out;
>  	}
> +	if (bmval0 & (FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE)) {
> +		status = nfsd4_deleg_getattr_conflict(rqstp, d_inode(dentry));
> +		if (status)
> +			goto out;
> +	}
>  
>  	err = vfs_getattr(&path, &stat,
>  			  STATX_BASIC_STATS | STATX_BTIME | STATX_CHANGE_COOKIE,
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index d49d3060ed4f..cbddcf484dba 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -732,4 +732,7 @@ static inline bool try_to_expire_client(struct nfs4_client *clp)
>  	cmpxchg(&clp->cl_state, NFSD4_COURTESY, NFSD4_EXPIRABLE);
>  	return clp->cl_state == NFSD4_EXPIRABLE;
>  }
> +
> +extern __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp,
> +				struct inode *inode);
>  #endif   /* NFSD4_STATE_H */

Reviewed-by: Jeff Layton <jlayton@kernel.org>

Re: [PATCH v3 2/2] NFSD: add counter for write delegation recall due to conflict with GETATTR

[Jeff Layton]

On Tue, 2023-05-30 at 19:35 -0700, Dai Ngo wrote:
> Add counter to keep track of how many times write delegations are
> recalled due to conflict with GETATTR.
> 
> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
> ---
>  fs/nfsd/nfs4state.c | 1 +
>  fs/nfsd/stats.c     | 2 ++
>  fs/nfsd/stats.h     | 7 +++++++
>  3 files changed, 10 insertions(+)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 29ed2e72b665..cba27dfa39e8 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -8402,6 +8402,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct inode *inode)
>  			}
>  break_lease:
>  			spin_unlock(&ctx->flc_lock);
> +			nfsd_stats_wdeleg_getattr_inc();
>  			status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ));
>  			if (status != nfserr_jukebox ||
>  					!nfsd_wait_for_delegreturn(rqstp, inode))
> diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c
> index 777e24e5da33..63797635e1c3 100644
> --- a/fs/nfsd/stats.c
> +++ b/fs/nfsd/stats.c
> @@ -65,6 +65,8 @@ static int nfsd_show(struct seq_file *seq, void *v)
>  		seq_printf(seq, " %lld",
>  			   percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_NFS4_OP(i)]));
>  	}
> +	seq_printf(seq, "\nwdeleg_getattr %lld",
> +		percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]));
>  
>  	seq_putc(seq, '\n');
>  #endif
> diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h
> index 9b43dc3d9991..cf5524e7ca06 100644
> --- a/fs/nfsd/stats.h
> +++ b/fs/nfsd/stats.h
> @@ -22,6 +22,7 @@ enum {
>  	NFSD_STATS_FIRST_NFS4_OP,	/* count of individual nfsv4 operations */
>  	NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP,
>  #define NFSD_STATS_NFS4_OP(op)	(NFSD_STATS_FIRST_NFS4_OP + (op))
> +	NFSD_STATS_WDELEG_GETATTR,	/* count of getattr conflict with wdeleg */
>  #endif
>  	NFSD_STATS_COUNTERS_NUM
>  };
> @@ -93,4 +94,10 @@ static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount)
>  	percpu_counter_sub(&nn->counter[NFSD_NET_DRC_MEM_USAGE], amount);
>  }
>  
> +#ifdef CONFIG_NFSD_V4
> +static inline void nfsd_stats_wdeleg_getattr_inc(void)
> +{
> +	percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]);
> +}
> +#endif
>  #endif /* _NFSD_STATS_H */

Personally, I think it would still be simpler to just do a CB_GETATTR.
We are issuing a callback in either case, but recalling the delegation
seems like a less optimal outcome.

Still for an interim step, this is fine...

Reviewed-by: Jeff Layton <jlayton@kernel.org>

Re: [PATCH v3 2/2] NFSD: add counter for write delegation recall due to conflict with GETATTR

[Chuck Lever III]

> On May 31, 2023, at 6:12 AM, Jeff Layton <jlayton@kernel.org> wrote:
> 
> On Tue, 2023-05-30 at 19:35 -0700, Dai Ngo wrote:
>> Add counter to keep track of how many times write delegations are
>> recalled due to conflict with GETATTR.
>> 
>> Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
>> ---
>> fs/nfsd/nfs4state.c | 1 +
>> fs/nfsd/stats.c     | 2 ++
>> fs/nfsd/stats.h     | 7 +++++++
>> 3 files changed, 10 insertions(+)
>> 
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index 29ed2e72b665..cba27dfa39e8 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -8402,6 +8402,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct inode *inode)
>> }
>> break_lease:
>> spin_unlock(&ctx->flc_lock);
>> + nfsd_stats_wdeleg_getattr_inc();
>> status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ));
>> if (status != nfserr_jukebox ||
>> !nfsd_wait_for_delegreturn(rqstp, inode))
>> diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c
>> index 777e24e5da33..63797635e1c3 100644
>> --- a/fs/nfsd/stats.c
>> +++ b/fs/nfsd/stats.c
>> @@ -65,6 +65,8 @@ static int nfsd_show(struct seq_file *seq, void *v)
>> seq_printf(seq, " %lld",
>>    percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_NFS4_OP(i)]));
>> }
>> + seq_printf(seq, "\nwdeleg_getattr %lld",
>> + percpu_counter_sum_positive(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]));
>> 
>> seq_putc(seq, '\n');
>> #endif
>> diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h
>> index 9b43dc3d9991..cf5524e7ca06 100644
>> --- a/fs/nfsd/stats.h
>> +++ b/fs/nfsd/stats.h
>> @@ -22,6 +22,7 @@ enum {
>> NFSD_STATS_FIRST_NFS4_OP, /* count of individual nfsv4 operations */
>> NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP,
>> #define NFSD_STATS_NFS4_OP(op) (NFSD_STATS_FIRST_NFS4_OP + (op))
>> + NFSD_STATS_WDELEG_GETATTR, /* count of getattr conflict with wdeleg */
>> #endif
>> NFSD_STATS_COUNTERS_NUM
>> };
>> @@ -93,4 +94,10 @@ static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount)
>> percpu_counter_sub(&nn->counter[NFSD_NET_DRC_MEM_USAGE], amount);
>> }
>> 
>> +#ifdef CONFIG_NFSD_V4
>> +static inline void nfsd_stats_wdeleg_getattr_inc(void)
>> +{
>> + percpu_counter_inc(&nfsdstats.counter[NFSD_STATS_WDELEG_GETATTR]);
>> +}
>> +#endif
>> #endif /* _NFSD_STATS_H */
> 
> Personally, I think it would still be simpler to just do a CB_GETATTR.
> We are issuing a callback in either case, but recalling the delegation
> seems like a less optimal outcome.

An NFS server has to support either CB_RECALL, or it has to
support CB_GETATTR and fall back to CB_RECALL. Some clients
might not support CB_GETATTR, after all.

Either way NFSD has to handle CB_RECALL properly in this case.


> Still for an interim step, this is fine...
> 
> Reviewed-by: Jeff Layton <jlayton@kernel.org>


--
Chuck Lever

Re: [PATCH v3 0/2] NFSD: recall write delegation on GETATTR conflict

[Chuck Lever III]

> On May 30, 2023, at 10:35 PM, Dai Ngo <dai.ngo@oracle.com> wrote:
> 
> This patch series adds the recall of write delegation when there is
> conflict with a GETATTR and a counter in /proc/net/rpc/nfsd to keep
> count of this recall.
> 
> Changes from v1:
> 
> - add comment for nfsd4_deleg_getattr_conflict
> - only wait 30ms for delegation to be returned before returing
>  NFS4ERR_DELAY
> - fix test robot undeclared NFSD_STATS_WDELEG_GETATTR error
> 
> Changes from v2:
> - call nfsd_open_break_lease for non-nfs lease with F_WRLCK
> 

I've applied v3 of this series to nfsd-next for merge into v6.5.
Thanks!


--
Chuck Lever