Re: [PATCH 1/2] kernfs: replace the mutex in kernfs_iop_permission with a rwlock

[Greg KH <gregkh@linuxfoundation.org>]

On Wed, Dec 02, 2020 at 10:58:36PM +0800, Fox Chen wrote:
> A big global mutex in kernfs_iop_permission will significanly drag
> system performance when processes concurrently open files
> on kernfs in Big machines(with >= 16 cpu cores).
> 
> This patch replace the big mutex with a rwlock specifically for
> protecting kernfs_node->iattribute. So that kernfs_iop_permission
> can perform concurrently.
> 
> Signed-off-by: Fox Chen <foxhlchen@gmail.com>
> ---
>  fs/kernfs/inode.c      | 16 ++++++++--------
>  include/linux/kernfs.h |  1 +
>  2 files changed, 9 insertions(+), 8 deletions(-)
> 
> diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c
> index fc2469a20fed..c8c2ea669e6d 100644
> --- a/fs/kernfs/inode.c
> +++ b/fs/kernfs/inode.c
> @@ -106,9 +106,9 @@ int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr)
>  {
>  	int ret;
>  
> -	mutex_lock(&kernfs_mutex);
> +	write_lock(&kn->iattr_rwlock);
>  	ret = __kernfs_setattr(kn, iattr);
> -	mutex_unlock(&kernfs_mutex);
> +	write_unlock(&kn->iattr_rwlock);
>  	return ret;
>  }
>  
> @@ -121,7 +121,7 @@ int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr)
>  	if (!kn)
>  		return -EINVAL;
>  
> -	mutex_lock(&kernfs_mutex);
> +	write_lock(&kn->iattr_rwlock);
>  	error = setattr_prepare(dentry, iattr);
>  	if (error)
>  		goto out;
> @@ -134,7 +134,7 @@ int kernfs_iop_setattr(struct dentry *dentry, struct iattr *iattr)
>  	setattr_copy(inode, iattr);
>  
>  out:
> -	mutex_unlock(&kernfs_mutex);
> +	write_unlock(&kn->iattr_rwlock);
>  	return error;
>  }
>  
> @@ -189,9 +189,9 @@ int kernfs_iop_getattr(const struct path *path, struct kstat *stat,
>  	struct inode *inode = d_inode(path->dentry);
>  	struct kernfs_node *kn = inode->i_private;
>  
> -	mutex_lock(&kernfs_mutex);
> +	read_lock(&kn->iattr_rwlock);
>  	kernfs_refresh_inode(kn, inode);
> -	mutex_unlock(&kernfs_mutex);
> +	read_unlock(&kn->iattr_rwlock);
>  
>  	generic_fillattr(inode, stat);
>  	return 0;
> @@ -281,9 +281,9 @@ int kernfs_iop_permission(struct inode *inode, int mask)
>  
>  	kn = inode->i_private;
>  
> -	mutex_lock(&kernfs_mutex);
> +	read_lock(&kn->iattr_rwlock);
>  	kernfs_refresh_inode(kn, inode);
> -	mutex_unlock(&kernfs_mutex);
> +	read_unlock(&kn->iattr_rwlock);
>  
>  	return generic_permission(inode, mask);
>  }
> diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h
> index 89f6a4214a70..545cdb39b34b 100644
> --- a/include/linux/kernfs.h
> +++ b/include/linux/kernfs.h
> @@ -156,6 +156,7 @@ struct kernfs_node {
>  	unsigned short		flags;
>  	umode_t			mode;
>  	struct kernfs_iattrs	*iattr;
> +	rwlock_t		iattr_rwlock;
>  };

Don't you have to call rwlock_init() somewhere in order to properly
initialize the lock?  Did you try running this under lockdep to ensure
that there are no issues?

thanks,

greg k-h