Re: [PATCH V7 mlx5-next 14/15] vfio/mlx5: Use its own PCI reset_done error handler

[Alex Williamson <alex.williamson@redhat.com>]

On Mon, 7 Feb 2022 19:22:15 +0200
Yishai Hadas <yishaih@nvidia.com> wrote:

> Register its own handler for pci_error_handlers.reset_done and update
> state accordingly.
> 
> Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
> Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> ---
>  drivers/vfio/pci/mlx5/main.c | 57 ++++++++++++++++++++++++++++++++++--
>  1 file changed, 55 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c
> index acd205bcff70..63a889210ef3 100644
> --- a/drivers/vfio/pci/mlx5/main.c
> +++ b/drivers/vfio/pci/mlx5/main.c
> @@ -28,9 +28,12 @@
>  struct mlx5vf_pci_core_device {
>  	struct vfio_pci_core_device core_device;
>  	u8 migrate_cap:1;
> +	u8 deferred_reset:1;
>  	/* protect migration state */
>  	struct mutex state_mutex;
>  	enum vfio_device_mig_state mig_state;
> +	/* protect the reset_done flow */
> +	spinlock_t reset_lock;
>  	u16 vhca_id;
>  	struct mlx5_vf_migration_file *resuming_migf;
>  	struct mlx5_vf_migration_file *saving_migf;
> @@ -437,6 +440,25 @@ mlx5vf_pci_step_device_state_locked(struct mlx5vf_pci_core_device *mvdev,
>  	return ERR_PTR(-EINVAL);
>  }
>  
> +/*
> + * This function is called in all state_mutex unlock cases to
> + * handle a 'deferred_reset' if exists.
> + */
> +static void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)
> +{
> +again:
> +	spin_lock(&mvdev->reset_lock);
> +	if (mvdev->deferred_reset) {
> +		mvdev->deferred_reset = false;
> +		spin_unlock(&mvdev->reset_lock);
> +		mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;
> +		mlx5vf_disable_fds(mvdev);
> +		goto again;
> +	}
> +	mutex_unlock(&mvdev->state_mutex);
> +	spin_unlock(&mvdev->reset_lock);
> +}
> +
>  static struct file *
>  mlx5vf_pci_set_device_state(struct vfio_device *vdev,
>  			    enum vfio_device_mig_state new_state)
> @@ -465,7 +487,7 @@ mlx5vf_pci_set_device_state(struct vfio_device *vdev,
>  			break;
>  		}
>  	}
> -	mutex_unlock(&mvdev->state_mutex);
> +	mlx5vf_state_mutex_unlock(mvdev);
>  	return res;
>  }
>  
> @@ -477,10 +499,34 @@ static int mlx5vf_pci_get_device_state(struct vfio_device *vdev,
>  
>  	mutex_lock(&mvdev->state_mutex);
>  	*curr_state = mvdev->mig_state;
> -	mutex_unlock(&mvdev->state_mutex);
> +	mlx5vf_state_mutex_unlock(mvdev);
>  	return 0;

I still can't see why it wouldn't be a both fairly trivial to implement
and a usability improvement if the unlock wrapper returned -EAGAIN on a
deferred reset so we could avoid returning a stale state to the user
and a dead fd in the former case.  Thanks,

Alex