[PATCH v2 1/3] virtio: update reset callback to return status

[PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

The reset device operation, usually is an operation that might fail from
various reasons. For example, the controller might be in a bad state and
can't answer to any request. Usually, the paravirt SW based virtio
devices always succeed in reset operation but this is not the case for
HW based virtio devices.

This commit is also a preparation for adding a timeout mechanism for
resetting virtio devices.

Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
---

changes from v1:
 - update virtio_ccw.c (Cornelia)
 - update virtio_uml.c
 - update mlxbf-tmfifo.c

---
 arch/um/drivers/virtio_uml.c             |  4 +++-
 drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
 drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
 drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
 drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
 drivers/virtio/virtio_mmio.c             |  3 ++-
 drivers/virtio/virtio_pci_legacy.c       |  4 +++-
 drivers/virtio/virtio_pci_modern.c       |  3 ++-
 drivers/virtio/virtio_vdpa.c             |  4 +++-
 include/linux/virtio_config.h            |  5 +++--
 10 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
index 91ddf74ca888..b6e66265ed32 100644
--- a/arch/um/drivers/virtio_uml.c
+++ b/arch/um/drivers/virtio_uml.c
@@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device *vdev, u8 status)
 	vu_dev->status = status;
 }
 
-static void vu_reset(struct virtio_device *vdev)
+static int vu_reset(struct virtio_device *vdev)
 {
 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 
 	vu_dev->status = 0;
+
+	return 0;
 }
 
 static void vu_del_vq(struct virtqueue *vq)
diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
index bbc4e71a16ff..c192b8ac5d9e 100644
--- a/drivers/platform/mellanox/mlxbf-tmfifo.c
+++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
@@ -980,11 +980,13 @@ static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
 }
 
 /* Reset the device. Not much here for now. */
-static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
+static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
 {
 	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
 
 	tm_vdev->status = 0;
+
+	return 0;
 }
 
 /* Read the value of a configuration field. */
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index 0cc617f76068..ca9573c62c3d 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status)
 	dev_dbg(&vdev->dev, "status: %d\n", status);
 }
 
-static void rproc_virtio_reset(struct virtio_device *vdev)
+static int rproc_virtio_reset(struct virtio_device *vdev)
 {
 	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
 	struct fw_rsc_vdev *rsc;
@@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
 
 	rsc->status = 0;
 	dev_dbg(&vdev->dev, "reset !\n");
+
+	return 0;
 }
 
 /* provide the vdev features as retrieved from the firmware */
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 54e686dca6de..52b32555e746 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 	return ret;
 }
 
-static void virtio_ccw_reset(struct virtio_device *vdev)
+static int virtio_ccw_reset(struct virtio_device *vdev)
 {
 	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
 	struct ccw1 *ccw;
+	int ret;
 
 	ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
 	if (!ccw)
-		return;
+		return -ENOMEM;
 
 	/* Zero status bits. */
 	vcdev->dma_area->status = 0;
@@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
 	ccw->flags = 0;
 	ccw->count = 0;
 	ccw->cda = 0;
-	ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
+	ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
 	ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
+
+	return ret;
 }
 
 static u64 virtio_ccw_get_features(struct virtio_device *vdev)
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index 4b15c00c0a0a..ddbfd5b5f3bd 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device *dev)
 	/* Assign a unique device index and hence name. */
 	err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
 	if (err < 0)
-		goto out;
+		goto out_err;
 
 	dev->index = err;
 	dev_set_name(&dev->dev, "virtio%u", dev->index);
@@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device *dev)
 
 	/* We always start by resetting the device, in case a previous
 	 * driver messed it up.  This also tests that code path a little. */
-	dev->config->reset(dev);
+	err = dev->config->reset(dev);
+	if (err)
+		goto out_ida;
 
 	/* Acknowledge that we've seen the device. */
 	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
@@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device *dev)
 	 */
 	err = device_add(&dev->dev);
 	if (err)
-		ida_simple_remove(&virtio_index_ida, dev->index);
-out:
-	if (err)
-		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
+		goto out_ida;
+
+	return 0;
+
+out_ida:
+	ida_simple_remove(&virtio_index_ida, dev->index);
+out_err:
+	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
 	return err;
 }
 EXPORT_SYMBOL_GPL(register_virtio_device);
@@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
 
 	/* We always start by resetting the device, in case a previous
 	 * driver messed it up. */
-	dev->config->reset(dev);
+	ret = dev->config->reset(dev);
+	if (ret)
+		goto err;
 
 	/* Acknowledge that we've seen the device. */
 	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 56128b9c46eb..12b8f048c48d 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)
 	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
 }
 
-static void vm_reset(struct virtio_device *vdev)
+static int vm_reset(struct virtio_device *vdev)
 {
 	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
 
 	/* 0 status means a reset. */
 	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
+	return 0;
 }
 
 
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
index d62e9835aeec..0b5d95e3efa1 100644
--- a/drivers/virtio/virtio_pci_legacy.c
+++ b/drivers/virtio/virtio_pci_legacy.c
@@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
 	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 }
 
-static void vp_reset(struct virtio_device *vdev)
+static int vp_reset(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	/* 0 status means a reset. */
@@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
 	ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 	/* Flush pending VQ/configuration callbacks. */
 	vp_synchronize_vectors(vdev);
+
+	return 0;
 }
 
 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index fbd4ebc00eb6..cc3412a96a17 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
 	vp_modern_set_status(&vp_dev->mdev, status);
 }
 
-static void vp_reset(struct virtio_device *vdev)
+static int vp_reset(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
@@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
 		msleep(1);
 	/* Flush pending VQ/configuration callbacks. */
 	vp_synchronize_vectors(vdev);
+	return 0;
 }
 
 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index e28acf482e0c..5fd4e627a9b0 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
 	return ops->set_status(vdpa, status);
 }
 
-static void virtio_vdpa_reset(struct virtio_device *vdev)
+static int virtio_vdpa_reset(struct virtio_device *vdev)
 {
 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 
 	vdpa_reset(vdpa);
+
+	return 0;
 }
 
 static bool virtio_vdpa_notify(struct virtqueue *vq)
diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 8519b3ae5d52..d2b0f1699a75 100644
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -44,9 +44,10 @@ struct virtio_shm_region {
  *	status: the new status byte
  * @reset: reset the device
  *	vdev: the virtio device
- *	After this, status and feature negotiation must be done again
+ *	Upon success, status and feature negotiation must be done again
  *	Device must not be reset from its vq/config callbacks, or in
  *	parallel with being added/removed.
+ *	Returns 0 on success or error status.
  * @find_vqs: find virtqueues and instantiate them.
  *	vdev: the virtio_device
  *	nvqs: the number of virtqueues to find
@@ -82,7 +83,7 @@ struct virtio_config_ops {
 	u32 (*generation)(struct virtio_device *vdev);
 	u8 (*get_status)(struct virtio_device *vdev);
 	void (*set_status)(struct virtio_device *vdev, u8 status);
-	void (*reset)(struct virtio_device *vdev);
+	int (*reset)(struct virtio_device *vdev);
 	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
 			struct virtqueue *vqs[], vq_callback_t *callbacks[],
 			const char * const names[], const bool *ctx,
-- 
2.25.4

[PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Max Gurtovoy]

According to the spec after writing 0 to device_status, the driver MUST
wait for a read of device_status to return 0 before reinitializing the
device. In case we have a device that won't return 0, the reset
operation will loop forever and cause the host/vm to stuck. Set timeout
for 3 minutes before giving up on the device.

Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
---
 drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index cc3412a96a17..dcee616e8d21 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
+	unsigned long timeout = jiffies + msecs_to_jiffies(180000);
 
 	/* 0 status means a reset. */
 	vp_modern_set_status(mdev, 0);
@@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
 	 * device_status to return 0 before reinitializing the device.
 	 * This will flush out the status write, and flush in device writes,
 	 * including MSI-X interrupts, if any.
+	 * Set a timeout before giving up on the device.
 	 */
-	while (vp_modern_get_status(mdev))
+	while (vp_modern_get_status(mdev)) {
+		if (time_after(jiffies, timeout)) {
+			dev_err(&vdev->dev, "virtio: device not ready. "
+				"Aborting. Try again later\n");
+			return -EAGAIN;
+		}
 		msleep(1);
+	}
 	/* Flush pending VQ/configuration callbacks. */
 	vp_synchronize_vectors(vdev);
 	return 0;
-- 
2.25.4

[PATCH v2 3/3] virtio_pci: add module param for reset_timeout

[Max Gurtovoy]

Enable the user to set the time for waiting for successful reset by the
virtio controller. Set the default to 180 seconds.

Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
---
 drivers/virtio/virtio_pci_common.c | 5 +++++
 drivers/virtio/virtio_pci_common.h | 2 ++
 drivers/virtio/virtio_pci_modern.c | 3 ++-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index 222d630c41fc..3a4c57839ed8 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -24,6 +24,11 @@ MODULE_PARM_DESC(force_legacy,
 		 "Force legacy mode for transitional virtio 1 devices");
 #endif
 
+unsigned int reset_timeout = 180;
+module_param_named(reset_timeout, reset_timeout, uint, 0644);
+MODULE_PARM_DESC(reset_timeout,
+		 "timeout in seconds for reset virtio device operation");
+
 /* wait for pending irq handlers */
 void vp_synchronize_vectors(struct virtio_device *vdev)
 {
diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
index beec047a8f8d..4760cdf74961 100644
--- a/drivers/virtio/virtio_pci_common.h
+++ b/drivers/virtio/virtio_pci_common.h
@@ -29,6 +29,8 @@
 #include <linux/highmem.h>
 #include <linux/spinlock.h>
 
+extern unsigned int reset_timeout;
+
 struct virtio_pci_vq_info {
 	/* the actual virtqueue */
 	struct virtqueue *vq;
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index dcee616e8d21..811fc1719d8c 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -162,7 +162,8 @@ static int vp_reset(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
-	unsigned long timeout = jiffies + msecs_to_jiffies(180000);
+	unsigned long timeout = jiffies +
+		msecs_to_jiffies(reset_timeout * 1000);
 
 	/* 0 status means a reset. */
 	vp_modern_set_status(mdev, 0);
-- 
2.25.4

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> The reset device operation, usually is an operation that might fail from
> various reasons. For example, the controller might be in a bad state and
> can't answer to any request. Usually, the paravirt SW based virtio
> devices always succeed in reset operation but this is not the case for
> HW based virtio devices.


I would like to know under what condition that the reset operation may 
fail (except for the case of a bugg guest).


>
> This commit is also a preparation for adding a timeout mechanism for
> resetting virtio devices.
>
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
>
> changes from v1:
>   - update virtio_ccw.c (Cornelia)
>   - update virtio_uml.c
>   - update mlxbf-tmfifo.c


Note that virtio driver may call reset, so you probably need to convert 
them.

Thanks


>
> ---
>   arch/um/drivers/virtio_uml.c             |  4 +++-
>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>   drivers/virtio/virtio_mmio.c             |  3 ++-
>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>   include/linux/virtio_config.h            |  5 +++--
>   10 files changed, 43 insertions(+), 19 deletions(-)
>
> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> index 91ddf74ca888..b6e66265ed32 100644
> --- a/arch/um/drivers/virtio_uml.c
> +++ b/arch/um/drivers/virtio_uml.c
> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device *vdev, u8 status)
>   	vu_dev->status = status;
>   }
>   
> -static void vu_reset(struct virtio_device *vdev)
> +static int vu_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>   
>   	vu_dev->status = 0;
> +
> +	return 0;
>   }
>   
>   static void vu_del_vq(struct virtqueue *vq)
> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
> index bbc4e71a16ff..c192b8ac5d9e 100644
> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> @@ -980,11 +980,13 @@ static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>   }
>   
>   /* Reset the device. Not much here for now. */
> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>   {
>   	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>   
>   	tm_vdev->status = 0;
> +
> +	return 0;
>   }
>   
>   /* Read the value of a configuration field. */
> diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
> index 0cc617f76068..ca9573c62c3d 100644
> --- a/drivers/remoteproc/remoteproc_virtio.c
> +++ b/drivers/remoteproc/remoteproc_virtio.c
> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status)
>   	dev_dbg(&vdev->dev, "status: %d\n", status);
>   }
>   
> -static void rproc_virtio_reset(struct virtio_device *vdev)
> +static int rproc_virtio_reset(struct virtio_device *vdev)
>   {
>   	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>   	struct fw_rsc_vdev *rsc;
> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
>   
>   	rsc->status = 0;
>   	dev_dbg(&vdev->dev, "reset !\n");
> +
> +	return 0;
>   }
>   
>   /* provide the vdev features as retrieved from the firmware */
> diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
> index 54e686dca6de..52b32555e746 100644
> --- a/drivers/s390/virtio/virtio_ccw.c
> +++ b/drivers/s390/virtio/virtio_ccw.c
> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
>   	return ret;
>   }
>   
> -static void virtio_ccw_reset(struct virtio_device *vdev)
> +static int virtio_ccw_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>   	struct ccw1 *ccw;
> +	int ret;
>   
>   	ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>   	if (!ccw)
> -		return;
> +		return -ENOMEM;
>   
>   	/* Zero status bits. */
>   	vcdev->dma_area->status = 0;
> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
>   	ccw->flags = 0;
>   	ccw->count = 0;
>   	ccw->cda = 0;
> -	ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> +	ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>   	ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> +
> +	return ret;
>   }
>   
>   static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> --- a/drivers/virtio/virtio.c
> +++ b/drivers/virtio/virtio.c
> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device *dev)
>   	/* Assign a unique device index and hence name. */
>   	err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>   	if (err < 0)
> -		goto out;
> +		goto out_err;
>   
>   	dev->index = err;
>   	dev_set_name(&dev->dev, "virtio%u", dev->index);
> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device *dev)
>   
>   	/* We always start by resetting the device, in case a previous
>   	 * driver messed it up.  This also tests that code path a little. */
> -	dev->config->reset(dev);
> +	err = dev->config->reset(dev);
> +	if (err)
> +		goto out_ida;
>   
>   	/* Acknowledge that we've seen the device. */
>   	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> @@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device *dev)
>   	 */
>   	err = device_add(&dev->dev);
>   	if (err)
> -		ida_simple_remove(&virtio_index_ida, dev->index);
> -out:
> -	if (err)
> -		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +		goto out_ida;
> +
> +	return 0;
> +
> +out_ida:
> +	ida_simple_remove(&virtio_index_ida, dev->index);
> +out_err:
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>   	return err;
>   }
>   EXPORT_SYMBOL_GPL(register_virtio_device);
> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>   
>   	/* We always start by resetting the device, in case a previous
>   	 * driver messed it up. */
> -	dev->config->reset(dev);
> +	ret = dev->config->reset(dev);
> +	if (ret)
> +		goto err;
>   
>   	/* Acknowledge that we've seen the device. */
>   	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> index 56128b9c46eb..12b8f048c48d 100644
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)
>   	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>   }
>   
> -static void vm_reset(struct virtio_device *vdev)
> +static int vm_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>   
>   	/* 0 status means a reset. */
>   	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> +	return 0;
>   }
>   
>   
> diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
> index d62e9835aeec..0b5d95e3efa1 100644
> --- a/drivers/virtio/virtio_pci_legacy.c
> +++ b/drivers/virtio/virtio_pci_legacy.c
> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>   	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>   }
>   
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	/* 0 status means a reset. */
> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>   	ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
> +
> +	return 0;
>   }
>   
>   static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index fbd4ebc00eb6..cc3412a96a17 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>   	vp_modern_set_status(&vp_dev->mdev, status);
>   }
>   
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>   		msleep(1);
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
> +	return 0;
>   }
>   
>   static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> index e28acf482e0c..5fd4e627a9b0 100644
> --- a/drivers/virtio/virtio_vdpa.c
> +++ b/drivers/virtio/virtio_vdpa.c
> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
>   	return ops->set_status(vdpa, status);
>   }
>   
> -static void virtio_vdpa_reset(struct virtio_device *vdev)
> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>   {
>   	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>   
>   	vdpa_reset(vdpa);
> +
> +	return 0;
>   }
>   
>   static bool virtio_vdpa_notify(struct virtqueue *vq)
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 8519b3ae5d52..d2b0f1699a75 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>    *	status: the new status byte
>    * @reset: reset the device
>    *	vdev: the virtio device
> - *	After this, status and feature negotiation must be done again
> + *	Upon success, status and feature negotiation must be done again
>    *	Device must not be reset from its vq/config callbacks, or in
>    *	parallel with being added/removed.
> + *	Returns 0 on success or error status.
>    * @find_vqs: find virtqueues and instantiate them.
>    *	vdev: the virtio_device
>    *	nvqs: the number of virtqueues to find
> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>   	u32 (*generation)(struct virtio_device *vdev);
>   	u8 (*get_status)(struct virtio_device *vdev);
>   	void (*set_status)(struct virtio_device *vdev, u8 status);
> -	void (*reset)(struct virtio_device *vdev);
> +	int (*reset)(struct virtio_device *vdev);
>   	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>   			struct virtqueue *vqs[], vq_callback_t *callbacks[],
>   			const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> The reset device operation, usually is an operation that might fail from
> various reasons. For example, the controller might be in a bad state and
> can't answer to any request. Usually, the paravirt SW based virtio
> devices always succeed in reset operation but this is not the case for
> HW based virtio devices.


I would like to know under what condition that the reset operation may 
fail (except for the case of a bugg guest).


>
> This commit is also a preparation for adding a timeout mechanism for
> resetting virtio devices.
>
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
>
> changes from v1:
>   - update virtio_ccw.c (Cornelia)
>   - update virtio_uml.c
>   - update mlxbf-tmfifo.c


Note that virtio driver may call reset, so you probably need to convert 
them.

Thanks


>
> ---
>   arch/um/drivers/virtio_uml.c             |  4 +++-
>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>   drivers/virtio/virtio_mmio.c             |  3 ++-
>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>   include/linux/virtio_config.h            |  5 +++--
>   10 files changed, 43 insertions(+), 19 deletions(-)
>
> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> index 91ddf74ca888..b6e66265ed32 100644
> --- a/arch/um/drivers/virtio_uml.c
> +++ b/arch/um/drivers/virtio_uml.c
> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device *vdev, u8 status)
>   	vu_dev->status = status;
>   }
>   
> -static void vu_reset(struct virtio_device *vdev)
> +static int vu_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>   
>   	vu_dev->status = 0;
> +
> +	return 0;
>   }
>   
>   static void vu_del_vq(struct virtqueue *vq)
> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
> index bbc4e71a16ff..c192b8ac5d9e 100644
> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> @@ -980,11 +980,13 @@ static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>   }
>   
>   /* Reset the device. Not much here for now. */
> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>   {
>   	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>   
>   	tm_vdev->status = 0;
> +
> +	return 0;
>   }
>   
>   /* Read the value of a configuration field. */
> diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
> index 0cc617f76068..ca9573c62c3d 100644
> --- a/drivers/remoteproc/remoteproc_virtio.c
> +++ b/drivers/remoteproc/remoteproc_virtio.c
> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status)
>   	dev_dbg(&vdev->dev, "status: %d\n", status);
>   }
>   
> -static void rproc_virtio_reset(struct virtio_device *vdev)
> +static int rproc_virtio_reset(struct virtio_device *vdev)
>   {
>   	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>   	struct fw_rsc_vdev *rsc;
> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
>   
>   	rsc->status = 0;
>   	dev_dbg(&vdev->dev, "reset !\n");
> +
> +	return 0;
>   }
>   
>   /* provide the vdev features as retrieved from the firmware */
> diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
> index 54e686dca6de..52b32555e746 100644
> --- a/drivers/s390/virtio/virtio_ccw.c
> +++ b/drivers/s390/virtio/virtio_ccw.c
> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
>   	return ret;
>   }
>   
> -static void virtio_ccw_reset(struct virtio_device *vdev)
> +static int virtio_ccw_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>   	struct ccw1 *ccw;
> +	int ret;
>   
>   	ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>   	if (!ccw)
> -		return;
> +		return -ENOMEM;
>   
>   	/* Zero status bits. */
>   	vcdev->dma_area->status = 0;
> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
>   	ccw->flags = 0;
>   	ccw->count = 0;
>   	ccw->cda = 0;
> -	ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> +	ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>   	ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> +
> +	return ret;
>   }
>   
>   static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> --- a/drivers/virtio/virtio.c
> +++ b/drivers/virtio/virtio.c
> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device *dev)
>   	/* Assign a unique device index and hence name. */
>   	err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>   	if (err < 0)
> -		goto out;
> +		goto out_err;
>   
>   	dev->index = err;
>   	dev_set_name(&dev->dev, "virtio%u", dev->index);
> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device *dev)
>   
>   	/* We always start by resetting the device, in case a previous
>   	 * driver messed it up.  This also tests that code path a little. */
> -	dev->config->reset(dev);
> +	err = dev->config->reset(dev);
> +	if (err)
> +		goto out_ida;
>   
>   	/* Acknowledge that we've seen the device. */
>   	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> @@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device *dev)
>   	 */
>   	err = device_add(&dev->dev);
>   	if (err)
> -		ida_simple_remove(&virtio_index_ida, dev->index);
> -out:
> -	if (err)
> -		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +		goto out_ida;
> +
> +	return 0;
> +
> +out_ida:
> +	ida_simple_remove(&virtio_index_ida, dev->index);
> +out_err:
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>   	return err;
>   }
>   EXPORT_SYMBOL_GPL(register_virtio_device);
> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>   
>   	/* We always start by resetting the device, in case a previous
>   	 * driver messed it up. */
> -	dev->config->reset(dev);
> +	ret = dev->config->reset(dev);
> +	if (ret)
> +		goto err;
>   
>   	/* Acknowledge that we've seen the device. */
>   	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> index 56128b9c46eb..12b8f048c48d 100644
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)
>   	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>   }
>   
> -static void vm_reset(struct virtio_device *vdev)
> +static int vm_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>   
>   	/* 0 status means a reset. */
>   	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> +	return 0;
>   }
>   
>   
> diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
> index d62e9835aeec..0b5d95e3efa1 100644
> --- a/drivers/virtio/virtio_pci_legacy.c
> +++ b/drivers/virtio/virtio_pci_legacy.c
> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>   	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>   }
>   
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	/* 0 status means a reset. */
> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>   	ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
> +
> +	return 0;
>   }
>   
>   static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index fbd4ebc00eb6..cc3412a96a17 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>   	vp_modern_set_status(&vp_dev->mdev, status);
>   }
>   
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>   		msleep(1);
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
> +	return 0;
>   }
>   
>   static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> index e28acf482e0c..5fd4e627a9b0 100644
> --- a/drivers/virtio/virtio_vdpa.c
> +++ b/drivers/virtio/virtio_vdpa.c
> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
>   	return ops->set_status(vdpa, status);
>   }
>   
> -static void virtio_vdpa_reset(struct virtio_device *vdev)
> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>   {
>   	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>   
>   	vdpa_reset(vdpa);
> +
> +	return 0;
>   }
>   
>   static bool virtio_vdpa_notify(struct virtqueue *vq)
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 8519b3ae5d52..d2b0f1699a75 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>    *	status: the new status byte
>    * @reset: reset the device
>    *	vdev: the virtio device
> - *	After this, status and feature negotiation must be done again
> + *	Upon success, status and feature negotiation must be done again
>    *	Device must not be reset from its vq/config callbacks, or in
>    *	parallel with being added/removed.
> + *	Returns 0 on success or error status.
>    * @find_vqs: find virtqueues and instantiate them.
>    *	vdev: the virtio_device
>    *	nvqs: the number of virtqueues to find
> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>   	u32 (*generation)(struct virtio_device *vdev);
>   	u8 (*get_status)(struct virtio_device *vdev);
>   	void (*set_status)(struct virtio_device *vdev, u8 status);
> -	void (*reset)(struct virtio_device *vdev);
> +	int (*reset)(struct virtio_device *vdev);
>   	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>   			struct virtqueue *vqs[], vq_callback_t *callbacks[],
>   			const char * const names[], const bool *ctx,

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> According to the spec after writing 0 to device_status, the driver MUST
> wait for a read of device_status to return 0 before reinitializing the
> device. In case we have a device that won't return 0, the reset
> operation will loop forever and cause the host/vm to stuck. Set timeout
> for 3 minutes before giving up on the device.
>
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>   1 file changed, 9 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index cc3412a96a17..dcee616e8d21 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> +	unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>   
>   	/* 0 status means a reset. */
>   	vp_modern_set_status(mdev, 0);
> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>   	 * device_status to return 0 before reinitializing the device.
>   	 * This will flush out the status write, and flush in device writes,
>   	 * including MSI-X interrupts, if any.
> +	 * Set a timeout before giving up on the device.
>   	 */
> -	while (vp_modern_get_status(mdev))
> +	while (vp_modern_get_status(mdev)) {
> +		if (time_after(jiffies, timeout)) {


What happens if the device finish the rest after the timeout?

Thanks


> +			dev_err(&vdev->dev, "virtio: device not ready. "
> +				"Aborting. Try again later\n");
> +			return -EAGAIN;
> +		}
>   		msleep(1);
> +	}
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
>   	return 0;

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> According to the spec after writing 0 to device_status, the driver MUST
> wait for a read of device_status to return 0 before reinitializing the
> device. In case we have a device that won't return 0, the reset
> operation will loop forever and cause the host/vm to stuck. Set timeout
> for 3 minutes before giving up on the device.
>
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>   1 file changed, 9 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index cc3412a96a17..dcee616e8d21 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>   {
>   	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>   	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> +	unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>   
>   	/* 0 status means a reset. */
>   	vp_modern_set_status(mdev, 0);
> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>   	 * device_status to return 0 before reinitializing the device.
>   	 * This will flush out the status write, and flush in device writes,
>   	 * including MSI-X interrupts, if any.
> +	 * Set a timeout before giving up on the device.
>   	 */
> -	while (vp_modern_get_status(mdev))
> +	while (vp_modern_get_status(mdev)) {
> +		if (time_after(jiffies, timeout)) {


What happens if the device finish the rest after the timeout?

Thanks


> +			dev_err(&vdev->dev, "virtio: device not ready. "
> +				"Aborting. Try again later\n");
> +			return -EAGAIN;
> +		}
>   		msleep(1);
> +	}
>   	/* Flush pending VQ/configuration callbacks. */
>   	vp_synchronize_vectors(vdev);
>   	return 0;

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Max Gurtovoy]

On 4/8/2021 12:01 PM, Jason Wang wrote:
>
> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>> According to the spec after writing 0 to device_status, the driver MUST
>> wait for a read of device_status to return 0 before reinitializing the
>> device. In case we have a device that won't return 0, the reset
>> operation will loop forever and cause the host/vm to stuck. Set timeout
>> for 3 minutes before giving up on the device.
>>
>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>> ---
>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>> b/drivers/virtio/virtio_pci_modern.c
>> index cc3412a96a17..dcee616e8d21 100644
>> --- a/drivers/virtio/virtio_pci_modern.c
>> +++ b/drivers/virtio/virtio_pci_modern.c
>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>         /* 0 status means a reset. */
>>       vp_modern_set_status(mdev, 0);
>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>        * device_status to return 0 before reinitializing the device.
>>        * This will flush out the status write, and flush in device 
>> writes,
>>        * including MSI-X interrupts, if any.
>> +     * Set a timeout before giving up on the device.
>>        */
>> -    while (vp_modern_get_status(mdev))
>> +    while (vp_modern_get_status(mdev)) {
>> +        if (time_after(jiffies, timeout)) {
>
>
> What happens if the device finish the rest after the timeout?


The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it later 
on (e.g by re-scanning the pci bus).


>
> Thanks
>
>
>> +            dev_err(&vdev->dev, "virtio: device not ready. "
>> +                "Aborting. Try again later\n");
>> +            return -EAGAIN;
>> +        }
>>           msleep(1);
>> +    }
>>       /* Flush pending VQ/configuration callbacks. */
>>       vp_synchronize_vectors(vdev);
>>       return 0;
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/8/2021 11:58 AM, Jason Wang wrote:
>
> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>> The reset device operation, usually is an operation that might fail from
>> various reasons. For example, the controller might be in a bad state and
>> can't answer to any request. Usually, the paravirt SW based virtio
>> devices always succeed in reset operation but this is not the case for
>> HW based virtio devices.
>
>
> I would like to know under what condition that the reset operation may 
> fail (except for the case of a bugg guest).

The controller might not be ready or stuck. This is a real use case for 
many PCI devices.

For real devices the FW might be in a bad state and it can happen also 
for paravirt device if you have a bug in the controller code or if you 
entered some error flow (Out of memory).

You don't want to be stuck because of one bad device.


>
>
>>
>> This commit is also a preparation for adding a timeout mechanism for
>> resetting virtio devices.
>>
>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>> ---
>>
>> changes from v1:
>>   - update virtio_ccw.c (Cornelia)
>>   - update virtio_uml.c
>>   - update mlxbf-tmfifo.c
>
>
> Note that virtio driver may call reset, so you probably need to 
> convert them.

I'm sure I understand.

Convert to what ?

Thanks.

>
> Thanks
>
>
>>
>> ---
>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>   include/linux/virtio_config.h            |  5 +++--
>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>
>> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
>> index 91ddf74ca888..b6e66265ed32 100644
>> --- a/arch/um/drivers/virtio_uml.c
>> +++ b/arch/um/drivers/virtio_uml.c
>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device 
>> *vdev, u8 status)
>>       vu_dev->status = status;
>>   }
>>   -static void vu_reset(struct virtio_device *vdev)
>> +static int vu_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>         vu_dev->status = 0;
>> +
>> +    return 0;
>>   }
>>     static void vu_del_vq(struct virtqueue *vq)
>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>> index bbc4e71a16ff..c192b8ac5d9e 100644
>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>> @@ -980,11 +980,13 @@ static void 
>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>   }
>>     /* Reset the device. Not much here for now. */
>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>   {
>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>         tm_vdev->status = 0;
>> +
>> +    return 0;
>>   }
>>     /* Read the value of a configuration field. */
>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>> b/drivers/remoteproc/remoteproc_virtio.c
>> index 0cc617f76068..ca9573c62c3d 100644
>> --- a/drivers/remoteproc/remoteproc_virtio.c
>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>> virtio_device *vdev, u8 status)
>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>   }
>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>   {
>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>       struct fw_rsc_vdev *rsc;
>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>> virtio_device *vdev)
>>         rsc->status = 0;
>>       dev_dbg(&vdev->dev, "reset !\n");
>> +
>> +    return 0;
>>   }
>>     /* provide the vdev features as retrieved from the firmware */
>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>> b/drivers/s390/virtio/virtio_ccw.c
>> index 54e686dca6de..52b32555e746 100644
>> --- a/drivers/s390/virtio/virtio_ccw.c
>> +++ b/drivers/s390/virtio/virtio_ccw.c
>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>> virtio_device *vdev, unsigned nvqs,
>>       return ret;
>>   }
>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>       struct ccw1 *ccw;
>> +    int ret;
>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>       if (!ccw)
>> -        return;
>> +        return -ENOMEM;
>>         /* Zero status bits. */
>>       vcdev->dma_area->status = 0;
>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>> virtio_device *vdev)
>>       ccw->flags = 0;
>>       ccw->count = 0;
>>       ccw->cda = 0;
>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>> +
>> +    return ret;
>>   }
>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>> --- a/drivers/virtio/virtio.c
>> +++ b/drivers/virtio/virtio.c
>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device 
>> *dev)
>>       /* Assign a unique device index and hence name. */
>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>       if (err < 0)
>> -        goto out;
>> +        goto out_err;
>>         dev->index = err;
>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device 
>> *dev)
>>         /* We always start by resetting the device, in case a previous
>>        * driver messed it up.  This also tests that code path a 
>> little. */
>> -    dev->config->reset(dev);
>> +    err = dev->config->reset(dev);
>> +    if (err)
>> +        goto out_ida;
>>         /* Acknowledge that we've seen the device. */
>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>> @@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device 
>> *dev)
>>        */
>>       err = device_add(&dev->dev);
>>       if (err)
>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>> -out:
>> -    if (err)
>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>> +        goto out_ida;
>> +
>> +    return 0;
>> +
>> +out_ida:
>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>> +out_err:
>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>       return err;
>>   }
>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>>         /* We always start by resetting the device, in case a previous
>>        * driver messed it up. */
>> -    dev->config->reset(dev);
>> +    ret = dev->config->reset(dev);
>> +    if (ret)
>> +        goto err;
>>         /* Acknowledge that we've seen the device. */
>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
>> index 56128b9c46eb..12b8f048c48d 100644
>> --- a/drivers/virtio/virtio_mmio.c
>> +++ b/drivers/virtio/virtio_mmio.c
>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device 
>> *vdev, u8 status)
>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>   }
>>   -static void vm_reset(struct virtio_device *vdev)
>> +static int vm_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>         /* 0 status means a reset. */
>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>> +    return 0;
>>   }
>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>> b/drivers/virtio/virtio_pci_legacy.c
>> index d62e9835aeec..0b5d95e3efa1 100644
>> --- a/drivers/virtio/virtio_pci_legacy.c
>> +++ b/drivers/virtio/virtio_pci_legacy.c
>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>> *vdev, u8 status)
>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>   }
>>   -static void vp_reset(struct virtio_device *vdev)
>> +static int vp_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>       /* 0 status means a reset. */
>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>       /* Flush pending VQ/configuration callbacks. */
>>       vp_synchronize_vectors(vdev);
>> +
>> +    return 0;
>>   }
>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 
>> vector)
>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>> b/drivers/virtio/virtio_pci_modern.c
>> index fbd4ebc00eb6..cc3412a96a17 100644
>> --- a/drivers/virtio/virtio_pci_modern.c
>> +++ b/drivers/virtio/virtio_pci_modern.c
>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>> *vdev, u8 status)
>>       vp_modern_set_status(&vp_dev->mdev, status);
>>   }
>>   -static void vp_reset(struct virtio_device *vdev)
>> +static int vp_reset(struct virtio_device *vdev)
>>   {
>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>           msleep(1);
>>       /* Flush pending VQ/configuration callbacks. */
>>       vp_synchronize_vectors(vdev);
>> +    return 0;
>>   }
>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 
>> vector)
>> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
>> index e28acf482e0c..5fd4e627a9b0 100644
>> --- a/drivers/virtio/virtio_vdpa.c
>> +++ b/drivers/virtio/virtio_vdpa.c
>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>> virtio_device *vdev, u8 status)
>>       return ops->set_status(vdpa, status);
>>   }
>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>   {
>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>         vdpa_reset(vdpa);
>> +
>> +    return 0;
>>   }
>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>> diff --git a/include/linux/virtio_config.h 
>> b/include/linux/virtio_config.h
>> index 8519b3ae5d52..d2b0f1699a75 100644
>> --- a/include/linux/virtio_config.h
>> +++ b/include/linux/virtio_config.h
>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>    *    status: the new status byte
>>    * @reset: reset the device
>>    *    vdev: the virtio device
>> - *    After this, status and feature negotiation must be done again
>> + *    Upon success, status and feature negotiation must be done again
>>    *    Device must not be reset from its vq/config callbacks, or in
>>    *    parallel with being added/removed.
>> + *    Returns 0 on success or error status.
>>    * @find_vqs: find virtqueues and instantiate them.
>>    *    vdev: the virtio_device
>>    *    nvqs: the number of virtqueues to find
>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>       u32 (*generation)(struct virtio_device *vdev);
>>       u8 (*get_status)(struct virtio_device *vdev);
>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>> -    void (*reset)(struct virtio_device *vdev);
>> +    int (*reset)(struct virtio_device *vdev);
>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>               const char * const names[], const bool *ctx,
>

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午5:44, Max Gurtovoy 写道:
>
> On 4/8/2021 12:01 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>> According to the spec after writing 0 to device_status, the driver MUST
>>> wait for a read of device_status to return 0 before reinitializing the
>>> device. In case we have a device that won't return 0, the reset
>>> operation will loop forever and cause the host/vm to stuck. Set timeout
>>> for 3 minutes before giving up on the device.
>>>
>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>> ---
>>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>> b/drivers/virtio/virtio_pci_modern.c
>>> index cc3412a96a17..dcee616e8d21 100644
>>> --- a/drivers/virtio/virtio_pci_modern.c
>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>>         /* 0 status means a reset. */
>>>       vp_modern_set_status(mdev, 0);
>>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>>        * device_status to return 0 before reinitializing the device.
>>>        * This will flush out the status write, and flush in device 
>>> writes,
>>>        * including MSI-X interrupts, if any.
>>> +     * Set a timeout before giving up on the device.
>>>        */
>>> -    while (vp_modern_get_status(mdev))
>>> +    while (vp_modern_get_status(mdev)) {
>>> +        if (time_after(jiffies, timeout)) {
>>
>>
>> What happens if the device finish the rest after the timeout?
>
>
> The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it 
> later on (e.g by re-scanning the pci bus).


Ok, so do we need the flush through vp_synchronize_vectors() here?

Thanks


>
>
>>
>> Thanks
>>
>>
>>> +            dev_err(&vdev->dev, "virtio: device not ready. "
>>> +                "Aborting. Try again later\n");
>>> +            return -EAGAIN;
>>> +        }
>>>           msleep(1);
>>> +    }
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>>       return 0;
>>
>

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午5:44, Max Gurtovoy 写道:
>
> On 4/8/2021 12:01 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>> According to the spec after writing 0 to device_status, the driver MUST
>>> wait for a read of device_status to return 0 before reinitializing the
>>> device. In case we have a device that won't return 0, the reset
>>> operation will loop forever and cause the host/vm to stuck. Set timeout
>>> for 3 minutes before giving up on the device.
>>>
>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>> ---
>>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>> b/drivers/virtio/virtio_pci_modern.c
>>> index cc3412a96a17..dcee616e8d21 100644
>>> --- a/drivers/virtio/virtio_pci_modern.c
>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>>         /* 0 status means a reset. */
>>>       vp_modern_set_status(mdev, 0);
>>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>>        * device_status to return 0 before reinitializing the device.
>>>        * This will flush out the status write, and flush in device 
>>> writes,
>>>        * including MSI-X interrupts, if any.
>>> +     * Set a timeout before giving up on the device.
>>>        */
>>> -    while (vp_modern_get_status(mdev))
>>> +    while (vp_modern_get_status(mdev)) {
>>> +        if (time_after(jiffies, timeout)) {
>>
>>
>> What happens if the device finish the rest after the timeout?
>
>
> The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it 
> later on (e.g by re-scanning the pci bus).


Ok, so do we need the flush through vp_synchronize_vectors() here?

Thanks


>
>
>>
>> Thanks
>>
>>
>>> +            dev_err(&vdev->dev, "virtio: device not ready. "
>>> +                "Aborting. Try again later\n");
>>> +            return -EAGAIN;
>>> +        }
>>>           msleep(1);
>>> +    }
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>>       return 0;
>>
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Max Gurtovoy]

On 4/8/2021 3:45 PM, Jason Wang wrote:
>
> 在 2021/4/8 下午5:44, Max Gurtovoy 写道:
>>
>> On 4/8/2021 12:01 PM, Jason Wang wrote:
>>>
>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>> According to the spec after writing 0 to device_status, the driver 
>>>> MUST
>>>> wait for a read of device_status to return 0 before reinitializing the
>>>> device. In case we have a device that won't return 0, the reset
>>>> operation will loop forever and cause the host/vm to stuck. Set 
>>>> timeout
>>>> for 3 minutes before giving up on the device.
>>>>
>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>> ---
>>>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>> b/drivers/virtio/virtio_pci_modern.c
>>>> index cc3412a96a17..dcee616e8d21 100644
>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>>>         /* 0 status means a reset. */
>>>>       vp_modern_set_status(mdev, 0);
>>>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>>>        * device_status to return 0 before reinitializing the device.
>>>>        * This will flush out the status write, and flush in device 
>>>> writes,
>>>>        * including MSI-X interrupts, if any.
>>>> +     * Set a timeout before giving up on the device.
>>>>        */
>>>> -    while (vp_modern_get_status(mdev))
>>>> +    while (vp_modern_get_status(mdev)) {
>>>> +        if (time_after(jiffies, timeout)) {
>>>
>>>
>>> What happens if the device finish the rest after the timeout?
>>
>>
>> The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it 
>> later on (e.g by re-scanning the pci bus).
>
>
> Ok, so do we need the flush through vp_synchronize_vectors() here?

If the device didn't write 0 to status I guess we don't need that.

The device shouldn't raise any interrupt before negotiation finish 
successfully.

MST, is that correct ?

>
> Thanks
>
>
>>
>>
>>>
>>> Thanks
>>>
>>>
>>>> + dev_err(&vdev->dev, "virtio: device not ready. "
>>>> +                "Aborting. Try again later\n");
>>>> +            return -EAGAIN;
>>>> +        }
>>>>           msleep(1);
>>>> +    }
>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>       vp_synchronize_vectors(vdev);
>>>>       return 0;
>>>
>>
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>
> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>> The reset device operation, usually is an operation that might fail 
>>> from
>>> various reasons. For example, the controller might be in a bad state 
>>> and
>>> can't answer to any request. Usually, the paravirt SW based virtio
>>> devices always succeed in reset operation but this is not the case for
>>> HW based virtio devices.
>>
>>
>> I would like to know under what condition that the reset operation 
>> may fail (except for the case of a bugg guest).
>
> The controller might not be ready or stuck. This is a real use case 
> for many PCI devices.
>
> For real devices the FW might be in a bad state and it can happen also 
> for paravirt device if you have a bug in the controller code or if you 
> entered some error flow (Out of memory).
>
> You don't want to be stuck because of one bad device.


So the buggy driver can damage the host through various ways, I'm not 
sure doing such workaround is worthwhile.

Note that this driver has been used for real hardware PCI devices for 
many years. We don't receive any report of this before.


>
>
>>
>>
>>>
>>> This commit is also a preparation for adding a timeout mechanism for
>>> resetting virtio devices.
>>>
>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>> ---
>>>
>>> changes from v1:
>>>   - update virtio_ccw.c (Cornelia)
>>>   - update virtio_uml.c
>>>   - update mlxbf-tmfifo.c
>>
>>
>> Note that virtio driver may call reset, so you probably need to 
>> convert them.
>
> I'm sure I understand.
>
> Convert to what ?


Convert to deal with the possible rest failure. E.g in virtblk_freeze() 
we had:

static int virtblk_freeze(struct virtio_device *vdev)
{
         struct virtio_blk *vblk = vdev->priv;

         /* Ensure we don't receive any more interrupts */
         vdev->config->reset(vdev);
...

We need fail the freeze here.

Another example is the driver remove which is not expected to be fail. A 
lot of virtio drivers tries to call reset there. I'm not sure how hard 
to deal with the failure in the path of remove (e.g 
__device_release_driver tends to ignore the return value of bus->remove().)

Thanks


>
> Thanks.
>
>>
>> Thanks
>>
>>
>>>
>>> ---
>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>   include/linux/virtio_config.h            |  5 +++--
>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>
>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>> b/arch/um/drivers/virtio_uml.c
>>> index 91ddf74ca888..b6e66265ed32 100644
>>> --- a/arch/um/drivers/virtio_uml.c
>>> +++ b/arch/um/drivers/virtio_uml.c
>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       vu_dev->status = status;
>>>   }
>>>   -static void vu_reset(struct virtio_device *vdev)
>>> +static int vu_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>         vu_dev->status = 0;
>>> +
>>> +    return 0;
>>>   }
>>>     static void vu_del_vq(struct virtqueue *vq)
>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> @@ -980,11 +980,13 @@ static void 
>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>   }
>>>     /* Reset the device. Not much here for now. */
>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>   {
>>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>         tm_vdev->status = 0;
>>> +
>>> +    return 0;
>>>   }
>>>     /* Read the value of a configuration field. */
>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>> b/drivers/remoteproc/remoteproc_virtio.c
>>> index 0cc617f76068..ca9573c62c3d 100644
>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>> virtio_device *vdev, u8 status)
>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>   }
>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>   {
>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>       struct fw_rsc_vdev *rsc;
>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>> virtio_device *vdev)
>>>         rsc->status = 0;
>>>       dev_dbg(&vdev->dev, "reset !\n");
>>> +
>>> +    return 0;
>>>   }
>>>     /* provide the vdev features as retrieved from the firmware */
>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>> b/drivers/s390/virtio/virtio_ccw.c
>>> index 54e686dca6de..52b32555e746 100644
>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>> virtio_device *vdev, unsigned nvqs,
>>>       return ret;
>>>   }
>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>       struct ccw1 *ccw;
>>> +    int ret;
>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>       if (!ccw)
>>> -        return;
>>> +        return -ENOMEM;
>>>         /* Zero status bits. */
>>>       vcdev->dma_area->status = 0;
>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>> virtio_device *vdev)
>>>       ccw->flags = 0;
>>>       ccw->count = 0;
>>>       ccw->cda = 0;
>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>> +
>>> +    return ret;
>>>   }
>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>> --- a/drivers/virtio/virtio.c
>>> +++ b/drivers/virtio/virtio.c
>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device 
>>> *dev)
>>>       /* Assign a unique device index and hence name. */
>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>       if (err < 0)
>>> -        goto out;
>>> +        goto out_err;
>>>         dev->index = err;
>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device 
>>> *dev)
>>>         /* We always start by resetting the device, in case a previous
>>>        * driver messed it up.  This also tests that code path a 
>>> little. */
>>> -    dev->config->reset(dev);
>>> +    err = dev->config->reset(dev);
>>> +    if (err)
>>> +        goto out_ida;
>>>         /* Acknowledge that we've seen the device. */
>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>> virtio_device *dev)
>>>        */
>>>       err = device_add(&dev->dev);
>>>       if (err)
>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>> -out:
>>> -    if (err)
>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>> +        goto out_ida;
>>> +
>>> +    return 0;
>>> +
>>> +out_ida:
>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>> +out_err:
>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>       return err;
>>>   }
>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device 
>>> *dev)
>>>         /* We always start by resetting the device, in case a previous
>>>        * driver messed it up. */
>>> -    dev->config->reset(dev);
>>> +    ret = dev->config->reset(dev);
>>> +    if (ret)
>>> +        goto err;
>>>         /* Acknowledge that we've seen the device. */
>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>> b/drivers/virtio/virtio_mmio.c
>>> index 56128b9c46eb..12b8f048c48d 100644
>>> --- a/drivers/virtio/virtio_mmio.c
>>> +++ b/drivers/virtio/virtio_mmio.c
>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>   }
>>>   -static void vm_reset(struct virtio_device *vdev)
>>> +static int vm_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>         /* 0 status means a reset. */
>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>> +    return 0;
>>>   }
>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>> b/drivers/virtio/virtio_pci_legacy.c
>>> index d62e9835aeec..0b5d95e3efa1 100644
>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>   }
>>>   -static void vp_reset(struct virtio_device *vdev)
>>> +static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       /* 0 status means a reset. */
>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>> +
>>> +    return 0;
>>>   }
>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>> u16 vector)
>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>> b/drivers/virtio/virtio_pci_modern.c
>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>> --- a/drivers/virtio/virtio_pci_modern.c
>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>   }
>>>   -static void vp_reset(struct virtio_device *vdev)
>>> +static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>           msleep(1);
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>> +    return 0;
>>>   }
>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>> u16 vector)
>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>> b/drivers/virtio/virtio_vdpa.c
>>> index e28acf482e0c..5fd4e627a9b0 100644
>>> --- a/drivers/virtio/virtio_vdpa.c
>>> +++ b/drivers/virtio/virtio_vdpa.c
>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>> virtio_device *vdev, u8 status)
>>>       return ops->set_status(vdpa, status);
>>>   }
>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>   {
>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>         vdpa_reset(vdpa);
>>> +
>>> +    return 0;
>>>   }
>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>> diff --git a/include/linux/virtio_config.h 
>>> b/include/linux/virtio_config.h
>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>> --- a/include/linux/virtio_config.h
>>> +++ b/include/linux/virtio_config.h
>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>    *    status: the new status byte
>>>    * @reset: reset the device
>>>    *    vdev: the virtio device
>>> - *    After this, status and feature negotiation must be done again
>>> + *    Upon success, status and feature negotiation must be done again
>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>    *    parallel with being added/removed.
>>> + *    Returns 0 on success or error status.
>>>    * @find_vqs: find virtqueues and instantiate them.
>>>    *    vdev: the virtio_device
>>>    *    nvqs: the number of virtqueues to find
>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>       u32 (*generation)(struct virtio_device *vdev);
>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>> -    void (*reset)(struct virtio_device *vdev);
>>> +    int (*reset)(struct virtio_device *vdev);
>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>               const char * const names[], const bool *ctx,
>>
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>
> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>> The reset device operation, usually is an operation that might fail 
>>> from
>>> various reasons. For example, the controller might be in a bad state 
>>> and
>>> can't answer to any request. Usually, the paravirt SW based virtio
>>> devices always succeed in reset operation but this is not the case for
>>> HW based virtio devices.
>>
>>
>> I would like to know under what condition that the reset operation 
>> may fail (except for the case of a bugg guest).
>
> The controller might not be ready or stuck. This is a real use case 
> for many PCI devices.
>
> For real devices the FW might be in a bad state and it can happen also 
> for paravirt device if you have a bug in the controller code or if you 
> entered some error flow (Out of memory).
>
> You don't want to be stuck because of one bad device.


So the buggy driver can damage the host through various ways, I'm not 
sure doing such workaround is worthwhile.

Note that this driver has been used for real hardware PCI devices for 
many years. We don't receive any report of this before.


>
>
>>
>>
>>>
>>> This commit is also a preparation for adding a timeout mechanism for
>>> resetting virtio devices.
>>>
>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>> ---
>>>
>>> changes from v1:
>>>   - update virtio_ccw.c (Cornelia)
>>>   - update virtio_uml.c
>>>   - update mlxbf-tmfifo.c
>>
>>
>> Note that virtio driver may call reset, so you probably need to 
>> convert them.
>
> I'm sure I understand.
>
> Convert to what ?


Convert to deal with the possible rest failure. E.g in virtblk_freeze() 
we had:

static int virtblk_freeze(struct virtio_device *vdev)
{
         struct virtio_blk *vblk = vdev->priv;

         /* Ensure we don't receive any more interrupts */
         vdev->config->reset(vdev);
...

We need fail the freeze here.

Another example is the driver remove which is not expected to be fail. A 
lot of virtio drivers tries to call reset there. I'm not sure how hard 
to deal with the failure in the path of remove (e.g 
__device_release_driver tends to ignore the return value of bus->remove().)

Thanks


>
> Thanks.
>
>>
>> Thanks
>>
>>
>>>
>>> ---
>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>   include/linux/virtio_config.h            |  5 +++--
>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>
>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>> b/arch/um/drivers/virtio_uml.c
>>> index 91ddf74ca888..b6e66265ed32 100644
>>> --- a/arch/um/drivers/virtio_uml.c
>>> +++ b/arch/um/drivers/virtio_uml.c
>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       vu_dev->status = status;
>>>   }
>>>   -static void vu_reset(struct virtio_device *vdev)
>>> +static int vu_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>         vu_dev->status = 0;
>>> +
>>> +    return 0;
>>>   }
>>>     static void vu_del_vq(struct virtqueue *vq)
>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>> @@ -980,11 +980,13 @@ static void 
>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>   }
>>>     /* Reset the device. Not much here for now. */
>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>   {
>>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>         tm_vdev->status = 0;
>>> +
>>> +    return 0;
>>>   }
>>>     /* Read the value of a configuration field. */
>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>> b/drivers/remoteproc/remoteproc_virtio.c
>>> index 0cc617f76068..ca9573c62c3d 100644
>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>> virtio_device *vdev, u8 status)
>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>   }
>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>   {
>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>       struct fw_rsc_vdev *rsc;
>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>> virtio_device *vdev)
>>>         rsc->status = 0;
>>>       dev_dbg(&vdev->dev, "reset !\n");
>>> +
>>> +    return 0;
>>>   }
>>>     /* provide the vdev features as retrieved from the firmware */
>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>> b/drivers/s390/virtio/virtio_ccw.c
>>> index 54e686dca6de..52b32555e746 100644
>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>> virtio_device *vdev, unsigned nvqs,
>>>       return ret;
>>>   }
>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>       struct ccw1 *ccw;
>>> +    int ret;
>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>       if (!ccw)
>>> -        return;
>>> +        return -ENOMEM;
>>>         /* Zero status bits. */
>>>       vcdev->dma_area->status = 0;
>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>> virtio_device *vdev)
>>>       ccw->flags = 0;
>>>       ccw->count = 0;
>>>       ccw->cda = 0;
>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>> +
>>> +    return ret;
>>>   }
>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>> --- a/drivers/virtio/virtio.c
>>> +++ b/drivers/virtio/virtio.c
>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device 
>>> *dev)
>>>       /* Assign a unique device index and hence name. */
>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>       if (err < 0)
>>> -        goto out;
>>> +        goto out_err;
>>>         dev->index = err;
>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device 
>>> *dev)
>>>         /* We always start by resetting the device, in case a previous
>>>        * driver messed it up.  This also tests that code path a 
>>> little. */
>>> -    dev->config->reset(dev);
>>> +    err = dev->config->reset(dev);
>>> +    if (err)
>>> +        goto out_ida;
>>>         /* Acknowledge that we've seen the device. */
>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>> virtio_device *dev)
>>>        */
>>>       err = device_add(&dev->dev);
>>>       if (err)
>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>> -out:
>>> -    if (err)
>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>> +        goto out_ida;
>>> +
>>> +    return 0;
>>> +
>>> +out_ida:
>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>> +out_err:
>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>       return err;
>>>   }
>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device 
>>> *dev)
>>>         /* We always start by resetting the device, in case a previous
>>>        * driver messed it up. */
>>> -    dev->config->reset(dev);
>>> +    ret = dev->config->reset(dev);
>>> +    if (ret)
>>> +        goto err;
>>>         /* Acknowledge that we've seen the device. */
>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>> b/drivers/virtio/virtio_mmio.c
>>> index 56128b9c46eb..12b8f048c48d 100644
>>> --- a/drivers/virtio/virtio_mmio.c
>>> +++ b/drivers/virtio/virtio_mmio.c
>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>   }
>>>   -static void vm_reset(struct virtio_device *vdev)
>>> +static int vm_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>         /* 0 status means a reset. */
>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>> +    return 0;
>>>   }
>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>> b/drivers/virtio/virtio_pci_legacy.c
>>> index d62e9835aeec..0b5d95e3efa1 100644
>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>   }
>>>   -static void vp_reset(struct virtio_device *vdev)
>>> +static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       /* 0 status means a reset. */
>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>> +
>>> +    return 0;
>>>   }
>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>> u16 vector)
>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>> b/drivers/virtio/virtio_pci_modern.c
>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>> --- a/drivers/virtio/virtio_pci_modern.c
>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>>> *vdev, u8 status)
>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>   }
>>>   -static void vp_reset(struct virtio_device *vdev)
>>> +static int vp_reset(struct virtio_device *vdev)
>>>   {
>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>           msleep(1);
>>>       /* Flush pending VQ/configuration callbacks. */
>>>       vp_synchronize_vectors(vdev);
>>> +    return 0;
>>>   }
>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>> u16 vector)
>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>> b/drivers/virtio/virtio_vdpa.c
>>> index e28acf482e0c..5fd4e627a9b0 100644
>>> --- a/drivers/virtio/virtio_vdpa.c
>>> +++ b/drivers/virtio/virtio_vdpa.c
>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>> virtio_device *vdev, u8 status)
>>>       return ops->set_status(vdpa, status);
>>>   }
>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>   {
>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>         vdpa_reset(vdpa);
>>> +
>>> +    return 0;
>>>   }
>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>> diff --git a/include/linux/virtio_config.h 
>>> b/include/linux/virtio_config.h
>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>> --- a/include/linux/virtio_config.h
>>> +++ b/include/linux/virtio_config.h
>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>    *    status: the new status byte
>>>    * @reset: reset the device
>>>    *    vdev: the virtio device
>>> - *    After this, status and feature negotiation must be done again
>>> + *    Upon success, status and feature negotiation must be done again
>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>    *    parallel with being added/removed.
>>> + *    Returns 0 on success or error status.
>>>    * @find_vqs: find virtqueues and instantiate them.
>>>    *    vdev: the virtio_device
>>>    *    nvqs: the number of virtqueues to find
>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>       u32 (*generation)(struct virtio_device *vdev);
>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>> -    void (*reset)(struct virtio_device *vdev);
>>> +    int (*reset)(struct virtio_device *vdev);
>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>               const char * const names[], const bool *ctx,
>>
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午8:57, Max Gurtovoy 写道:
>
> On 4/8/2021 3:45 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午5:44, Max Gurtovoy 写道:
>>>
>>> On 4/8/2021 12:01 PM, Jason Wang wrote:
>>>>
>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>> According to the spec after writing 0 to device_status, the driver 
>>>>> MUST
>>>>> wait for a read of device_status to return 0 before reinitializing 
>>>>> the
>>>>> device. In case we have a device that won't return 0, the reset
>>>>> operation will loop forever and cause the host/vm to stuck. Set 
>>>>> timeout
>>>>> for 3 minutes before giving up on the device.
>>>>>
>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>> ---
>>>>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>>>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>>>>
>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>> index cc3412a96a17..dcee616e8d21 100644
>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>>>>         /* 0 status means a reset. */
>>>>>       vp_modern_set_status(mdev, 0);
>>>>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>>>>        * device_status to return 0 before reinitializing the device.
>>>>>        * This will flush out the status write, and flush in device 
>>>>> writes,
>>>>>        * including MSI-X interrupts, if any.
>>>>> +     * Set a timeout before giving up on the device.
>>>>>        */
>>>>> -    while (vp_modern_get_status(mdev))
>>>>> +    while (vp_modern_get_status(mdev)) {
>>>>> +        if (time_after(jiffies, timeout)) {
>>>>
>>>>
>>>> What happens if the device finish the rest after the timeout?
>>>
>>>
>>> The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it 
>>> later on (e.g by re-scanning the pci bus).
>>
>>
>> Ok, so do we need the flush through vp_synchronize_vectors() here?
>
> If the device didn't write 0 to status I guess we don't need that.
>
> The device shouldn't raise any interrupt before negotiation finish 
> successfully.


The reset could be triggered in other places like driver removing.

Thanks


>
> MST, is that correct ?
>
>>
>> Thanks
>>
>>
>>>
>>>
>>>>
>>>> Thanks
>>>>
>>>>
>>>>> + dev_err(&vdev->dev, "virtio: device not ready. "
>>>>> +                "Aborting. Try again later\n");
>>>>> +            return -EAGAIN;
>>>>> +        }
>>>>>           msleep(1);
>>>>> +    }
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>>       return 0;
>>>>
>>>
>>
>

Re: [PATCH v2 2/3] virito_pci: add timeout to reset device operation

[Jason Wang]

在 2021/4/8 下午8:57, Max Gurtovoy 写道:
>
> On 4/8/2021 3:45 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午5:44, Max Gurtovoy 写道:
>>>
>>> On 4/8/2021 12:01 PM, Jason Wang wrote:
>>>>
>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>> According to the spec after writing 0 to device_status, the driver 
>>>>> MUST
>>>>> wait for a read of device_status to return 0 before reinitializing 
>>>>> the
>>>>> device. In case we have a device that won't return 0, the reset
>>>>> operation will loop forever and cause the host/vm to stuck. Set 
>>>>> timeout
>>>>> for 3 minutes before giving up on the device.
>>>>>
>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>> ---
>>>>>   drivers/virtio/virtio_pci_modern.c | 10 +++++++++-
>>>>>   1 file changed, 9 insertions(+), 1 deletion(-)
>>>>>
>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>> index cc3412a96a17..dcee616e8d21 100644
>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>> @@ -162,6 +162,7 @@ static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>> +    unsigned long timeout = jiffies + msecs_to_jiffies(180000);
>>>>>         /* 0 status means a reset. */
>>>>>       vp_modern_set_status(mdev, 0);
>>>>> @@ -169,9 +170,16 @@ static int vp_reset(struct virtio_device *vdev)
>>>>>        * device_status to return 0 before reinitializing the device.
>>>>>        * This will flush out the status write, and flush in device 
>>>>> writes,
>>>>>        * including MSI-X interrupts, if any.
>>>>> +     * Set a timeout before giving up on the device.
>>>>>        */
>>>>> -    while (vp_modern_get_status(mdev))
>>>>> +    while (vp_modern_get_status(mdev)) {
>>>>> +        if (time_after(jiffies, timeout)) {
>>>>
>>>>
>>>> What happens if the device finish the rest after the timeout?
>>>
>>>
>>> The driver will set VIRTIO_CONFIG_S_FAILED and one can re-probe it 
>>> later on (e.g by re-scanning the pci bus).
>>
>>
>> Ok, so do we need the flush through vp_synchronize_vectors() here?
>
> If the device didn't write 0 to status I guess we don't need that.
>
> The device shouldn't raise any interrupt before negotiation finish 
> successfully.


The reset could be triggered in other places like driver removing.

Thanks


>
> MST, is that correct ?
>
>>
>> Thanks
>>
>>
>>>
>>>
>>>>
>>>> Thanks
>>>>
>>>>
>>>>> + dev_err(&vdev->dev, "virtio: device not ready. "
>>>>> +                "Aborting. Try again later\n");
>>>>> +            return -EAGAIN;
>>>>> +        }
>>>>>           msleep(1);
>>>>> +    }
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>>       return 0;
>>>>
>>>
>>
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/8/2021 4:14 PM, Jason Wang wrote:
>
> 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>>
>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>
>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>> The reset device operation, usually is an operation that might fail 
>>>> from
>>>> various reasons. For example, the controller might be in a bad 
>>>> state and
>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>> devices always succeed in reset operation but this is not the case for
>>>> HW based virtio devices.
>>>
>>>
>>> I would like to know under what condition that the reset operation 
>>> may fail (except for the case of a bugg guest).
>>
>> The controller might not be ready or stuck. This is a real use case 
>> for many PCI devices.
>>
>> For real devices the FW might be in a bad state and it can happen 
>> also for paravirt device if you have a bug in the controller code or 
>> if you entered some error flow (Out of memory).
>>
>> You don't want to be stuck because of one bad device.
>
>
> So the buggy driver can damage the host through various ways, I'm not 
> sure doing such workaround is worthwhile.

do you mean device ?

sometimes you need to replace device FW and it will work.

I don't think it's a workaround. Other protocols, such as NVMe, solved 
this in the specification.

PCI config space and PCI controller are sometimes 2 different components 
and sometimes the controller is not active, although the device is 
plugged and seen by the PCI subsystem.


>
> Note that this driver has been used for real hardware PCI devices for 
> many years. We don't receive any report of this before.
>
>
>>
>>
>>>
>>>
>>>>
>>>> This commit is also a preparation for adding a timeout mechanism for
>>>> resetting virtio devices.
>>>>
>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>> ---
>>>>
>>>> changes from v1:
>>>>   - update virtio_ccw.c (Cornelia)
>>>>   - update virtio_uml.c
>>>>   - update mlxbf-tmfifo.c
>>>
>>>
>>> Note that virtio driver may call reset, so you probably need to 
>>> convert them.
>>
>> I'm sure I understand.
>>
>> Convert to what ?
>
>
> Convert to deal with the possible rest failure. E.g in 
> virtblk_freeze() we had:
>
> static int virtblk_freeze(struct virtio_device *vdev)
> {
>         struct virtio_blk *vblk = vdev->priv;
>
>         /* Ensure we don't receive any more interrupts */
>         vdev->config->reset(vdev);
> ...
>
> We need fail the freeze here.


Agree.


>
> Another example is the driver remove which is not expected to be fail. 
> A lot of virtio drivers tries to call reset there. I'm not sure how 
> hard to deal with the failure in the path of remove (e.g 
> __device_release_driver tends to ignore the return value of 
> bus->remove().)

I think it can stay as-is and ignore the ->reset return value and 
continue free the other resources to avoid leakage.


>
> Thanks
>
>
>>
>> Thanks.
>>
>>>
>>> Thanks
>>>
>>>
>>>>
>>>> ---
>>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>   include/linux/virtio_config.h            |  5 +++--
>>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>>
>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>> b/arch/um/drivers/virtio_uml.c
>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>> --- a/arch/um/drivers/virtio_uml.c
>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>> virtio_device *vdev, u8 status)
>>>>       vu_dev->status = status;
>>>>   }
>>>>   -static void vu_reset(struct virtio_device *vdev)
>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>         vu_dev->status = 0;
>>>> +
>>>> +    return 0;
>>>>   }
>>>>     static void vu_del_vq(struct virtqueue *vq)
>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> @@ -980,11 +980,13 @@ static void 
>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>   }
>>>>     /* Reset the device. Not much here for now. */
>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>         tm_vdev->status = 0;
>>>> +
>>>> +    return 0;
>>>>   }
>>>>     /* Read the value of a configuration field. */
>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>>> virtio_device *vdev, u8 status)
>>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>   }
>>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>       struct fw_rsc_vdev *rsc;
>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>>> virtio_device *vdev)
>>>>         rsc->status = 0;
>>>>       dev_dbg(&vdev->dev, "reset !\n");
>>>> +
>>>> +    return 0;
>>>>   }
>>>>     /* provide the vdev features as retrieved from the firmware */
>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>> index 54e686dca6de..52b32555e746 100644
>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>>> virtio_device *vdev, unsigned nvqs,
>>>>       return ret;
>>>>   }
>>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>       struct ccw1 *ccw;
>>>> +    int ret;
>>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>       if (!ccw)
>>>> -        return;
>>>> +        return -ENOMEM;
>>>>         /* Zero status bits. */
>>>>       vcdev->dma_area->status = 0;
>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>>> virtio_device *vdev)
>>>>       ccw->flags = 0;
>>>>       ccw->count = 0;
>>>>       ccw->cda = 0;
>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>> +
>>>> +    return ret;
>>>>   }
>>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>> --- a/drivers/virtio/virtio.c
>>>> +++ b/drivers/virtio/virtio.c
>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device 
>>>> *dev)
>>>>       /* Assign a unique device index and hence name. */
>>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>       if (err < 0)
>>>> -        goto out;
>>>> +        goto out_err;
>>>>         dev->index = err;
>>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device 
>>>> *dev)
>>>>         /* We always start by resetting the device, in case a previous
>>>>        * driver messed it up.  This also tests that code path a 
>>>> little. */
>>>> -    dev->config->reset(dev);
>>>> +    err = dev->config->reset(dev);
>>>> +    if (err)
>>>> +        goto out_ida;
>>>>         /* Acknowledge that we've seen the device. */
>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>>> virtio_device *dev)
>>>>        */
>>>>       err = device_add(&dev->dev);
>>>>       if (err)
>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>> -out:
>>>> -    if (err)
>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>> +        goto out_ida;
>>>> +
>>>> +    return 0;
>>>> +
>>>> +out_ida:
>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>> +out_err:
>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>       return err;
>>>>   }
>>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device 
>>>> *dev)
>>>>         /* We always start by resetting the device, in case a previous
>>>>        * driver messed it up. */
>>>> -    dev->config->reset(dev);
>>>> +    ret = dev->config->reset(dev);
>>>> +    if (ret)
>>>> +        goto err;
>>>>         /* Acknowledge that we've seen the device. */
>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>> b/drivers/virtio/virtio_mmio.c
>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>> --- a/drivers/virtio/virtio_mmio.c
>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>> virtio_device *vdev, u8 status)
>>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>   }
>>>>   -static void vm_reset(struct virtio_device *vdev)
>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>>         /* 0 status means a reset. */
>>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>> +    return 0;
>>>>   }
>>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>>> *vdev, u8 status)
>>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>   }
>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>       /* 0 status means a reset. */
>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>       vp_synchronize_vectors(vdev);
>>>> +
>>>> +    return 0;
>>>>   }
>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>> u16 vector)
>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>> b/drivers/virtio/virtio_pci_modern.c
>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>>>> *vdev, u8 status)
>>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>>   }
>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>           msleep(1);
>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>       vp_synchronize_vectors(vdev);
>>>> +    return 0;
>>>>   }
>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>> u16 vector)
>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>> b/drivers/virtio/virtio_vdpa.c
>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>>> virtio_device *vdev, u8 status)
>>>>       return ops->set_status(vdpa, status);
>>>>   }
>>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>   {
>>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>         vdpa_reset(vdpa);
>>>> +
>>>> +    return 0;
>>>>   }
>>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>> diff --git a/include/linux/virtio_config.h 
>>>> b/include/linux/virtio_config.h
>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>> --- a/include/linux/virtio_config.h
>>>> +++ b/include/linux/virtio_config.h
>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>    *    status: the new status byte
>>>>    * @reset: reset the device
>>>>    *    vdev: the virtio device
>>>> - *    After this, status and feature negotiation must be done again
>>>> + *    Upon success, status and feature negotiation must be done again
>>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>>    *    parallel with being added/removed.
>>>> + *    Returns 0 on success or error status.
>>>>    * @find_vqs: find virtqueues and instantiate them.
>>>>    *    vdev: the virtio_device
>>>>    *    nvqs: the number of virtqueues to find
>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>       u32 (*generation)(struct virtio_device *vdev);
>>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>>> -    void (*reset)(struct virtio_device *vdev);
>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>               const char * const names[], const bool *ctx,
>>>
>>
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> 
> On 4/8/2021 11:58 AM, Jason Wang wrote:
> > 
> > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > The reset device operation, usually is an operation that might fail from
> > > various reasons. For example, the controller might be in a bad state and
> > > can't answer to any request. Usually, the paravirt SW based virtio
> > > devices always succeed in reset operation but this is not the case for
> > > HW based virtio devices.
> > 
> > 
> > I would like to know under what condition that the reset operation may
> > fail (except for the case of a bugg guest).
> 
> The controller might not be ready or stuck. This is a real use case for many
> PCI devices.
> 
> For real devices the FW might be in a bad state and it can happen also for
> paravirt device if you have a bug in the controller code or if you entered
> some error flow (Out of memory).
> 
> You don't want to be stuck because of one bad device.

OK so maybe we can do more to detect the bad device.
Won't we get all 1's on a read in this case?


> 
> > 
> > 
> > > 
> > > This commit is also a preparation for adding a timeout mechanism for
> > > resetting virtio devices.
> > > 
> > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > ---
> > > 
> > > changes from v1:
> > >   - update virtio_ccw.c (Cornelia)
> > >   - update virtio_uml.c
> > >   - update mlxbf-tmfifo.c
> > 
> > 
> > Note that virtio driver may call reset, so you probably need to convert
> > them.
> 
> I'm sure I understand.
> 
> Convert to what ?
> 
> Thanks.
> 
> > 
> > Thanks
> > 
> > 
> > > 
> > > ---
> > >   arch/um/drivers/virtio_uml.c             |  4 +++-
> > >   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > >   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > >   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > >   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > >   drivers/virtio/virtio_mmio.c             |  3 ++-
> > >   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > >   drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > >   drivers/virtio/virtio_vdpa.c             |  4 +++-
> > >   include/linux/virtio_config.h            |  5 +++--
> > >   10 files changed, 43 insertions(+), 19 deletions(-)
> > > 
> > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > index 91ddf74ca888..b6e66265ed32 100644
> > > --- a/arch/um/drivers/virtio_uml.c
> > > +++ b/arch/um/drivers/virtio_uml.c
> > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       vu_dev->status = status;
> > >   }
> > >   -static void vu_reset(struct virtio_device *vdev)
> > > +static int vu_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > >         vu_dev->status = 0;
> > > +
> > > +    return 0;
> > >   }
> > >     static void vu_del_vq(struct virtqueue *vq)
> > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > @@ -980,11 +980,13 @@ static void
> > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > >   }
> > >     /* Reset the device. Not much here for now. */
> > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > >   {
> > >       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > >         tm_vdev->status = 0;
> > > +
> > > +    return 0;
> > >   }
> > >     /* Read the value of a configuration field. */
> > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > b/drivers/remoteproc/remoteproc_virtio.c
> > > index 0cc617f76068..ca9573c62c3d 100644
> > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > virtio_device *vdev, u8 status)
> > >       dev_dbg(&vdev->dev, "status: %d\n", status);
> > >   }
> > >   -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > >   {
> > >       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > >       struct fw_rsc_vdev *rsc;
> > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > virtio_device *vdev)
> > >         rsc->status = 0;
> > >       dev_dbg(&vdev->dev, "reset !\n");
> > > +
> > > +    return 0;
> > >   }
> > >     /* provide the vdev features as retrieved from the firmware */
> > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > b/drivers/s390/virtio/virtio_ccw.c
> > > index 54e686dca6de..52b32555e746 100644
> > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > virtio_device *vdev, unsigned nvqs,
> > >       return ret;
> > >   }
> > >   -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > >       struct ccw1 *ccw;
> > > +    int ret;
> > >         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > >       if (!ccw)
> > > -        return;
> > > +        return -ENOMEM;
> > >         /* Zero status bits. */
> > >       vcdev->dma_area->status = 0;
> > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > virtio_device *vdev)
> > >       ccw->flags = 0;
> > >       ccw->count = 0;
> > >       ccw->cda = 0;
> > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > >       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > +
> > > +    return ret;
> > >   }
> > >     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > --- a/drivers/virtio/virtio.c
> > > +++ b/drivers/virtio/virtio.c
> > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > *dev)
> > >       /* Assign a unique device index and hence name. */
> > >       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > >       if (err < 0)
> > > -        goto out;
> > > +        goto out_err;
> > >         dev->index = err;
> > >       dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > *dev)
> > >         /* We always start by resetting the device, in case a previous
> > >        * driver messed it up.  This also tests that code path a
> > > little. */
> > > -    dev->config->reset(dev);
> > > +    err = dev->config->reset(dev);
> > > +    if (err)
> > > +        goto out_ida;
> > >         /* Acknowledge that we've seen the device. */
> > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > virtio_device *dev)
> > >        */
> > >       err = device_add(&dev->dev);
> > >       if (err)
> > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > -out:
> > > -    if (err)
> > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > +        goto out_ida;
> > > +
> > > +    return 0;
> > > +
> > > +out_ida:
> > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > +out_err:
> > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > >       return err;
> > >   }
> > >   EXPORT_SYMBOL_GPL(register_virtio_device);
> > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > >         /* We always start by resetting the device, in case a previous
> > >        * driver messed it up. */
> > > -    dev->config->reset(dev);
> > > +    ret = dev->config->reset(dev);
> > > +    if (ret)
> > > +        goto err;
> > >         /* Acknowledge that we've seen the device. */
> > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > index 56128b9c46eb..12b8f048c48d 100644
> > > --- a/drivers/virtio/virtio_mmio.c
> > > +++ b/drivers/virtio/virtio_mmio.c
> > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > >   }
> > >   -static void vm_reset(struct virtio_device *vdev)
> > > +static int vm_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > >         /* 0 status means a reset. */
> > >       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > +    return 0;
> > >   }
> > >     diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > b/drivers/virtio/virtio_pci_legacy.c
> > > index d62e9835aeec..0b5d95e3efa1 100644
> > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > >   }
> > >   -static void vp_reset(struct virtio_device *vdev)
> > > +static int vp_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > >       /* 0 status means a reset. */
> > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > >       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > >       /* Flush pending VQ/configuration callbacks. */
> > >       vp_synchronize_vectors(vdev);
> > > +
> > > +    return 0;
> > >   }
> > >     static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > u16 vector)
> > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > b/drivers/virtio/virtio_pci_modern.c
> > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > --- a/drivers/virtio/virtio_pci_modern.c
> > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       vp_modern_set_status(&vp_dev->mdev, status);
> > >   }
> > >   -static void vp_reset(struct virtio_device *vdev)
> > > +static int vp_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > >       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > >           msleep(1);
> > >       /* Flush pending VQ/configuration callbacks. */
> > >       vp_synchronize_vectors(vdev);
> > > +    return 0;
> > >   }
> > >     static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > u16 vector)
> > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > index e28acf482e0c..5fd4e627a9b0 100644
> > > --- a/drivers/virtio/virtio_vdpa.c
> > > +++ b/drivers/virtio/virtio_vdpa.c
> > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > virtio_device *vdev, u8 status)
> > >       return ops->set_status(vdpa, status);
> > >   }
> > >   -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > >   {
> > >       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > >         vdpa_reset(vdpa);
> > > +
> > > +    return 0;
> > >   }
> > >     static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > diff --git a/include/linux/virtio_config.h
> > > b/include/linux/virtio_config.h
> > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > --- a/include/linux/virtio_config.h
> > > +++ b/include/linux/virtio_config.h
> > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > >    *    status: the new status byte
> > >    * @reset: reset the device
> > >    *    vdev: the virtio device
> > > - *    After this, status and feature negotiation must be done again
> > > + *    Upon success, status and feature negotiation must be done again
> > >    *    Device must not be reset from its vq/config callbacks, or in
> > >    *    parallel with being added/removed.
> > > + *    Returns 0 on success or error status.
> > >    * @find_vqs: find virtqueues and instantiate them.
> > >    *    vdev: the virtio_device
> > >    *    nvqs: the number of virtqueues to find
> > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > >       u32 (*generation)(struct virtio_device *vdev);
> > >       u8 (*get_status)(struct virtio_device *vdev);
> > >       void (*set_status)(struct virtio_device *vdev, u8 status);
> > > -    void (*reset)(struct virtio_device *vdev);
> > > +    int (*reset)(struct virtio_device *vdev);
> > >       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > >               struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > >               const char * const names[], const bool *ctx,
> > 

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> 
> On 4/8/2021 11:58 AM, Jason Wang wrote:
> > 
> > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > The reset device operation, usually is an operation that might fail from
> > > various reasons. For example, the controller might be in a bad state and
> > > can't answer to any request. Usually, the paravirt SW based virtio
> > > devices always succeed in reset operation but this is not the case for
> > > HW based virtio devices.
> > 
> > 
> > I would like to know under what condition that the reset operation may
> > fail (except for the case of a bugg guest).
> 
> The controller might not be ready or stuck. This is a real use case for many
> PCI devices.
> 
> For real devices the FW might be in a bad state and it can happen also for
> paravirt device if you have a bug in the controller code or if you entered
> some error flow (Out of memory).
> 
> You don't want to be stuck because of one bad device.

OK so maybe we can do more to detect the bad device.
Won't we get all 1's on a read in this case?


> 
> > 
> > 
> > > 
> > > This commit is also a preparation for adding a timeout mechanism for
> > > resetting virtio devices.
> > > 
> > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > ---
> > > 
> > > changes from v1:
> > >   - update virtio_ccw.c (Cornelia)
> > >   - update virtio_uml.c
> > >   - update mlxbf-tmfifo.c
> > 
> > 
> > Note that virtio driver may call reset, so you probably need to convert
> > them.
> 
> I'm sure I understand.
> 
> Convert to what ?
> 
> Thanks.
> 
> > 
> > Thanks
> > 
> > 
> > > 
> > > ---
> > >   arch/um/drivers/virtio_uml.c             |  4 +++-
> > >   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > >   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > >   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > >   drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > >   drivers/virtio/virtio_mmio.c             |  3 ++-
> > >   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > >   drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > >   drivers/virtio/virtio_vdpa.c             |  4 +++-
> > >   include/linux/virtio_config.h            |  5 +++--
> > >   10 files changed, 43 insertions(+), 19 deletions(-)
> > > 
> > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > index 91ddf74ca888..b6e66265ed32 100644
> > > --- a/arch/um/drivers/virtio_uml.c
> > > +++ b/arch/um/drivers/virtio_uml.c
> > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       vu_dev->status = status;
> > >   }
> > >   -static void vu_reset(struct virtio_device *vdev)
> > > +static int vu_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > >         vu_dev->status = 0;
> > > +
> > > +    return 0;
> > >   }
> > >     static void vu_del_vq(struct virtqueue *vq)
> > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > @@ -980,11 +980,13 @@ static void
> > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > >   }
> > >     /* Reset the device. Not much here for now. */
> > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > >   {
> > >       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > >         tm_vdev->status = 0;
> > > +
> > > +    return 0;
> > >   }
> > >     /* Read the value of a configuration field. */
> > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > b/drivers/remoteproc/remoteproc_virtio.c
> > > index 0cc617f76068..ca9573c62c3d 100644
> > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > virtio_device *vdev, u8 status)
> > >       dev_dbg(&vdev->dev, "status: %d\n", status);
> > >   }
> > >   -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > >   {
> > >       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > >       struct fw_rsc_vdev *rsc;
> > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > virtio_device *vdev)
> > >         rsc->status = 0;
> > >       dev_dbg(&vdev->dev, "reset !\n");
> > > +
> > > +    return 0;
> > >   }
> > >     /* provide the vdev features as retrieved from the firmware */
> > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > b/drivers/s390/virtio/virtio_ccw.c
> > > index 54e686dca6de..52b32555e746 100644
> > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > virtio_device *vdev, unsigned nvqs,
> > >       return ret;
> > >   }
> > >   -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > >       struct ccw1 *ccw;
> > > +    int ret;
> > >         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > >       if (!ccw)
> > > -        return;
> > > +        return -ENOMEM;
> > >         /* Zero status bits. */
> > >       vcdev->dma_area->status = 0;
> > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > virtio_device *vdev)
> > >       ccw->flags = 0;
> > >       ccw->count = 0;
> > >       ccw->cda = 0;
> > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > >       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > +
> > > +    return ret;
> > >   }
> > >     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > --- a/drivers/virtio/virtio.c
> > > +++ b/drivers/virtio/virtio.c
> > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > *dev)
> > >       /* Assign a unique device index and hence name. */
> > >       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > >       if (err < 0)
> > > -        goto out;
> > > +        goto out_err;
> > >         dev->index = err;
> > >       dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > *dev)
> > >         /* We always start by resetting the device, in case a previous
> > >        * driver messed it up.  This also tests that code path a
> > > little. */
> > > -    dev->config->reset(dev);
> > > +    err = dev->config->reset(dev);
> > > +    if (err)
> > > +        goto out_ida;
> > >         /* Acknowledge that we've seen the device. */
> > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > virtio_device *dev)
> > >        */
> > >       err = device_add(&dev->dev);
> > >       if (err)
> > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > -out:
> > > -    if (err)
> > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > +        goto out_ida;
> > > +
> > > +    return 0;
> > > +
> > > +out_ida:
> > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > +out_err:
> > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > >       return err;
> > >   }
> > >   EXPORT_SYMBOL_GPL(register_virtio_device);
> > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > >         /* We always start by resetting the device, in case a previous
> > >        * driver messed it up. */
> > > -    dev->config->reset(dev);
> > > +    ret = dev->config->reset(dev);
> > > +    if (ret)
> > > +        goto err;
> > >         /* Acknowledge that we've seen the device. */
> > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > index 56128b9c46eb..12b8f048c48d 100644
> > > --- a/drivers/virtio/virtio_mmio.c
> > > +++ b/drivers/virtio/virtio_mmio.c
> > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > >   }
> > >   -static void vm_reset(struct virtio_device *vdev)
> > > +static int vm_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > >         /* 0 status means a reset. */
> > >       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > +    return 0;
> > >   }
> > >     diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > b/drivers/virtio/virtio_pci_legacy.c
> > > index d62e9835aeec..0b5d95e3efa1 100644
> > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > >   }
> > >   -static void vp_reset(struct virtio_device *vdev)
> > > +static int vp_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > >       /* 0 status means a reset. */
> > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > >       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > >       /* Flush pending VQ/configuration callbacks. */
> > >       vp_synchronize_vectors(vdev);
> > > +
> > > +    return 0;
> > >   }
> > >     static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > u16 vector)
> > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > b/drivers/virtio/virtio_pci_modern.c
> > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > --- a/drivers/virtio/virtio_pci_modern.c
> > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > *vdev, u8 status)
> > >       vp_modern_set_status(&vp_dev->mdev, status);
> > >   }
> > >   -static void vp_reset(struct virtio_device *vdev)
> > > +static int vp_reset(struct virtio_device *vdev)
> > >   {
> > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > >       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > >           msleep(1);
> > >       /* Flush pending VQ/configuration callbacks. */
> > >       vp_synchronize_vectors(vdev);
> > > +    return 0;
> > >   }
> > >     static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > u16 vector)
> > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > index e28acf482e0c..5fd4e627a9b0 100644
> > > --- a/drivers/virtio/virtio_vdpa.c
> > > +++ b/drivers/virtio/virtio_vdpa.c
> > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > virtio_device *vdev, u8 status)
> > >       return ops->set_status(vdpa, status);
> > >   }
> > >   -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > >   {
> > >       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > >         vdpa_reset(vdpa);
> > > +
> > > +    return 0;
> > >   }
> > >     static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > diff --git a/include/linux/virtio_config.h
> > > b/include/linux/virtio_config.h
> > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > --- a/include/linux/virtio_config.h
> > > +++ b/include/linux/virtio_config.h
> > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > >    *    status: the new status byte
> > >    * @reset: reset the device
> > >    *    vdev: the virtio device
> > > - *    After this, status and feature negotiation must be done again
> > > + *    Upon success, status and feature negotiation must be done again
> > >    *    Device must not be reset from its vq/config callbacks, or in
> > >    *    parallel with being added/removed.
> > > + *    Returns 0 on success or error status.
> > >    * @find_vqs: find virtqueues and instantiate them.
> > >    *    vdev: the virtio_device
> > >    *    nvqs: the number of virtqueues to find
> > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > >       u32 (*generation)(struct virtio_device *vdev);
> > >       u8 (*get_status)(struct virtio_device *vdev);
> > >       void (*set_status)(struct virtio_device *vdev, u8 status);
> > > -    void (*reset)(struct virtio_device *vdev);
> > > +    int (*reset)(struct virtio_device *vdev);
> > >       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > >               struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > >               const char * const names[], const bool *ctx,
> > 

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Thu, Apr 08, 2021 at 08:11:07AM +0000, Max Gurtovoy wrote:
> The reset device operation, usually is an operation that might fail from
> various reasons. For example, the controller might be in a bad state and
> can't answer to any request. Usually, the paravirt SW based virtio
> devices always succeed in reset operation but this is not the case for
> HW based virtio devices.

Well it reports the error to callers however
no callers check the return value. So what good is it?
If callers are expected to just proceed then we don't
really need to return a value at all ...

> This commit is also a preparation for adding a timeout mechanism for
> resetting virtio devices.
> 
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
> 
> changes from v1:
>  - update virtio_ccw.c (Cornelia)
>  - update virtio_uml.c
>  - update mlxbf-tmfifo.c
> 
> ---
>  arch/um/drivers/virtio_uml.c             |  4 +++-
>  drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>  drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>  drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>  drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>  drivers/virtio/virtio_mmio.c             |  3 ++-
>  drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>  drivers/virtio/virtio_pci_modern.c       |  3 ++-
>  drivers/virtio/virtio_vdpa.c             |  4 +++-
>  include/linux/virtio_config.h            |  5 +++--
>  10 files changed, 43 insertions(+), 19 deletions(-)
> 
> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> index 91ddf74ca888..b6e66265ed32 100644
> --- a/arch/um/drivers/virtio_uml.c
> +++ b/arch/um/drivers/virtio_uml.c
> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device *vdev, u8 status)
>  	vu_dev->status = status;
>  }
>  
> -static void vu_reset(struct virtio_device *vdev)
> +static int vu_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>  
>  	vu_dev->status = 0;
> +
> +	return 0;
>  }
>  
>  static void vu_del_vq(struct virtqueue *vq)
> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
> index bbc4e71a16ff..c192b8ac5d9e 100644
> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> @@ -980,11 +980,13 @@ static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>  }
>  
>  /* Reset the device. Not much here for now. */
> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>  {
>  	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>  
>  	tm_vdev->status = 0;
> +
> +	return 0;
>  }
>  
>  /* Read the value of a configuration field. */
> diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
> index 0cc617f76068..ca9573c62c3d 100644
> --- a/drivers/remoteproc/remoteproc_virtio.c
> +++ b/drivers/remoteproc/remoteproc_virtio.c
> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status)
>  	dev_dbg(&vdev->dev, "status: %d\n", status);
>  }
>  
> -static void rproc_virtio_reset(struct virtio_device *vdev)
> +static int rproc_virtio_reset(struct virtio_device *vdev)
>  {
>  	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>  	struct fw_rsc_vdev *rsc;
> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
>  
>  	rsc->status = 0;
>  	dev_dbg(&vdev->dev, "reset !\n");
> +
> +	return 0;
>  }
>  
>  /* provide the vdev features as retrieved from the firmware */
> diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
> index 54e686dca6de..52b32555e746 100644
> --- a/drivers/s390/virtio/virtio_ccw.c
> +++ b/drivers/s390/virtio/virtio_ccw.c
> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
>  	return ret;
>  }
>  
> -static void virtio_ccw_reset(struct virtio_device *vdev)
> +static int virtio_ccw_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>  	struct ccw1 *ccw;
> +	int ret;
>  
>  	ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>  	if (!ccw)
> -		return;
> +		return -ENOMEM;
>  
>  	/* Zero status bits. */
>  	vcdev->dma_area->status = 0;
> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
>  	ccw->flags = 0;
>  	ccw->count = 0;
>  	ccw->cda = 0;
> -	ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> +	ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>  	ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> +
> +	return ret;
>  }
>  
>  static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> --- a/drivers/virtio/virtio.c
> +++ b/drivers/virtio/virtio.c
> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device *dev)
>  	/* Assign a unique device index and hence name. */
>  	err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>  	if (err < 0)
> -		goto out;
> +		goto out_err;
>  
>  	dev->index = err;
>  	dev_set_name(&dev->dev, "virtio%u", dev->index);
> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device *dev)
>  
>  	/* We always start by resetting the device, in case a previous
>  	 * driver messed it up.  This also tests that code path a little. */
> -	dev->config->reset(dev);
> +	err = dev->config->reset(dev);
> +	if (err)
> +		goto out_ida;
>  
>  	/* Acknowledge that we've seen the device. */
>  	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> @@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device *dev)
>  	 */
>  	err = device_add(&dev->dev);
>  	if (err)
> -		ida_simple_remove(&virtio_index_ida, dev->index);
> -out:
> -	if (err)
> -		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +		goto out_ida;
> +
> +	return 0;
> +
> +out_ida:
> +	ida_simple_remove(&virtio_index_ida, dev->index);
> +out_err:
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>  	return err;
>  }
>  EXPORT_SYMBOL_GPL(register_virtio_device);
> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>  
>  	/* We always start by resetting the device, in case a previous
>  	 * driver messed it up. */
> -	dev->config->reset(dev);
> +	ret = dev->config->reset(dev);
> +	if (ret)
> +		goto err;
>  
>  	/* Acknowledge that we've seen the device. */
>  	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> index 56128b9c46eb..12b8f048c48d 100644
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)
>  	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>  }
>  
> -static void vm_reset(struct virtio_device *vdev)
> +static int vm_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>  
>  	/* 0 status means a reset. */
>  	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> +	return 0;
>  }
>  
>  
> diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
> index d62e9835aeec..0b5d95e3efa1 100644
> --- a/drivers/virtio/virtio_pci_legacy.c
> +++ b/drivers/virtio/virtio_pci_legacy.c
> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>  	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>  }
>  
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>  	/* 0 status means a reset. */
> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>  	ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>  	/* Flush pending VQ/configuration callbacks. */
>  	vp_synchronize_vectors(vdev);
> +
> +	return 0;
>  }
>  
>  static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index fbd4ebc00eb6..cc3412a96a17 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>  	vp_modern_set_status(&vp_dev->mdev, status);
>  }
>  
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>  	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>  		msleep(1);
>  	/* Flush pending VQ/configuration callbacks. */
>  	vp_synchronize_vectors(vdev);
> +	return 0;
>  }
>  
>  static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> index e28acf482e0c..5fd4e627a9b0 100644
> --- a/drivers/virtio/virtio_vdpa.c
> +++ b/drivers/virtio/virtio_vdpa.c
> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
>  	return ops->set_status(vdpa, status);
>  }
>  
> -static void virtio_vdpa_reset(struct virtio_device *vdev)
> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>  {
>  	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>  
>  	vdpa_reset(vdpa);
> +
> +	return 0;
>  }
>  
>  static bool virtio_vdpa_notify(struct virtqueue *vq)
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 8519b3ae5d52..d2b0f1699a75 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>   *	status: the new status byte
>   * @reset: reset the device
>   *	vdev: the virtio device
> - *	After this, status and feature negotiation must be done again
> + *	Upon success, status and feature negotiation must be done again
>   *	Device must not be reset from its vq/config callbacks, or in
>   *	parallel with being added/removed.
> + *	Returns 0 on success or error status.
>   * @find_vqs: find virtqueues and instantiate them.
>   *	vdev: the virtio_device
>   *	nvqs: the number of virtqueues to find
> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>  	u32 (*generation)(struct virtio_device *vdev);
>  	u8 (*get_status)(struct virtio_device *vdev);
>  	void (*set_status)(struct virtio_device *vdev, u8 status);
> -	void (*reset)(struct virtio_device *vdev);
> +	int (*reset)(struct virtio_device *vdev);
>  	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>  			struct virtqueue *vqs[], vq_callback_t *callbacks[],
>  			const char * const names[], const bool *ctx,
> -- 
> 2.25.4

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Thu, Apr 08, 2021 at 08:11:07AM +0000, Max Gurtovoy wrote:
> The reset device operation, usually is an operation that might fail from
> various reasons. For example, the controller might be in a bad state and
> can't answer to any request. Usually, the paravirt SW based virtio
> devices always succeed in reset operation but this is not the case for
> HW based virtio devices.

Well it reports the error to callers however
no callers check the return value. So what good is it?
If callers are expected to just proceed then we don't
really need to return a value at all ...

> This commit is also a preparation for adding a timeout mechanism for
> resetting virtio devices.
> 
> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> ---
> 
> changes from v1:
>  - update virtio_ccw.c (Cornelia)
>  - update virtio_uml.c
>  - update mlxbf-tmfifo.c
> 
> ---
>  arch/um/drivers/virtio_uml.c             |  4 +++-
>  drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>  drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>  drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>  drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>  drivers/virtio/virtio_mmio.c             |  3 ++-
>  drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>  drivers/virtio/virtio_pci_modern.c       |  3 ++-
>  drivers/virtio/virtio_vdpa.c             |  4 +++-
>  include/linux/virtio_config.h            |  5 +++--
>  10 files changed, 43 insertions(+), 19 deletions(-)
> 
> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> index 91ddf74ca888..b6e66265ed32 100644
> --- a/arch/um/drivers/virtio_uml.c
> +++ b/arch/um/drivers/virtio_uml.c
> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device *vdev, u8 status)
>  	vu_dev->status = status;
>  }
>  
> -static void vu_reset(struct virtio_device *vdev)
> +static int vu_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>  
>  	vu_dev->status = 0;
> +
> +	return 0;
>  }
>  
>  static void vu_del_vq(struct virtqueue *vq)
> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
> index bbc4e71a16ff..c192b8ac5d9e 100644
> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> @@ -980,11 +980,13 @@ static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>  }
>  
>  /* Reset the device. Not much here for now. */
> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>  {
>  	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>  
>  	tm_vdev->status = 0;
> +
> +	return 0;
>  }
>  
>  /* Read the value of a configuration field. */
> diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
> index 0cc617f76068..ca9573c62c3d 100644
> --- a/drivers/remoteproc/remoteproc_virtio.c
> +++ b/drivers/remoteproc/remoteproc_virtio.c
> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct virtio_device *vdev, u8 status)
>  	dev_dbg(&vdev->dev, "status: %d\n", status);
>  }
>  
> -static void rproc_virtio_reset(struct virtio_device *vdev)
> +static int rproc_virtio_reset(struct virtio_device *vdev)
>  {
>  	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>  	struct fw_rsc_vdev *rsc;
> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
>  
>  	rsc->status = 0;
>  	dev_dbg(&vdev->dev, "reset !\n");
> +
> +	return 0;
>  }
>  
>  /* provide the vdev features as retrieved from the firmware */
> diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
> index 54e686dca6de..52b32555e746 100644
> --- a/drivers/s390/virtio/virtio_ccw.c
> +++ b/drivers/s390/virtio/virtio_ccw.c
> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
>  	return ret;
>  }
>  
> -static void virtio_ccw_reset(struct virtio_device *vdev)
> +static int virtio_ccw_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>  	struct ccw1 *ccw;
> +	int ret;
>  
>  	ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>  	if (!ccw)
> -		return;
> +		return -ENOMEM;
>  
>  	/* Zero status bits. */
>  	vcdev->dma_area->status = 0;
> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct virtio_device *vdev)
>  	ccw->flags = 0;
>  	ccw->count = 0;
>  	ccw->cda = 0;
> -	ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> +	ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>  	ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> +
> +	return ret;
>  }
>  
>  static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> --- a/drivers/virtio/virtio.c
> +++ b/drivers/virtio/virtio.c
> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device *dev)
>  	/* Assign a unique device index and hence name. */
>  	err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>  	if (err < 0)
> -		goto out;
> +		goto out_err;
>  
>  	dev->index = err;
>  	dev_set_name(&dev->dev, "virtio%u", dev->index);
> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device *dev)
>  
>  	/* We always start by resetting the device, in case a previous
>  	 * driver messed it up.  This also tests that code path a little. */
> -	dev->config->reset(dev);
> +	err = dev->config->reset(dev);
> +	if (err)
> +		goto out_ida;
>  
>  	/* Acknowledge that we've seen the device. */
>  	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> @@ -362,10 +364,14 @@ int register_virtio_device(struct virtio_device *dev)
>  	 */
>  	err = device_add(&dev->dev);
>  	if (err)
> -		ida_simple_remove(&virtio_index_ida, dev->index);
> -out:
> -	if (err)
> -		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +		goto out_ida;
> +
> +	return 0;
> +
> +out_ida:
> +	ida_simple_remove(&virtio_index_ida, dev->index);
> +out_err:
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>  	return err;
>  }
>  EXPORT_SYMBOL_GPL(register_virtio_device);
> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>  
>  	/* We always start by resetting the device, in case a previous
>  	 * driver messed it up. */
> -	dev->config->reset(dev);
> +	ret = dev->config->reset(dev);
> +	if (ret)
> +		goto err;
>  
>  	/* Acknowledge that we've seen the device. */
>  	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> index 56128b9c46eb..12b8f048c48d 100644
> --- a/drivers/virtio/virtio_mmio.c
> +++ b/drivers/virtio/virtio_mmio.c
> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)
>  	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>  }
>  
> -static void vm_reset(struct virtio_device *vdev)
> +static int vm_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>  
>  	/* 0 status means a reset. */
>  	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> +	return 0;
>  }
>  
>  
> diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
> index d62e9835aeec..0b5d95e3efa1 100644
> --- a/drivers/virtio/virtio_pci_legacy.c
> +++ b/drivers/virtio/virtio_pci_legacy.c
> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>  	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>  }
>  
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>  	/* 0 status means a reset. */
> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>  	ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>  	/* Flush pending VQ/configuration callbacks. */
>  	vp_synchronize_vectors(vdev);
> +
> +	return 0;
>  }
>  
>  static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
> index fbd4ebc00eb6..cc3412a96a17 100644
> --- a/drivers/virtio/virtio_pci_modern.c
> +++ b/drivers/virtio/virtio_pci_modern.c
> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device *vdev, u8 status)
>  	vp_modern_set_status(&vp_dev->mdev, status);
>  }
>  
> -static void vp_reset(struct virtio_device *vdev)
> +static int vp_reset(struct virtio_device *vdev)
>  {
>  	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>  	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>  		msleep(1);
>  	/* Flush pending VQ/configuration callbacks. */
>  	vp_synchronize_vectors(vdev);
> +	return 0;
>  }
>  
>  static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> index e28acf482e0c..5fd4e627a9b0 100644
> --- a/drivers/virtio/virtio_vdpa.c
> +++ b/drivers/virtio/virtio_vdpa.c
> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
>  	return ops->set_status(vdpa, status);
>  }
>  
> -static void virtio_vdpa_reset(struct virtio_device *vdev)
> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>  {
>  	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>  
>  	vdpa_reset(vdpa);
> +
> +	return 0;
>  }
>  
>  static bool virtio_vdpa_notify(struct virtqueue *vq)
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 8519b3ae5d52..d2b0f1699a75 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>   *	status: the new status byte
>   * @reset: reset the device
>   *	vdev: the virtio device
> - *	After this, status and feature negotiation must be done again
> + *	Upon success, status and feature negotiation must be done again
>   *	Device must not be reset from its vq/config callbacks, or in
>   *	parallel with being added/removed.
> + *	Returns 0 on success or error status.
>   * @find_vqs: find virtqueues and instantiate them.
>   *	vdev: the virtio_device
>   *	nvqs: the number of virtqueues to find
> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>  	u32 (*generation)(struct virtio_device *vdev);
>  	u8 (*get_status)(struct virtio_device *vdev);
>  	void (*set_status)(struct virtio_device *vdev, u8 status);
> -	void (*reset)(struct virtio_device *vdev);
> +	int (*reset)(struct virtio_device *vdev);
>  	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>  			struct virtqueue *vqs[], vq_callback_t *callbacks[],
>  			const char * const names[], const bool *ctx,
> -- 
> 2.25.4

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午10:24, Max Gurtovoy 写道:
>
> On 4/8/2021 4:14 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>>>
>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>
>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>> The reset device operation, usually is an operation that might 
>>>>> fail from
>>>>> various reasons. For example, the controller might be in a bad 
>>>>> state and
>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>> devices always succeed in reset operation but this is not the case 
>>>>> for
>>>>> HW based virtio devices.
>>>>
>>>>
>>>> I would like to know under what condition that the reset operation 
>>>> may fail (except for the case of a bugg guest).
>>>
>>> The controller might not be ready or stuck. This is a real use case 
>>> for many PCI devices.
>>>
>>> For real devices the FW might be in a bad state and it can happen 
>>> also for paravirt device if you have a bug in the controller code or 
>>> if you entered some error flow (Out of memory).
>>>
>>> You don't want to be stuck because of one bad device.
>>
>>
>> So the buggy driver can damage the host through various ways, I'm not 
>> sure doing such workaround is worthwhile.
>
> do you mean device ?


Yes.


>
> sometimes you need to replace device FW and it will work.
>
> I don't think it's a workaround. Other protocols, such as NVMe, solved 
> this in the specification.
>
> PCI config space and PCI controller are sometimes 2 different 
> components and sometimes the controller is not active, although the 
> device is plugged and seen by the PCI subsystem.


So I think we need patch to spec to see if it works first.


>
>
>>
>> Note that this driver has been used for real hardware PCI devices for 
>> many years. We don't receive any report of this before.
>>
>>
>>>
>>>
>>>>
>>>>
>>>>>
>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>> resetting virtio devices.
>>>>>
>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>> ---
>>>>>
>>>>> changes from v1:
>>>>>   - update virtio_ccw.c (Cornelia)
>>>>>   - update virtio_uml.c
>>>>>   - update mlxbf-tmfifo.c
>>>>
>>>>
>>>> Note that virtio driver may call reset, so you probably need to 
>>>> convert them.
>>>
>>> I'm sure I understand.
>>>
>>> Convert to what ?
>>
>>
>> Convert to deal with the possible rest failure. E.g in 
>> virtblk_freeze() we had:
>>
>> static int virtblk_freeze(struct virtio_device *vdev)
>> {
>>         struct virtio_blk *vblk = vdev->priv;
>>
>>         /* Ensure we don't receive any more interrupts */
>>         vdev->config->reset(vdev);
>> ...
>>
>> We need fail the freeze here.
>
>
> Agree.
>
>
>>
>> Another example is the driver remove which is not expected to be 
>> fail. A lot of virtio drivers tries to call reset there. I'm not sure 
>> how hard to deal with the failure in the path of remove (e.g 
>> __device_release_driver tends to ignore the return value of 
>> bus->remove().)
>
> I think it can stay as-is and ignore the ->reset return value and 
> continue free the other resources to avoid leakage.


The problem is that it's unclear that what kind of behaviour would the 
device do, e.g can it still send interrupts?

That's why we need to formalize the bahviour first if necessary.

Thanks


>
>
>>
>> Thanks
>>
>>
>>>
>>> Thanks.
>>>
>>>>
>>>> Thanks
>>>>
>>>>
>>>>>
>>>>> ---
>>>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>   drivers/virtio/virtio.c                  | 22 
>>>>> +++++++++++++++-------
>>>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>   include/linux/virtio_config.h            |  5 +++--
>>>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>
>>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>>> b/arch/um/drivers/virtio_uml.c
>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       vu_dev->status = status;
>>>>>   }
>>>>>   -static void vu_reset(struct virtio_device *vdev)
>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>         vu_dev->status = 0;
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static void vu_del_vq(struct virtqueue *vq)
>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> @@ -980,11 +980,13 @@ static void 
>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>   }
>>>>>     /* Reset the device. Not much here for now. */
>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>>         tm_vdev->status = 0;
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     /* Read the value of a configuration field. */
>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>   }
>>>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>       struct fw_rsc_vdev *rsc;
>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>>>> virtio_device *vdev)
>>>>>         rsc->status = 0;
>>>>>       dev_dbg(&vdev->dev, "reset !\n");
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     /* provide the vdev features as retrieved from the firmware */
>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>> index 54e686dca6de..52b32555e746 100644
>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>       return ret;
>>>>>   }
>>>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>       struct ccw1 *ccw;
>>>>> +    int ret;
>>>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>       if (!ccw)
>>>>> -        return;
>>>>> +        return -ENOMEM;
>>>>>         /* Zero status bits. */
>>>>>       vcdev->dma_area->status = 0;
>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>>>> virtio_device *vdev)
>>>>>       ccw->flags = 0;
>>>>>       ccw->count = 0;
>>>>>       ccw->cda = 0;
>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>> +
>>>>> +    return ret;
>>>>>   }
>>>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>> --- a/drivers/virtio/virtio.c
>>>>> +++ b/drivers/virtio/virtio.c
>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>       /* Assign a unique device index and hence name. */
>>>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>       if (err < 0)
>>>>> -        goto out;
>>>>> +        goto out_err;
>>>>>         dev->index = err;
>>>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>         /* We always start by resetting the device, in case a 
>>>>> previous
>>>>>        * driver messed it up.  This also tests that code path a 
>>>>> little. */
>>>>> -    dev->config->reset(dev);
>>>>> +    err = dev->config->reset(dev);
>>>>> +    if (err)
>>>>> +        goto out_ida;
>>>>>         /* Acknowledge that we've seen the device. */
>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>        */
>>>>>       err = device_add(&dev->dev);
>>>>>       if (err)
>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>> -out:
>>>>> -    if (err)
>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>> +        goto out_ida;
>>>>> +
>>>>> +    return 0;
>>>>> +
>>>>> +out_ida:
>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>> +out_err:
>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>       return err;
>>>>>   }
>>>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device 
>>>>> *dev)
>>>>>         /* We always start by resetting the device, in case a 
>>>>> previous
>>>>>        * driver messed it up. */
>>>>> -    dev->config->reset(dev);
>>>>> +    ret = dev->config->reset(dev);
>>>>> +    if (ret)
>>>>> +        goto err;
>>>>>         /* Acknowledge that we've seen the device. */
>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>>> b/drivers/virtio/virtio_mmio.c
>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>   }
>>>>>   -static void vm_reset(struct virtio_device *vdev)
>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_mmio_device *vm_dev = 
>>>>> to_virtio_mmio_device(vdev);
>>>>>         /* 0 status means a reset. */
>>>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>> +    return 0;
>>>>>   }
>>>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>>>> *vdev, u8 status)
>>>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>   }
>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       /* 0 status means a reset. */
>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>> u16 vector)
>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>>>>> *vdev, u8 status)
>>>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>>>   }
>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>           msleep(1);
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>> +    return 0;
>>>>>   }
>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>> u16 vector)
>>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       return ops->set_status(vdpa, status);
>>>>>   }
>>>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>         vdpa_reset(vdpa);
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>> diff --git a/include/linux/virtio_config.h 
>>>>> b/include/linux/virtio_config.h
>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>> --- a/include/linux/virtio_config.h
>>>>> +++ b/include/linux/virtio_config.h
>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>    *    status: the new status byte
>>>>>    * @reset: reset the device
>>>>>    *    vdev: the virtio device
>>>>> - *    After this, status and feature negotiation must be done again
>>>>> + *    Upon success, status and feature negotiation must be done 
>>>>> again
>>>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>>>    *    parallel with being added/removed.
>>>>> + *    Returns 0 on success or error status.
>>>>>    * @find_vqs: find virtqueues and instantiate them.
>>>>>    *    vdev: the virtio_device
>>>>>    *    nvqs: the number of virtqueues to find
>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>       u32 (*generation)(struct virtio_device *vdev);
>>>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>               const char * const names[], const bool *ctx,
>>>>
>>>
>>
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/8 下午10:24, Max Gurtovoy 写道:
>
> On 4/8/2021 4:14 PM, Jason Wang wrote:
>>
>> 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>>>
>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>
>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>> The reset device operation, usually is an operation that might 
>>>>> fail from
>>>>> various reasons. For example, the controller might be in a bad 
>>>>> state and
>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>> devices always succeed in reset operation but this is not the case 
>>>>> for
>>>>> HW based virtio devices.
>>>>
>>>>
>>>> I would like to know under what condition that the reset operation 
>>>> may fail (except for the case of a bugg guest).
>>>
>>> The controller might not be ready or stuck. This is a real use case 
>>> for many PCI devices.
>>>
>>> For real devices the FW might be in a bad state and it can happen 
>>> also for paravirt device if you have a bug in the controller code or 
>>> if you entered some error flow (Out of memory).
>>>
>>> You don't want to be stuck because of one bad device.
>>
>>
>> So the buggy driver can damage the host through various ways, I'm not 
>> sure doing such workaround is worthwhile.
>
> do you mean device ?


Yes.


>
> sometimes you need to replace device FW and it will work.
>
> I don't think it's a workaround. Other protocols, such as NVMe, solved 
> this in the specification.
>
> PCI config space and PCI controller are sometimes 2 different 
> components and sometimes the controller is not active, although the 
> device is plugged and seen by the PCI subsystem.


So I think we need patch to spec to see if it works first.


>
>
>>
>> Note that this driver has been used for real hardware PCI devices for 
>> many years. We don't receive any report of this before.
>>
>>
>>>
>>>
>>>>
>>>>
>>>>>
>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>> resetting virtio devices.
>>>>>
>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>> ---
>>>>>
>>>>> changes from v1:
>>>>>   - update virtio_ccw.c (Cornelia)
>>>>>   - update virtio_uml.c
>>>>>   - update mlxbf-tmfifo.c
>>>>
>>>>
>>>> Note that virtio driver may call reset, so you probably need to 
>>>> convert them.
>>>
>>> I'm sure I understand.
>>>
>>> Convert to what ?
>>
>>
>> Convert to deal with the possible rest failure. E.g in 
>> virtblk_freeze() we had:
>>
>> static int virtblk_freeze(struct virtio_device *vdev)
>> {
>>         struct virtio_blk *vblk = vdev->priv;
>>
>>         /* Ensure we don't receive any more interrupts */
>>         vdev->config->reset(vdev);
>> ...
>>
>> We need fail the freeze here.
>
>
> Agree.
>
>
>>
>> Another example is the driver remove which is not expected to be 
>> fail. A lot of virtio drivers tries to call reset there. I'm not sure 
>> how hard to deal with the failure in the path of remove (e.g 
>> __device_release_driver tends to ignore the return value of 
>> bus->remove().)
>
> I think it can stay as-is and ignore the ->reset return value and 
> continue free the other resources to avoid leakage.


The problem is that it's unclear that what kind of behaviour would the 
device do, e.g can it still send interrupts?

That's why we need to formalize the bahviour first if necessary.

Thanks


>
>
>>
>> Thanks
>>
>>
>>>
>>> Thanks.
>>>
>>>>
>>>> Thanks
>>>>
>>>>
>>>>>
>>>>> ---
>>>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>   drivers/virtio/virtio.c                  | 22 
>>>>> +++++++++++++++-------
>>>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>   include/linux/virtio_config.h            |  5 +++--
>>>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>
>>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>>> b/arch/um/drivers/virtio_uml.c
>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       vu_dev->status = status;
>>>>>   }
>>>>>   -static void vu_reset(struct virtio_device *vdev)
>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>         vu_dev->status = 0;
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static void vu_del_vq(struct virtqueue *vq)
>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>> @@ -980,11 +980,13 @@ static void 
>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>   }
>>>>>     /* Reset the device. Not much here for now. */
>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>>         tm_vdev->status = 0;
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     /* Read the value of a configuration field. */
>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>   }
>>>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>       struct fw_rsc_vdev *rsc;
>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>>>> virtio_device *vdev)
>>>>>         rsc->status = 0;
>>>>>       dev_dbg(&vdev->dev, "reset !\n");
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     /* provide the vdev features as retrieved from the firmware */
>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>> index 54e686dca6de..52b32555e746 100644
>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>       return ret;
>>>>>   }
>>>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>       struct ccw1 *ccw;
>>>>> +    int ret;
>>>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>       if (!ccw)
>>>>> -        return;
>>>>> +        return -ENOMEM;
>>>>>         /* Zero status bits. */
>>>>>       vcdev->dma_area->status = 0;
>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>>>> virtio_device *vdev)
>>>>>       ccw->flags = 0;
>>>>>       ccw->count = 0;
>>>>>       ccw->cda = 0;
>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>> +
>>>>> +    return ret;
>>>>>   }
>>>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>> --- a/drivers/virtio/virtio.c
>>>>> +++ b/drivers/virtio/virtio.c
>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>       /* Assign a unique device index and hence name. */
>>>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>       if (err < 0)
>>>>> -        goto out;
>>>>> +        goto out_err;
>>>>>         dev->index = err;
>>>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>         /* We always start by resetting the device, in case a 
>>>>> previous
>>>>>        * driver messed it up.  This also tests that code path a 
>>>>> little. */
>>>>> -    dev->config->reset(dev);
>>>>> +    err = dev->config->reset(dev);
>>>>> +    if (err)
>>>>> +        goto out_ida;
>>>>>         /* Acknowledge that we've seen the device. */
>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>>>> virtio_device *dev)
>>>>>        */
>>>>>       err = device_add(&dev->dev);
>>>>>       if (err)
>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>> -out:
>>>>> -    if (err)
>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>> +        goto out_ida;
>>>>> +
>>>>> +    return 0;
>>>>> +
>>>>> +out_ida:
>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>> +out_err:
>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>       return err;
>>>>>   }
>>>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device 
>>>>> *dev)
>>>>>         /* We always start by resetting the device, in case a 
>>>>> previous
>>>>>        * driver messed it up. */
>>>>> -    dev->config->reset(dev);
>>>>> +    ret = dev->config->reset(dev);
>>>>> +    if (ret)
>>>>> +        goto err;
>>>>>         /* Acknowledge that we've seen the device. */
>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>>> b/drivers/virtio/virtio_mmio.c
>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>   }
>>>>>   -static void vm_reset(struct virtio_device *vdev)
>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_mmio_device *vm_dev = 
>>>>> to_virtio_mmio_device(vdev);
>>>>>         /* 0 status means a reset. */
>>>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>> +    return 0;
>>>>>   }
>>>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>>>> *vdev, u8 status)
>>>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>   }
>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       /* 0 status means a reset. */
>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>> u16 vector)
>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device 
>>>>> *vdev, u8 status)
>>>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>>>   }
>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>           msleep(1);
>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>       vp_synchronize_vectors(vdev);
>>>>> +    return 0;
>>>>>   }
>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>> u16 vector)
>>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>>>> virtio_device *vdev, u8 status)
>>>>>       return ops->set_status(vdpa, status);
>>>>>   }
>>>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>   {
>>>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>         vdpa_reset(vdpa);
>>>>> +
>>>>> +    return 0;
>>>>>   }
>>>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>> diff --git a/include/linux/virtio_config.h 
>>>>> b/include/linux/virtio_config.h
>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>> --- a/include/linux/virtio_config.h
>>>>> +++ b/include/linux/virtio_config.h
>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>    *    status: the new status byte
>>>>>    * @reset: reset the device
>>>>>    *    vdev: the virtio device
>>>>> - *    After this, status and feature negotiation must be done again
>>>>> + *    Upon success, status and feature negotiation must be done 
>>>>> again
>>>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>>>    *    parallel with being added/removed.
>>>>> + *    Returns 0 on success or error status.
>>>>>    * @find_vqs: find virtqueues and instantiate them.
>>>>>    *    vdev: the virtio_device
>>>>>    *    nvqs: the number of virtqueues to find
>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>       u32 (*generation)(struct virtio_device *vdev);
>>>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>               const char * const names[], const bool *ctx,
>>>>
>>>
>>
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/9/2021 8:22 AM, Jason Wang wrote:
>
> 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
>>
>> On 4/8/2021 4:14 PM, Jason Wang wrote:
>>>
>>> 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>>>>
>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>
>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>> The reset device operation, usually is an operation that might 
>>>>>> fail from
>>>>>> various reasons. For example, the controller might be in a bad 
>>>>>> state and
>>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>>> devices always succeed in reset operation but this is not the 
>>>>>> case for
>>>>>> HW based virtio devices.
>>>>>
>>>>>
>>>>> I would like to know under what condition that the reset operation 
>>>>> may fail (except for the case of a bugg guest).
>>>>
>>>> The controller might not be ready or stuck. This is a real use case 
>>>> for many PCI devices.
>>>>
>>>> For real devices the FW might be in a bad state and it can happen 
>>>> also for paravirt device if you have a bug in the controller code 
>>>> or if you entered some error flow (Out of memory).
>>>>
>>>> You don't want to be stuck because of one bad device.
>>>
>>>
>>> So the buggy driver can damage the host through various ways, I'm 
>>> not sure doing such workaround is worthwhile.
>>
>> do you mean device ?
>
>
> Yes.
>
>
>>
>> sometimes you need to replace device FW and it will work.
>>
>> I don't think it's a workaround. Other protocols, such as NVMe, 
>> solved this in the specification.
>>
>> PCI config space and PCI controller are sometimes 2 different 
>> components and sometimes the controller is not active, although the 
>> device is plugged and seen by the PCI subsystem.
>
>
> So I think we need patch to spec to see if it works first.

We can't leave the driver to loop forever without allowing next "good" 
virtio devices to probe.

We can in parallel look for spec fixes but for now we must fix the driver.

The fix can be using the mechanism introduced in this series or adding 
an async probing mechanism.

In both solutions, we can't allow looping forever.

>
>
>>
>>
>>>
>>> Note that this driver has been used for real hardware PCI devices 
>>> for many years. We don't receive any report of this before.
>>>
>>>
>>>>
>>>>
>>>>>
>>>>>
>>>>>>
>>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>>> resetting virtio devices.
>>>>>>
>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>> ---
>>>>>>
>>>>>> changes from v1:
>>>>>>   - update virtio_ccw.c (Cornelia)
>>>>>>   - update virtio_uml.c
>>>>>>   - update mlxbf-tmfifo.c
>>>>>
>>>>>
>>>>> Note that virtio driver may call reset, so you probably need to 
>>>>> convert them.
>>>>
>>>> I'm sure I understand.
>>>>
>>>> Convert to what ?
>>>
>>>
>>> Convert to deal with the possible rest failure. E.g in 
>>> virtblk_freeze() we had:
>>>
>>> static int virtblk_freeze(struct virtio_device *vdev)
>>> {
>>>         struct virtio_blk *vblk = vdev->priv;
>>>
>>>         /* Ensure we don't receive any more interrupts */
>>>         vdev->config->reset(vdev);
>>> ...
>>>
>>> We need fail the freeze here.
>>
>>
>> Agree.
>>
>>
>>>
>>> Another example is the driver remove which is not expected to be 
>>> fail. A lot of virtio drivers tries to call reset there. I'm not 
>>> sure how hard to deal with the failure in the path of remove (e.g 
>>> __device_release_driver tends to ignore the return value of 
>>> bus->remove().)
>>
>> I think it can stay as-is and ignore the ->reset return value and 
>> continue free the other resources to avoid leakage.
>
>
> The problem is that it's unclear that what kind of behaviour would the 
> device do, e.g can it still send interrupts?
>
> That's why we need to formalize the bahviour first if necessary.
>
> Thanks
>
>
>>
>>
>>>
>>> Thanks
>>>
>>>
>>>>
>>>> Thanks.
>>>>
>>>>>
>>>>> Thanks
>>>>>
>>>>>
>>>>>>
>>>>>> ---
>>>>>>   arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>   drivers/virtio/virtio.c                  | 22 
>>>>>> +++++++++++++++-------
>>>>>>   drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>   drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>   drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>   include/linux/virtio_config.h            |  5 +++--
>>>>>>   10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>
>>>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>>>> b/arch/um/drivers/virtio_uml.c
>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>>>> virtio_device *vdev, u8 status)
>>>>>>       vu_dev->status = status;
>>>>>>   }
>>>>>>   -static void vu_reset(struct virtio_device *vdev)
>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>>         vu_dev->status = 0;
>>>>>> +
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     static void vu_del_vq(struct virtqueue *vq)
>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c 
>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> @@ -980,11 +980,13 @@ static void 
>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>   }
>>>>>>     /* Reset the device. Not much here for now. */
>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct mlxbf_tmfifo_vdev *tm_vdev = 
>>>>>> mlxbf_vdev_to_tmfifo(vdev);
>>>>>>         tm_vdev->status = 0;
>>>>>> +
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     /* Read the value of a configuration field. */
>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c 
>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct 
>>>>>> virtio_device *vdev, u8 status)
>>>>>>       dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>   }
>>>>>>   -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>       struct fw_rsc_vdev *rsc;
>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct 
>>>>>> virtio_device *vdev)
>>>>>>         rsc->status = 0;
>>>>>>       dev_dbg(&vdev->dev, "reset !\n");
>>>>>> +
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     /* provide the vdev features as retrieved from the firmware */
>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c 
>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct 
>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>       return ret;
>>>>>>   }
>>>>>>   -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>       struct ccw1 *ccw;
>>>>>> +    int ret;
>>>>>>         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>>       if (!ccw)
>>>>>> -        return;
>>>>>> +        return -ENOMEM;
>>>>>>         /* Zero status bits. */
>>>>>>       vcdev->dma_area->status = 0;
>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct 
>>>>>> virtio_device *vdev)
>>>>>>       ccw->flags = 0;
>>>>>>       ccw->count = 0;
>>>>>>       ccw->cda = 0;
>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>> +
>>>>>> +    return ret;
>>>>>>   }
>>>>>>     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>> --- a/drivers/virtio/virtio.c
>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct 
>>>>>> virtio_device *dev)
>>>>>>       /* Assign a unique device index and hence name. */
>>>>>>       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>>       if (err < 0)
>>>>>> -        goto out;
>>>>>> +        goto out_err;
>>>>>>         dev->index = err;
>>>>>>       dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct 
>>>>>> virtio_device *dev)
>>>>>>         /* We always start by resetting the device, in case a 
>>>>>> previous
>>>>>>        * driver messed it up.  This also tests that code path a 
>>>>>> little. */
>>>>>> -    dev->config->reset(dev);
>>>>>> +    err = dev->config->reset(dev);
>>>>>> +    if (err)
>>>>>> +        goto out_ida;
>>>>>>         /* Acknowledge that we've seen the device. */
>>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct 
>>>>>> virtio_device *dev)
>>>>>>        */
>>>>>>       err = device_add(&dev->dev);
>>>>>>       if (err)
>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>> -out:
>>>>>> -    if (err)
>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>> +        goto out_ida;
>>>>>> +
>>>>>> +    return 0;
>>>>>> +
>>>>>> +out_ida:
>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>> +out_err:
>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>       return err;
>>>>>>   }
>>>>>>   EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct 
>>>>>> virtio_device *dev)
>>>>>>         /* We always start by resetting the device, in case a 
>>>>>> previous
>>>>>>        * driver messed it up. */
>>>>>> -    dev->config->reset(dev);
>>>>>> +    ret = dev->config->reset(dev);
>>>>>> +    if (ret)
>>>>>> +        goto err;
>>>>>>         /* Acknowledge that we've seen the device. */
>>>>>>       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>>>> b/drivers/virtio/virtio_mmio.c
>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>>>> virtio_device *vdev, u8 status)
>>>>>>       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>   }
>>>>>>   -static void vm_reset(struct virtio_device *vdev)
>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct virtio_mmio_device *vm_dev = 
>>>>>> to_virtio_mmio_device(vdev);
>>>>>>         /* 0 status means a reset. */
>>>>>>       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     diff --git a/drivers/virtio/virtio_pci_legacy.c 
>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device 
>>>>>> *vdev, u8 status)
>>>>>>       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>   }
>>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>       /* 0 status means a reset. */
>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>>       vp_synchronize_vectors(vdev);
>>>>>> +
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>>> u16 vector)
>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c 
>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct 
>>>>>> virtio_device *vdev, u8 status)
>>>>>>       vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>   }
>>>>>>   -static void vp_reset(struct virtio_device *vdev)
>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>           msleep(1);
>>>>>>       /* Flush pending VQ/configuration callbacks. */
>>>>>>       vp_synchronize_vectors(vdev);
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     static u16 vp_config_vector(struct virtio_pci_device *vp_dev, 
>>>>>> u16 vector)
>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct 
>>>>>> virtio_device *vdev, u8 status)
>>>>>>       return ops->set_status(vdpa, status);
>>>>>>   }
>>>>>>   -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>   {
>>>>>>       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>         vdpa_reset(vdpa);
>>>>>> +
>>>>>> +    return 0;
>>>>>>   }
>>>>>>     static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>> diff --git a/include/linux/virtio_config.h 
>>>>>> b/include/linux/virtio_config.h
>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>> --- a/include/linux/virtio_config.h
>>>>>> +++ b/include/linux/virtio_config.h
>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>    *    status: the new status byte
>>>>>>    * @reset: reset the device
>>>>>>    *    vdev: the virtio device
>>>>>> - *    After this, status and feature negotiation must be done again
>>>>>> + *    Upon success, status and feature negotiation must be done 
>>>>>> again
>>>>>>    *    Device must not be reset from its vq/config callbacks, or in
>>>>>>    *    parallel with being added/removed.
>>>>>> + *    Returns 0 on success or error status.
>>>>>>    * @find_vqs: find virtqueues and instantiate them.
>>>>>>    *    vdev: the virtio_device
>>>>>>    *    nvqs: the number of virtqueues to find
>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>       u32 (*generation)(struct virtio_device *vdev);
>>>>>>       u8 (*get_status)(struct virtio_device *vdev);
>>>>>>       void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>               struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>>               const char * const names[], const bool *ctx,
>>>>>
>>>>
>>>
>>
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Sun, Apr 11, 2021 at 12:50:22PM +0300, Max Gurtovoy wrote:
> 
> On 4/9/2021 8:22 AM, Jason Wang wrote:
> > 
> > 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
> > > 
> > > On 4/8/2021 4:14 PM, Jason Wang wrote:
> > > > 
> > > > 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
> > > > > 
> > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > 
> > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > The reset device operation, usually is an operation
> > > > > > > that might fail from
> > > > > > > various reasons. For example, the controller might
> > > > > > > be in a bad state and
> > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > devices always succeed in reset operation but this
> > > > > > > is not the case for
> > > > > > > HW based virtio devices.
> > > > > > 
> > > > > > 
> > > > > > I would like to know under what condition that the reset
> > > > > > operation may fail (except for the case of a bugg
> > > > > > guest).
> > > > > 
> > > > > The controller might not be ready or stuck. This is a real
> > > > > use case for many PCI devices.
> > > > > 
> > > > > For real devices the FW might be in a bad state and it can
> > > > > happen also for paravirt device if you have a bug in the
> > > > > controller code or if you entered some error flow (Out of
> > > > > memory).
> > > > > 
> > > > > You don't want to be stuck because of one bad device.
> > > > 
> > > > 
> > > > So the buggy driver can damage the host through various ways,
> > > > I'm not sure doing such workaround is worthwhile.
> > > 
> > > do you mean device ?
> > 
> > 
> > Yes.
> > 
> > 
> > > 
> > > sometimes you need to replace device FW and it will work.
> > > 
> > > I don't think it's a workaround. Other protocols, such as NVMe,
> > > solved this in the specification.
> > > 
> > > PCI config space and PCI controller are sometimes 2 different
> > > components and sometimes the controller is not active, although the
> > > device is plugged and seen by the PCI subsystem.
> > 
> > 
> > So I think we need patch to spec to see if it works first.
> 
> We can't leave the driver to loop forever without allowing next "good"
> virtio devices to probe.
> 
> We can in parallel look for spec fixes but for now we must fix the driver.

I'd like to narrow the case though. the proper thing
would probably be for device to clear the status.
Provided it entered some state where it can not do it -
is there anything special about
the device state that *can* be detected?
If yes what is it?


> The fix can be using the mechanism introduced in this series or adding an
> async probing mechanism.

What would that be?

> In both solutions, we can't allow looping forever.

Multiple minute downtime isn't much better either. I'd like a
much shorter timer or even no timer at all, instead
verifying that device is alive every X iterations.

> > 
> > 
> > > 
> > > 
> > > > 
> > > > Note that this driver has been used for real hardware PCI
> > > > devices for many years. We don't receive any report of this
> > > > before.
> > > > 
> > > > 
> > > > > 
> > > > > 
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > resetting virtio devices.
> > > > > > > 
> > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > ---
> > > > > > > 
> > > > > > > changes from v1:
> > > > > > >   - update virtio_ccw.c (Cornelia)
> > > > > > >   - update virtio_uml.c
> > > > > > >   - update mlxbf-tmfifo.c
> > > > > > 
> > > > > > 
> > > > > > Note that virtio driver may call reset, so you probably
> > > > > > need to convert them.
> > > > > 
> > > > > I'm sure I understand.
> > > > > 
> > > > > Convert to what ?
> > > > 
> > > > 
> > > > Convert to deal with the possible rest failure. E.g in
> > > > virtblk_freeze() we had:
> > > > 
> > > > static int virtblk_freeze(struct virtio_device *vdev)
> > > > {
> > > >         struct virtio_blk *vblk = vdev->priv;
> > > > 
> > > >         /* Ensure we don't receive any more interrupts */
> > > >         vdev->config->reset(vdev);
> > > > ...
> > > > 
> > > > We need fail the freeze here.
> > > 
> > > 
> > > Agree.
> > > 
> > > 
> > > > 
> > > > Another example is the driver remove which is not expected to be
> > > > fail. A lot of virtio drivers tries to call reset there. I'm not
> > > > sure how hard to deal with the failure in the path of remove
> > > > (e.g __device_release_driver tends to ignore the return value of
> > > > bus->remove().)
> > > 
> > > I think it can stay as-is and ignore the ->reset return value and
> > > continue free the other resources to avoid leakage.
> > 
> > 
> > The problem is that it's unclear that what kind of behaviour would the
> > device do, e.g can it still send interrupts?
> > 
> > That's why we need to formalize the bahviour first if necessary.
> > 
> > Thanks
> > 
> > 
> > > 
> > > 
> > > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > 
> > > > > Thanks.
> > > > > 
> > > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > ---
> > > > > > >   arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > >   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > >   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > >   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > >   drivers/virtio/virtio.c                  | 22
> > > > > > > +++++++++++++++-------
> > > > > > >   drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > >   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > >   drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > >   drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > >   include/linux/virtio_config.h            |  5 +++--
> > > > > > >   10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > 
> > > > > > > diff --git a/arch/um/drivers/virtio_uml.c
> > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > vu_set_status(struct virtio_device *vdev, u8 status)
> > > > > > >       vu_dev->status = status;
> > > > > > >   }
> > > > > > >   -static void vu_reset(struct virtio_device *vdev)
> > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > >         vu_dev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static void vu_del_vq(struct virtqueue *vq)
> > > > > > > diff --git
> > > > > > > a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device
> > > > > > > *vdev,
> > > > > > >   }
> > > > > > >     /* Reset the device. Not much here for now. */
> > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct mlxbf_tmfifo_vdev *tm_vdev =
> > > > > > > mlxbf_vdev_to_tmfifo(vdev);
> > > > > > >         tm_vdev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     /* Read the value of a configuration field. */
> > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > @@ -191,7 +191,7 @@ static void
> > > > > > > rproc_virtio_set_status(struct virtio_device *vdev,
> > > > > > > u8 status)
> > > > > > >       dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > >   }
> > > > > > >   -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > >       struct fw_rsc_vdev *rsc;
> > > > > > > @@ -200,6 +200,8 @@ static void
> > > > > > > rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >         rsc->status = 0;
> > > > > > >       dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     /* provide the vdev features as retrieved from the firmware */
> > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > @@ -732,14 +732,15 @@ static int
> > > > > > > virtio_ccw_find_vqs(struct virtio_device *vdev,
> > > > > > > unsigned nvqs,
> > > > > > >       return ret;
> > > > > > >   }
> > > > > > >   -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > >       struct ccw1 *ccw;
> > > > > > > +    int ret;
> > > > > > >         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > >       if (!ccw)
> > > > > > > -        return;
> > > > > > > +        return -ENOMEM;
> > > > > > >         /* Zero status bits. */
> > > > > > >       vcdev->dma_area->status = 0;
> > > > > > > @@ -749,8 +750,10 @@ static void
> > > > > > > virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >       ccw->flags = 0;
> > > > > > >       ccw->count = 0;
> > > > > > >       ccw->cda = 0;
> > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > >       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > +
> > > > > > > +    return ret;
> > > > > > >   }
> > > > > > >     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >       /* Assign a unique device index and hence name. */
> > > > > > >       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > >       if (err < 0)
> > > > > > > -        goto out;
> > > > > > > +        goto out_err;
> > > > > > >         dev->index = err;
> > > > > > >       dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >         /* We always start by resetting the device,
> > > > > > > in case a previous
> > > > > > >        * driver messed it up.  This also tests that
> > > > > > > code path a little. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    err = dev->config->reset(dev);
> > > > > > > +    if (err)
> > > > > > > +        goto out_ida;
> > > > > > >         /* Acknowledge that we've seen the device. */
> > > > > > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > @@ -362,10 +364,14 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >        */
> > > > > > >       err = device_add(&dev->dev);
> > > > > > >       if (err)
> > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > -out:
> > > > > > > -    if (err)
> > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > +        goto out_ida;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > > +
> > > > > > > +out_ida:
> > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > +out_err:
> > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > >       return err;
> > > > > > >   }
> > > > > > >   EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct
> > > > > > > virtio_device *dev)
> > > > > > >         /* We always start by resetting the device,
> > > > > > > in case a previous
> > > > > > >        * driver messed it up. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > +    if (ret)
> > > > > > > +        goto err;
> > > > > > >         /* Acknowledge that we've seen the device. */
> > > > > > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > diff --git a/drivers/virtio/virtio_mmio.c
> > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > vm_set_status(struct virtio_device *vdev, u8 status)
> > > > > > >       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > >   }
> > > > > > >   -static void vm_reset(struct virtio_device *vdev)
> > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_mmio_device *vm_dev =
> > > > > > > to_virtio_mmio_device(vdev);
> > > > > > >         /* 0 status means a reset. */
> > > > > > >       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >   }
> > > > > > >   -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >       /* 0 status means a reset. */
> > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >       /* Flush pending VQ/configuration callbacks. */
> > > > > > >       vp_synchronize_vectors(vdev);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static u16 vp_config_vector(struct
> > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >       vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > >   }
> > > > > > >   -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >           msleep(1);
> > > > > > >       /* Flush pending VQ/configuration callbacks. */
> > > > > > >       vp_synchronize_vectors(vdev);
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static u16 vp_config_vector(struct
> > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c
> > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > @@ -97,11 +97,13 @@ static void
> > > > > > > virtio_vdpa_set_status(struct virtio_device *vdev,
> > > > > > > u8 status)
> > > > > > >       return ops->set_status(vdpa, status);
> > > > > > >   }
> > > > > > >   -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > >         vdpa_reset(vdpa);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > b/include/linux/virtio_config.h
> > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > >    *    status: the new status byte
> > > > > > >    * @reset: reset the device
> > > > > > >    *    vdev: the virtio device
> > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > + *    Upon success, status and feature negotiation
> > > > > > > must be done again
> > > > > > >    *    Device must not be reset from its vq/config callbacks, or in
> > > > > > >    *    parallel with being added/removed.
> > > > > > > + *    Returns 0 on success or error status.
> > > > > > >    * @find_vqs: find virtqueues and instantiate them.
> > > > > > >    *    vdev: the virtio_device
> > > > > > >    *    nvqs: the number of virtqueues to find
> > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > >       u32 (*generation)(struct virtio_device *vdev);
> > > > > > >       u8 (*get_status)(struct virtio_device *vdev);
> > > > > > >       void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > >       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > >               struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > >               const char * const names[], const bool *ctx,
> > > > > > 
> > > > > 
> > > > 
> > > 
> > 

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Sun, Apr 11, 2021 at 12:50:22PM +0300, Max Gurtovoy wrote:
> 
> On 4/9/2021 8:22 AM, Jason Wang wrote:
> > 
> > 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
> > > 
> > > On 4/8/2021 4:14 PM, Jason Wang wrote:
> > > > 
> > > > 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
> > > > > 
> > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > 
> > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > The reset device operation, usually is an operation
> > > > > > > that might fail from
> > > > > > > various reasons. For example, the controller might
> > > > > > > be in a bad state and
> > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > devices always succeed in reset operation but this
> > > > > > > is not the case for
> > > > > > > HW based virtio devices.
> > > > > > 
> > > > > > 
> > > > > > I would like to know under what condition that the reset
> > > > > > operation may fail (except for the case of a bugg
> > > > > > guest).
> > > > > 
> > > > > The controller might not be ready or stuck. This is a real
> > > > > use case for many PCI devices.
> > > > > 
> > > > > For real devices the FW might be in a bad state and it can
> > > > > happen also for paravirt device if you have a bug in the
> > > > > controller code or if you entered some error flow (Out of
> > > > > memory).
> > > > > 
> > > > > You don't want to be stuck because of one bad device.
> > > > 
> > > > 
> > > > So the buggy driver can damage the host through various ways,
> > > > I'm not sure doing such workaround is worthwhile.
> > > 
> > > do you mean device ?
> > 
> > 
> > Yes.
> > 
> > 
> > > 
> > > sometimes you need to replace device FW and it will work.
> > > 
> > > I don't think it's a workaround. Other protocols, such as NVMe,
> > > solved this in the specification.
> > > 
> > > PCI config space and PCI controller are sometimes 2 different
> > > components and sometimes the controller is not active, although the
> > > device is plugged and seen by the PCI subsystem.
> > 
> > 
> > So I think we need patch to spec to see if it works first.
> 
> We can't leave the driver to loop forever without allowing next "good"
> virtio devices to probe.
> 
> We can in parallel look for spec fixes but for now we must fix the driver.

I'd like to narrow the case though. the proper thing
would probably be for device to clear the status.
Provided it entered some state where it can not do it -
is there anything special about
the device state that *can* be detected?
If yes what is it?


> The fix can be using the mechanism introduced in this series or adding an
> async probing mechanism.

What would that be?

> In both solutions, we can't allow looping forever.

Multiple minute downtime isn't much better either. I'd like a
much shorter timer or even no timer at all, instead
verifying that device is alive every X iterations.

> > 
> > 
> > > 
> > > 
> > > > 
> > > > Note that this driver has been used for real hardware PCI
> > > > devices for many years. We don't receive any report of this
> > > > before.
> > > > 
> > > > 
> > > > > 
> > > > > 
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > resetting virtio devices.
> > > > > > > 
> > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > ---
> > > > > > > 
> > > > > > > changes from v1:
> > > > > > >   - update virtio_ccw.c (Cornelia)
> > > > > > >   - update virtio_uml.c
> > > > > > >   - update mlxbf-tmfifo.c
> > > > > > 
> > > > > > 
> > > > > > Note that virtio driver may call reset, so you probably
> > > > > > need to convert them.
> > > > > 
> > > > > I'm sure I understand.
> > > > > 
> > > > > Convert to what ?
> > > > 
> > > > 
> > > > Convert to deal with the possible rest failure. E.g in
> > > > virtblk_freeze() we had:
> > > > 
> > > > static int virtblk_freeze(struct virtio_device *vdev)
> > > > {
> > > >         struct virtio_blk *vblk = vdev->priv;
> > > > 
> > > >         /* Ensure we don't receive any more interrupts */
> > > >         vdev->config->reset(vdev);
> > > > ...
> > > > 
> > > > We need fail the freeze here.
> > > 
> > > 
> > > Agree.
> > > 
> > > 
> > > > 
> > > > Another example is the driver remove which is not expected to be
> > > > fail. A lot of virtio drivers tries to call reset there. I'm not
> > > > sure how hard to deal with the failure in the path of remove
> > > > (e.g __device_release_driver tends to ignore the return value of
> > > > bus->remove().)
> > > 
> > > I think it can stay as-is and ignore the ->reset return value and
> > > continue free the other resources to avoid leakage.
> > 
> > 
> > The problem is that it's unclear that what kind of behaviour would the
> > device do, e.g can it still send interrupts?
> > 
> > That's why we need to formalize the bahviour first if necessary.
> > 
> > Thanks
> > 
> > 
> > > 
> > > 
> > > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > 
> > > > > Thanks.
> > > > > 
> > > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > ---
> > > > > > >   arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > >   drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > >   drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > >   drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > >   drivers/virtio/virtio.c                  | 22
> > > > > > > +++++++++++++++-------
> > > > > > >   drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > >   drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > >   drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > >   drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > >   include/linux/virtio_config.h            |  5 +++--
> > > > > > >   10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > 
> > > > > > > diff --git a/arch/um/drivers/virtio_uml.c
> > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > vu_set_status(struct virtio_device *vdev, u8 status)
> > > > > > >       vu_dev->status = status;
> > > > > > >   }
> > > > > > >   -static void vu_reset(struct virtio_device *vdev)
> > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > >         vu_dev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static void vu_del_vq(struct virtqueue *vq)
> > > > > > > diff --git
> > > > > > > a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device
> > > > > > > *vdev,
> > > > > > >   }
> > > > > > >     /* Reset the device. Not much here for now. */
> > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct mlxbf_tmfifo_vdev *tm_vdev =
> > > > > > > mlxbf_vdev_to_tmfifo(vdev);
> > > > > > >         tm_vdev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     /* Read the value of a configuration field. */
> > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > @@ -191,7 +191,7 @@ static void
> > > > > > > rproc_virtio_set_status(struct virtio_device *vdev,
> > > > > > > u8 status)
> > > > > > >       dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > >   }
> > > > > > >   -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > >       struct fw_rsc_vdev *rsc;
> > > > > > > @@ -200,6 +200,8 @@ static void
> > > > > > > rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >         rsc->status = 0;
> > > > > > >       dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     /* provide the vdev features as retrieved from the firmware */
> > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > @@ -732,14 +732,15 @@ static int
> > > > > > > virtio_ccw_find_vqs(struct virtio_device *vdev,
> > > > > > > unsigned nvqs,
> > > > > > >       return ret;
> > > > > > >   }
> > > > > > >   -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > >       struct ccw1 *ccw;
> > > > > > > +    int ret;
> > > > > > >         ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > >       if (!ccw)
> > > > > > > -        return;
> > > > > > > +        return -ENOMEM;
> > > > > > >         /* Zero status bits. */
> > > > > > >       vcdev->dma_area->status = 0;
> > > > > > > @@ -749,8 +750,10 @@ static void
> > > > > > > virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >       ccw->flags = 0;
> > > > > > >       ccw->count = 0;
> > > > > > >       ccw->cda = 0;
> > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > >       ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > +
> > > > > > > +    return ret;
> > > > > > >   }
> > > > > > >     static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >       /* Assign a unique device index and hence name. */
> > > > > > >       err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > >       if (err < 0)
> > > > > > > -        goto out;
> > > > > > > +        goto out_err;
> > > > > > >         dev->index = err;
> > > > > > >       dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >         /* We always start by resetting the device,
> > > > > > > in case a previous
> > > > > > >        * driver messed it up.  This also tests that
> > > > > > > code path a little. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    err = dev->config->reset(dev);
> > > > > > > +    if (err)
> > > > > > > +        goto out_ida;
> > > > > > >         /* Acknowledge that we've seen the device. */
> > > > > > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > @@ -362,10 +364,14 @@ int
> > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > >        */
> > > > > > >       err = device_add(&dev->dev);
> > > > > > >       if (err)
> > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > -out:
> > > > > > > -    if (err)
> > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > +        goto out_ida;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > > +
> > > > > > > +out_ida:
> > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > +out_err:
> > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > >       return err;
> > > > > > >   }
> > > > > > >   EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct
> > > > > > > virtio_device *dev)
> > > > > > >         /* We always start by resetting the device,
> > > > > > > in case a previous
> > > > > > >        * driver messed it up. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > +    if (ret)
> > > > > > > +        goto err;
> > > > > > >         /* Acknowledge that we've seen the device. */
> > > > > > >       virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > diff --git a/drivers/virtio/virtio_mmio.c
> > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > vm_set_status(struct virtio_device *vdev, u8 status)
> > > > > > >       writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > >   }
> > > > > > >   -static void vm_reset(struct virtio_device *vdev)
> > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_mmio_device *vm_dev =
> > > > > > > to_virtio_mmio_device(vdev);
> > > > > > >         /* 0 status means a reset. */
> > > > > > >       writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >       iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >   }
> > > > > > >   -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >       /* 0 status means a reset. */
> > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >       ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >       /* Flush pending VQ/configuration callbacks. */
> > > > > > >       vp_synchronize_vectors(vdev);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static u16 vp_config_vector(struct
> > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >       vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > >   }
> > > > > > >   -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >       struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >           msleep(1);
> > > > > > >       /* Flush pending VQ/configuration callbacks. */
> > > > > > >       vp_synchronize_vectors(vdev);
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static u16 vp_config_vector(struct
> > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c
> > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > @@ -97,11 +97,13 @@ static void
> > > > > > > virtio_vdpa_set_status(struct virtio_device *vdev,
> > > > > > > u8 status)
> > > > > > >       return ops->set_status(vdpa, status);
> > > > > > >   }
> > > > > > >   -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > >   {
> > > > > > >       struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > >         vdpa_reset(vdpa);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >   }
> > > > > > >     static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > b/include/linux/virtio_config.h
> > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > >    *    status: the new status byte
> > > > > > >    * @reset: reset the device
> > > > > > >    *    vdev: the virtio device
> > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > + *    Upon success, status and feature negotiation
> > > > > > > must be done again
> > > > > > >    *    Device must not be reset from its vq/config callbacks, or in
> > > > > > >    *    parallel with being added/removed.
> > > > > > > + *    Returns 0 on success or error status.
> > > > > > >    * @find_vqs: find virtqueues and instantiate them.
> > > > > > >    *    vdev: the virtio_device
> > > > > > >    *    nvqs: the number of virtqueues to find
> > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > >       u32 (*generation)(struct virtio_device *vdev);
> > > > > > >       u8 (*get_status)(struct virtio_device *vdev);
> > > > > > >       void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > >       int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > >               struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > >               const char * const names[], const bool *ctx,
> > > > > > 
> > > > > 
> > > > 
> > > 
> > 

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>> The reset device operation, usually is an operation that might fail from
>>>> various reasons. For example, the controller might be in a bad state and
>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>> devices always succeed in reset operation but this is not the case for
>>>> HW based virtio devices.
>>>
>>> I would like to know under what condition that the reset operation may
>>> fail (except for the case of a bugg guest).
>> The controller might not be ready or stuck. This is a real use case for many
>> PCI devices.
>>
>> For real devices the FW might be in a bad state and it can happen also for
>> paravirt device if you have a bug in the controller code or if you entered
>> some error flow (Out of memory).
>>
>> You don't want to be stuck because of one bad device.
> OK so maybe we can do more to detect the bad device.
> Won't we get all 1's on a read in this case?

No. how can we guarantee it ?


>
>
>>>
>>>> This commit is also a preparation for adding a timeout mechanism for
>>>> resetting virtio devices.
>>>>
>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>> ---
>>>>
>>>> changes from v1:
>>>>    - update virtio_ccw.c (Cornelia)
>>>>    - update virtio_uml.c
>>>>    - update mlxbf-tmfifo.c
>>>
>>> Note that virtio driver may call reset, so you probably need to convert
>>> them.
>> I'm sure I understand.
>>
>> Convert to what ?
>>
>> Thanks.
>>
>>> Thanks
>>>
>>>
>>>> ---
>>>>    arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>    drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>>    drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>    drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>    drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>    include/linux/virtio_config.h            |  5 +++--
>>>>    10 files changed, 43 insertions(+), 19 deletions(-)
>>>>
>>>> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>> --- a/arch/um/drivers/virtio_uml.c
>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
>>>> *vdev, u8 status)
>>>>        vu_dev->status = status;
>>>>    }
>>>>    -static void vu_reset(struct virtio_device *vdev)
>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>          vu_dev->status = 0;
>>>> +
>>>> +    return 0;
>>>>    }
>>>>      static void vu_del_vq(struct virtqueue *vq)
>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>> @@ -980,11 +980,13 @@ static void
>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>    }
>>>>      /* Reset the device. Not much here for now. */
>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>          tm_vdev->status = 0;
>>>> +
>>>> +    return 0;
>>>>    }
>>>>      /* Read the value of a configuration field. */
>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>> virtio_device *vdev, u8 status)
>>>>        dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>    }
>>>>    -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>        struct fw_rsc_vdev *rsc;
>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>> virtio_device *vdev)
>>>>          rsc->status = 0;
>>>>        dev_dbg(&vdev->dev, "reset !\n");
>>>> +
>>>> +    return 0;
>>>>    }
>>>>      /* provide the vdev features as retrieved from the firmware */
>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>> index 54e686dca6de..52b32555e746 100644
>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>> virtio_device *vdev, unsigned nvqs,
>>>>        return ret;
>>>>    }
>>>>    -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>        struct ccw1 *ccw;
>>>> +    int ret;
>>>>          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>        if (!ccw)
>>>> -        return;
>>>> +        return -ENOMEM;
>>>>          /* Zero status bits. */
>>>>        vcdev->dma_area->status = 0;
>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>> virtio_device *vdev)
>>>>        ccw->flags = 0;
>>>>        ccw->count = 0;
>>>>        ccw->cda = 0;
>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>> +
>>>> +    return ret;
>>>>    }
>>>>      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>> --- a/drivers/virtio/virtio.c
>>>> +++ b/drivers/virtio/virtio.c
>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
>>>> *dev)
>>>>        /* Assign a unique device index and hence name. */
>>>>        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>        if (err < 0)
>>>> -        goto out;
>>>> +        goto out_err;
>>>>          dev->index = err;
>>>>        dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
>>>> *dev)
>>>>          /* We always start by resetting the device, in case a previous
>>>>         * driver messed it up.  This also tests that code path a
>>>> little. */
>>>> -    dev->config->reset(dev);
>>>> +    err = dev->config->reset(dev);
>>>> +    if (err)
>>>> +        goto out_ida;
>>>>          /* Acknowledge that we've seen the device. */
>>>>        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>> virtio_device *dev)
>>>>         */
>>>>        err = device_add(&dev->dev);
>>>>        if (err)
>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>> -out:
>>>> -    if (err)
>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>> +        goto out_ida;
>>>> +
>>>> +    return 0;
>>>> +
>>>> +out_ida:
>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>> +out_err:
>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>        return err;
>>>>    }
>>>>    EXPORT_SYMBOL_GPL(register_virtio_device);
>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>>>>          /* We always start by resetting the device, in case a previous
>>>>         * driver messed it up. */
>>>> -    dev->config->reset(dev);
>>>> +    ret = dev->config->reset(dev);
>>>> +    if (ret)
>>>> +        goto err;
>>>>          /* Acknowledge that we've seen the device. */
>>>>        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>> --- a/drivers/virtio/virtio_mmio.c
>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
>>>> *vdev, u8 status)
>>>>        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>    }
>>>>    -static void vm_reset(struct virtio_device *vdev)
>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>>          /* 0 status means a reset. */
>>>>        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>> +    return 0;
>>>>    }
>>>>      diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>> *vdev, u8 status)
>>>>        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>    }
>>>>    -static void vp_reset(struct virtio_device *vdev)
>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>        /* 0 status means a reset. */
>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>        /* Flush pending VQ/configuration callbacks. */
>>>>        vp_synchronize_vectors(vdev);
>>>> +
>>>> +    return 0;
>>>>    }
>>>>      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>> u16 vector)
>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>> b/drivers/virtio/virtio_pci_modern.c
>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
>>>> *vdev, u8 status)
>>>>        vp_modern_set_status(&vp_dev->mdev, status);
>>>>    }
>>>>    -static void vp_reset(struct virtio_device *vdev)
>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>            msleep(1);
>>>>        /* Flush pending VQ/configuration callbacks. */
>>>>        vp_synchronize_vectors(vdev);
>>>> +    return 0;
>>>>    }
>>>>      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>> u16 vector)
>>>> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>> virtio_device *vdev, u8 status)
>>>>        return ops->set_status(vdpa, status);
>>>>    }
>>>>    -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>    {
>>>>        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>          vdpa_reset(vdpa);
>>>> +
>>>> +    return 0;
>>>>    }
>>>>      static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>> diff --git a/include/linux/virtio_config.h
>>>> b/include/linux/virtio_config.h
>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>> --- a/include/linux/virtio_config.h
>>>> +++ b/include/linux/virtio_config.h
>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>     *    status: the new status byte
>>>>     * @reset: reset the device
>>>>     *    vdev: the virtio device
>>>> - *    After this, status and feature negotiation must be done again
>>>> + *    Upon success, status and feature negotiation must be done again
>>>>     *    Device must not be reset from its vq/config callbacks, or in
>>>>     *    parallel with being added/removed.
>>>> + *    Returns 0 on success or error status.
>>>>     * @find_vqs: find virtqueues and instantiate them.
>>>>     *    vdev: the virtio_device
>>>>     *    nvqs: the number of virtqueues to find
>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>        u32 (*generation)(struct virtio_device *vdev);
>>>>        u8 (*get_status)(struct virtio_device *vdev);
>>>>        void (*set_status)(struct virtio_device *vdev, u8 status);
>>>> -    void (*reset)(struct virtio_device *vdev);
>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>                struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>                const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> 
> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > The reset device operation, usually is an operation that might fail from
> > > > > various reasons. For example, the controller might be in a bad state and
> > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > devices always succeed in reset operation but this is not the case for
> > > > > HW based virtio devices.
> > > > 
> > > > I would like to know under what condition that the reset operation may
> > > > fail (except for the case of a bugg guest).
> > > The controller might not be ready or stuck. This is a real use case for many
> > > PCI devices.
> > > 
> > > For real devices the FW might be in a bad state and it can happen also for
> > > paravirt device if you have a bug in the controller code or if you entered
> > > some error flow (Out of memory).
> > > 
> > > You don't want to be stuck because of one bad device.
> > OK so maybe we can do more to detect the bad device.
> > Won't we get all 1's on a read in this case?
> 
> No. how can we guarantee it ?
> 

Well this is what you tend to get if e.g. you disable device memory.

Anyway, you know about hardware, I don't ... It's not returning 0 after
reset as it should ... what does it return? Hopefully not random noise -
I don't think it's very practical to write a driver for a device that
starts doing that at random times ...

> > 
> > 
> > > > 
> > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > resetting virtio devices.
> > > > > 
> > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > ---
> > > > > 
> > > > > changes from v1:
> > > > >    - update virtio_ccw.c (Cornelia)
> > > > >    - update virtio_uml.c
> > > > >    - update mlxbf-tmfifo.c
> > > > 
> > > > Note that virtio driver may call reset, so you probably need to convert
> > > > them.
> > > I'm sure I understand.
> > > 
> > > Convert to what ?
> > > 
> > > Thanks.
> > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > ---
> > > > >    arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > >    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > >    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > >    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > >    drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > >    drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > >    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > >    drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > >    drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > >    include/linux/virtio_config.h            |  5 +++--
> > > > >    10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > 
> > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        vu_dev->status = status;
> > > > >    }
> > > > >    -static void vu_reset(struct virtio_device *vdev)
> > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > >          vu_dev->status = 0;
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static void vu_del_vq(struct virtqueue *vq)
> > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > @@ -980,11 +980,13 @@ static void
> > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > >    }
> > > > >      /* Reset the device. Not much here for now. */
> > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > >          tm_vdev->status = 0;
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      /* Read the value of a configuration field. */
> > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > virtio_device *vdev, u8 status)
> > > > >        dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > >    }
> > > > >    -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > >        struct fw_rsc_vdev *rsc;
> > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > virtio_device *vdev)
> > > > >          rsc->status = 0;
> > > > >        dev_dbg(&vdev->dev, "reset !\n");
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      /* provide the vdev features as retrieved from the firmware */
> > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > index 54e686dca6de..52b32555e746 100644
> > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > virtio_device *vdev, unsigned nvqs,
> > > > >        return ret;
> > > > >    }
> > > > >    -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > >        struct ccw1 *ccw;
> > > > > +    int ret;
> > > > >          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > >        if (!ccw)
> > > > > -        return;
> > > > > +        return -ENOMEM;
> > > > >          /* Zero status bits. */
> > > > >        vcdev->dma_area->status = 0;
> > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > virtio_device *vdev)
> > > > >        ccw->flags = 0;
> > > > >        ccw->count = 0;
> > > > >        ccw->cda = 0;
> > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > >        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > +
> > > > > +    return ret;
> > > > >    }
> > > > >      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > --- a/drivers/virtio/virtio.c
> > > > > +++ b/drivers/virtio/virtio.c
> > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > *dev)
> > > > >        /* Assign a unique device index and hence name. */
> > > > >        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > >        if (err < 0)
> > > > > -        goto out;
> > > > > +        goto out_err;
> > > > >          dev->index = err;
> > > > >        dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > *dev)
> > > > >          /* We always start by resetting the device, in case a previous
> > > > >         * driver messed it up.  This also tests that code path a
> > > > > little. */
> > > > > -    dev->config->reset(dev);
> > > > > +    err = dev->config->reset(dev);
> > > > > +    if (err)
> > > > > +        goto out_ida;
> > > > >          /* Acknowledge that we've seen the device. */
> > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > virtio_device *dev)
> > > > >         */
> > > > >        err = device_add(&dev->dev);
> > > > >        if (err)
> > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > -out:
> > > > > -    if (err)
> > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > +        goto out_ida;
> > > > > +
> > > > > +    return 0;
> > > > > +
> > > > > +out_ida:
> > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > +out_err:
> > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > >        return err;
> > > > >    }
> > > > >    EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > >          /* We always start by resetting the device, in case a previous
> > > > >         * driver messed it up. */
> > > > > -    dev->config->reset(dev);
> > > > > +    ret = dev->config->reset(dev);
> > > > > +    if (ret)
> > > > > +        goto err;
> > > > >          /* Acknowledge that we've seen the device. */
> > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > >    }
> > > > >    -static void vm_reset(struct virtio_device *vdev)
> > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > >          /* 0 status means a reset. */
> > > > >        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > +    return 0;
> > > > >    }
> > > > >      diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > >    }
> > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > >        /* 0 status means a reset. */
> > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > >        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > >        vp_synchronize_vectors(vdev);
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > u16 vector)
> > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        vp_modern_set_status(&vp_dev->mdev, status);
> > > > >    }
> > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > >        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > >            msleep(1);
> > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > >        vp_synchronize_vectors(vdev);
> > > > > +    return 0;
> > > > >    }
> > > > >      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > u16 vector)
> > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > virtio_device *vdev, u8 status)
> > > > >        return ops->set_status(vdpa, status);
> > > > >    }
> > > > >    -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > >          vdpa_reset(vdpa);
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > diff --git a/include/linux/virtio_config.h
> > > > > b/include/linux/virtio_config.h
> > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > --- a/include/linux/virtio_config.h
> > > > > +++ b/include/linux/virtio_config.h
> > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > >     *    status: the new status byte
> > > > >     * @reset: reset the device
> > > > >     *    vdev: the virtio device
> > > > > - *    After this, status and feature negotiation must be done again
> > > > > + *    Upon success, status and feature negotiation must be done again
> > > > >     *    Device must not be reset from its vq/config callbacks, or in
> > > > >     *    parallel with being added/removed.
> > > > > + *    Returns 0 on success or error status.
> > > > >     * @find_vqs: find virtqueues and instantiate them.
> > > > >     *    vdev: the virtio_device
> > > > >     *    nvqs: the number of virtqueues to find
> > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > >        u32 (*generation)(struct virtio_device *vdev);
> > > > >        u8 (*get_status)(struct virtio_device *vdev);
> > > > >        void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > >        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > >                struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > >                const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> 
> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > The reset device operation, usually is an operation that might fail from
> > > > > various reasons. For example, the controller might be in a bad state and
> > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > devices always succeed in reset operation but this is not the case for
> > > > > HW based virtio devices.
> > > > 
> > > > I would like to know under what condition that the reset operation may
> > > > fail (except for the case of a bugg guest).
> > > The controller might not be ready or stuck. This is a real use case for many
> > > PCI devices.
> > > 
> > > For real devices the FW might be in a bad state and it can happen also for
> > > paravirt device if you have a bug in the controller code or if you entered
> > > some error flow (Out of memory).
> > > 
> > > You don't want to be stuck because of one bad device.
> > OK so maybe we can do more to detect the bad device.
> > Won't we get all 1's on a read in this case?
> 
> No. how can we guarantee it ?
> 

Well this is what you tend to get if e.g. you disable device memory.

Anyway, you know about hardware, I don't ... It's not returning 0 after
reset as it should ... what does it return? Hopefully not random noise -
I don't think it's very practical to write a driver for a device that
starts doing that at random times ...

> > 
> > 
> > > > 
> > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > resetting virtio devices.
> > > > > 
> > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > ---
> > > > > 
> > > > > changes from v1:
> > > > >    - update virtio_ccw.c (Cornelia)
> > > > >    - update virtio_uml.c
> > > > >    - update mlxbf-tmfifo.c
> > > > 
> > > > Note that virtio driver may call reset, so you probably need to convert
> > > > them.
> > > I'm sure I understand.
> > > 
> > > Convert to what ?
> > > 
> > > Thanks.
> > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > ---
> > > > >    arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > >    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > >    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > >    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > >    drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > >    drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > >    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > >    drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > >    drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > >    include/linux/virtio_config.h            |  5 +++--
> > > > >    10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > 
> > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        vu_dev->status = status;
> > > > >    }
> > > > >    -static void vu_reset(struct virtio_device *vdev)
> > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > >          vu_dev->status = 0;
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static void vu_del_vq(struct virtqueue *vq)
> > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > @@ -980,11 +980,13 @@ static void
> > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > >    }
> > > > >      /* Reset the device. Not much here for now. */
> > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > >          tm_vdev->status = 0;
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      /* Read the value of a configuration field. */
> > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > virtio_device *vdev, u8 status)
> > > > >        dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > >    }
> > > > >    -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > >        struct fw_rsc_vdev *rsc;
> > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > virtio_device *vdev)
> > > > >          rsc->status = 0;
> > > > >        dev_dbg(&vdev->dev, "reset !\n");
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      /* provide the vdev features as retrieved from the firmware */
> > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > index 54e686dca6de..52b32555e746 100644
> > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > virtio_device *vdev, unsigned nvqs,
> > > > >        return ret;
> > > > >    }
> > > > >    -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > >        struct ccw1 *ccw;
> > > > > +    int ret;
> > > > >          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > >        if (!ccw)
> > > > > -        return;
> > > > > +        return -ENOMEM;
> > > > >          /* Zero status bits. */
> > > > >        vcdev->dma_area->status = 0;
> > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > virtio_device *vdev)
> > > > >        ccw->flags = 0;
> > > > >        ccw->count = 0;
> > > > >        ccw->cda = 0;
> > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > >        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > +
> > > > > +    return ret;
> > > > >    }
> > > > >      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > --- a/drivers/virtio/virtio.c
> > > > > +++ b/drivers/virtio/virtio.c
> > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > *dev)
> > > > >        /* Assign a unique device index and hence name. */
> > > > >        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > >        if (err < 0)
> > > > > -        goto out;
> > > > > +        goto out_err;
> > > > >          dev->index = err;
> > > > >        dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > *dev)
> > > > >          /* We always start by resetting the device, in case a previous
> > > > >         * driver messed it up.  This also tests that code path a
> > > > > little. */
> > > > > -    dev->config->reset(dev);
> > > > > +    err = dev->config->reset(dev);
> > > > > +    if (err)
> > > > > +        goto out_ida;
> > > > >          /* Acknowledge that we've seen the device. */
> > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > virtio_device *dev)
> > > > >         */
> > > > >        err = device_add(&dev->dev);
> > > > >        if (err)
> > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > -out:
> > > > > -    if (err)
> > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > +        goto out_ida;
> > > > > +
> > > > > +    return 0;
> > > > > +
> > > > > +out_ida:
> > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > +out_err:
> > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > >        return err;
> > > > >    }
> > > > >    EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > >          /* We always start by resetting the device, in case a previous
> > > > >         * driver messed it up. */
> > > > > -    dev->config->reset(dev);
> > > > > +    ret = dev->config->reset(dev);
> > > > > +    if (ret)
> > > > > +        goto err;
> > > > >          /* Acknowledge that we've seen the device. */
> > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > >    }
> > > > >    -static void vm_reset(struct virtio_device *vdev)
> > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > >          /* 0 status means a reset. */
> > > > >        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > +    return 0;
> > > > >    }
> > > > >      diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > >    }
> > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > >        /* 0 status means a reset. */
> > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > >        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > >        vp_synchronize_vectors(vdev);
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > u16 vector)
> > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > *vdev, u8 status)
> > > > >        vp_modern_set_status(&vp_dev->mdev, status);
> > > > >    }
> > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > >        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > >            msleep(1);
> > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > >        vp_synchronize_vectors(vdev);
> > > > > +    return 0;
> > > > >    }
> > > > >      static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > u16 vector)
> > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > virtio_device *vdev, u8 status)
> > > > >        return ops->set_status(vdpa, status);
> > > > >    }
> > > > >    -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > >    {
> > > > >        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > >          vdpa_reset(vdpa);
> > > > > +
> > > > > +    return 0;
> > > > >    }
> > > > >      static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > diff --git a/include/linux/virtio_config.h
> > > > > b/include/linux/virtio_config.h
> > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > --- a/include/linux/virtio_config.h
> > > > > +++ b/include/linux/virtio_config.h
> > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > >     *    status: the new status byte
> > > > >     * @reset: reset the device
> > > > >     *    vdev: the virtio device
> > > > > - *    After this, status and feature negotiation must be done again
> > > > > + *    Upon success, status and feature negotiation must be done again
> > > > >     *    Device must not be reset from its vq/config callbacks, or in
> > > > >     *    parallel with being added/removed.
> > > > > + *    Returns 0 on success or error status.
> > > > >     * @find_vqs: find virtqueues and instantiate them.
> > > > >     *    vdev: the virtio_device
> > > > >     *    nvqs: the number of virtqueues to find
> > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > >        u32 (*generation)(struct virtio_device *vdev);
> > > > >        u8 (*get_status)(struct virtio_device *vdev);
> > > > >        void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > >        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > >                struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > >                const char * const names[], const bool *ctx,

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/12/2021 12:07 PM, Michael S. Tsirkin wrote:
> On Sun, Apr 11, 2021 at 12:50:22PM +0300, Max Gurtovoy wrote:
>> On 4/9/2021 8:22 AM, Jason Wang wrote:
>>> 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
>>>> On 4/8/2021 4:14 PM, Jason Wang wrote:
>>>>> 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
>>>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>>>> The reset device operation, usually is an operation
>>>>>>>> that might fail from
>>>>>>>> various reasons. For example, the controller might
>>>>>>>> be in a bad state and
>>>>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>>>>> devices always succeed in reset operation but this
>>>>>>>> is not the case for
>>>>>>>> HW based virtio devices.
>>>>>>>
>>>>>>> I would like to know under what condition that the reset
>>>>>>> operation may fail (except for the case of a bugg
>>>>>>> guest).
>>>>>> The controller might not be ready or stuck. This is a real
>>>>>> use case for many PCI devices.
>>>>>>
>>>>>> For real devices the FW might be in a bad state and it can
>>>>>> happen also for paravirt device if you have a bug in the
>>>>>> controller code or if you entered some error flow (Out of
>>>>>> memory).
>>>>>>
>>>>>> You don't want to be stuck because of one bad device.
>>>>>
>>>>> So the buggy driver can damage the host through various ways,
>>>>> I'm not sure doing such workaround is worthwhile.
>>>> do you mean device ?
>>>
>>> Yes.
>>>
>>>
>>>> sometimes you need to replace device FW and it will work.
>>>>
>>>> I don't think it's a workaround. Other protocols, such as NVMe,
>>>> solved this in the specification.
>>>>
>>>> PCI config space and PCI controller are sometimes 2 different
>>>> components and sometimes the controller is not active, although the
>>>> device is plugged and seen by the PCI subsystem.
>>>
>>> So I think we need patch to spec to see if it works first.
>> We can't leave the driver to loop forever without allowing next "good"
>> virtio devices to probe.
>>
>> We can in parallel look for spec fixes but for now we must fix the driver.
> I'd like to narrow the case though. the proper thing
> would probably be for device to clear the status.
> Provided it entered some state where it can not do it -
> is there anything special about
> the device state that *can* be detected?
> If yes what is it?

We don't have anything in the spec to detect it.

Different devices can have different behavior.

The only thing we can count on is that the device didn't present a 0 in 
device_status once that was requested.


>
>> The fix can be using the mechanism introduced in this series or adding an
>> async probing mechanism.
> What would that be?

just schedule some work using queue_work/schedule_work.

And wait for it in the virtio_pci_remove

>> In both solutions, we can't allow looping forever.
> Multiple minute downtime isn't much better either. I'd like a
> much shorter timer or even no timer at all, instead
> verifying that device is alive every X iterations.

I added an option to configure the module parameter.

I think 2-3 minutes to wait is not so bad and one can configure it for 
any value that fits.


>
>>>
>>>>
>>>>> Note that this driver has been used for real hardware PCI
>>>>> devices for many years. We don't receive any report of this
>>>>> before.
>>>>>
>>>>>
>>>>>>
>>>>>>>
>>>>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>>>>> resetting virtio devices.
>>>>>>>>
>>>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>>>> ---
>>>>>>>>
>>>>>>>> changes from v1:
>>>>>>>>    - update virtio_ccw.c (Cornelia)
>>>>>>>>    - update virtio_uml.c
>>>>>>>>    - update mlxbf-tmfifo.c
>>>>>>>
>>>>>>> Note that virtio driver may call reset, so you probably
>>>>>>> need to convert them.
>>>>>> I'm sure I understand.
>>>>>>
>>>>>> Convert to what ?
>>>>>
>>>>> Convert to deal with the possible rest failure. E.g in
>>>>> virtblk_freeze() we had:
>>>>>
>>>>> static int virtblk_freeze(struct virtio_device *vdev)
>>>>> {
>>>>>          struct virtio_blk *vblk = vdev->priv;
>>>>>
>>>>>          /* Ensure we don't receive any more interrupts */
>>>>>          vdev->config->reset(vdev);
>>>>> ...
>>>>>
>>>>> We need fail the freeze here.
>>>>
>>>> Agree.
>>>>
>>>>
>>>>> Another example is the driver remove which is not expected to be
>>>>> fail. A lot of virtio drivers tries to call reset there. I'm not
>>>>> sure how hard to deal with the failure in the path of remove
>>>>> (e.g __device_release_driver tends to ignore the return value of
>>>>> bus->remove().)
>>>> I think it can stay as-is and ignore the ->reset return value and
>>>> continue free the other resources to avoid leakage.
>>>
>>> The problem is that it's unclear that what kind of behaviour would the
>>> device do, e.g can it still send interrupts?
>>>
>>> That's why we need to formalize the bahviour first if necessary.
>>>
>>> Thanks
>>>
>>>
>>>>
>>>>> Thanks
>>>>>
>>>>>
>>>>>> Thanks.
>>>>>>
>>>>>>> Thanks
>>>>>>>
>>>>>>>
>>>>>>>> ---
>>>>>>>>    arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>>>    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>>>    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>>>    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>>>    drivers/virtio/virtio.c                  | 22
>>>>>>>> +++++++++++++++-------
>>>>>>>>    drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>>>    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>>>    drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>>>    drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>>>    include/linux/virtio_config.h            |  5 +++--
>>>>>>>>    10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/arch/um/drivers/virtio_uml.c
>>>>>>>> b/arch/um/drivers/virtio_uml.c
>>>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>>>> @@ -827,11 +827,13 @@ static void
>>>>>>>> vu_set_status(struct virtio_device *vdev, u8 status)
>>>>>>>>        vu_dev->status = status;
>>>>>>>>    }
>>>>>>>>    -static void vu_reset(struct virtio_device *vdev)
>>>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>>>>          vu_dev->status = 0;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      static void vu_del_vq(struct virtqueue *vq)
>>>>>>>> diff --git
>>>>>>>> a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device
>>>>>>>> *vdev,
>>>>>>>>    }
>>>>>>>>      /* Reset the device. Not much here for now. */
>>>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct mlxbf_tmfifo_vdev *tm_vdev =
>>>>>>>> mlxbf_vdev_to_tmfifo(vdev);
>>>>>>>>          tm_vdev->status = 0;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      /* Read the value of a configuration field. */
>>>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> @@ -191,7 +191,7 @@ static void
>>>>>>>> rproc_virtio_set_status(struct virtio_device *vdev,
>>>>>>>> u8 status)
>>>>>>>>        dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>>>    }
>>>>>>>>    -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>>>        struct fw_rsc_vdev *rsc;
>>>>>>>> @@ -200,6 +200,8 @@ static void
>>>>>>>> rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>          rsc->status = 0;
>>>>>>>>        dev_dbg(&vdev->dev, "reset !\n");
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      /* provide the vdev features as retrieved from the firmware */
>>>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> @@ -732,14 +732,15 @@ static int
>>>>>>>> virtio_ccw_find_vqs(struct virtio_device *vdev,
>>>>>>>> unsigned nvqs,
>>>>>>>>        return ret;
>>>>>>>>    }
>>>>>>>>    -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>>>        struct ccw1 *ccw;
>>>>>>>> +    int ret;
>>>>>>>>          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>>>>        if (!ccw)
>>>>>>>> -        return;
>>>>>>>> +        return -ENOMEM;
>>>>>>>>          /* Zero status bits. */
>>>>>>>>        vcdev->dma_area->status = 0;
>>>>>>>> @@ -749,8 +750,10 @@ static void
>>>>>>>> virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>        ccw->flags = 0;
>>>>>>>>        ccw->count = 0;
>>>>>>>>        ccw->cda = 0;
>>>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>>>> +
>>>>>>>> +    return ret;
>>>>>>>>    }
>>>>>>>>      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>>>> --- a/drivers/virtio/virtio.c
>>>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>>>> @@ -338,7 +338,7 @@ int
>>>>>>>> register_virtio_device(struct virtio_device *dev)
>>>>>>>>        /* Assign a unique device index and hence name. */
>>>>>>>>        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>>>>        if (err < 0)
>>>>>>>> -        goto out;
>>>>>>>> +        goto out_err;
>>>>>>>>          dev->index = err;
>>>>>>>>        dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>>>> @@ -349,7 +349,9 @@ int
>>>>>>>> register_virtio_device(struct virtio_device *dev)
>>>>>>>>          /* We always start by resetting the device,
>>>>>>>> in case a previous
>>>>>>>>         * driver messed it up.  This also tests that
>>>>>>>> code path a little. */
>>>>>>>> -    dev->config->reset(dev);
>>>>>>>> +    err = dev->config->reset(dev);
>>>>>>>> +    if (err)
>>>>>>>> +        goto out_ida;
>>>>>>>>          /* Acknowledge that we've seen the device. */
>>>>>>>>        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>> @@ -362,10 +364,14 @@ int
>>>>>>>> register_virtio_device(struct virtio_device *dev)
>>>>>>>>         */
>>>>>>>>        err = device_add(&dev->dev);
>>>>>>>>        if (err)
>>>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>> -out:
>>>>>>>> -    if (err)
>>>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>> +        goto out_ida;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>> +
>>>>>>>> +out_ida:
>>>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>> +out_err:
>>>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>        return err;
>>>>>>>>    }
>>>>>>>>    EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct
>>>>>>>> virtio_device *dev)
>>>>>>>>          /* We always start by resetting the device,
>>>>>>>> in case a previous
>>>>>>>>         * driver messed it up. */
>>>>>>>> -    dev->config->reset(dev);
>>>>>>>> +    ret = dev->config->reset(dev);
>>>>>>>> +    if (ret)
>>>>>>>> +        goto err;
>>>>>>>>          /* Acknowledge that we've seen the device. */
>>>>>>>>        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>> diff --git a/drivers/virtio/virtio_mmio.c
>>>>>>>> b/drivers/virtio/virtio_mmio.c
>>>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>>>> @@ -256,12 +256,13 @@ static void
>>>>>>>> vm_set_status(struct virtio_device *vdev, u8 status)
>>>>>>>>        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>    }
>>>>>>>>    -static void vm_reset(struct virtio_device *vdev)
>>>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct virtio_mmio_device *vm_dev =
>>>>>>>> to_virtio_mmio_device(vdev);
>>>>>>>>          /* 0 status means a reset. */
>>>>>>>>        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct
>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>    }
>>>>>>>>    -static void vp_reset(struct virtio_device *vdev)
>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>        /* 0 status means a reset. */
>>>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>        /* Flush pending VQ/configuration callbacks. */
>>>>>>>>        vp_synchronize_vectors(vdev);
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      static u16 vp_config_vector(struct
>>>>>>>> virtio_pci_device *vp_dev, u16 vector)
>>>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct
>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>        vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>>>    }
>>>>>>>>    -static void vp_reset(struct virtio_device *vdev)
>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>            msleep(1);
>>>>>>>>        /* Flush pending VQ/configuration callbacks. */
>>>>>>>>        vp_synchronize_vectors(vdev);
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      static u16 vp_config_vector(struct
>>>>>>>> virtio_pci_device *vp_dev, u16 vector)
>>>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c
>>>>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>>>> @@ -97,11 +97,13 @@ static void
>>>>>>>> virtio_vdpa_set_status(struct virtio_device *vdev,
>>>>>>>> u8 status)
>>>>>>>>        return ops->set_status(vdpa, status);
>>>>>>>>    }
>>>>>>>>    -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>    {
>>>>>>>>        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>>>          vdpa_reset(vdpa);
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>    }
>>>>>>>>      static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>>>> b/include/linux/virtio_config.h
>>>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>>>> --- a/include/linux/virtio_config.h
>>>>>>>> +++ b/include/linux/virtio_config.h
>>>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>>>     *    status: the new status byte
>>>>>>>>     * @reset: reset the device
>>>>>>>>     *    vdev: the virtio device
>>>>>>>> - *    After this, status and feature negotiation must be done again
>>>>>>>> + *    Upon success, status and feature negotiation
>>>>>>>> must be done again
>>>>>>>>     *    Device must not be reset from its vq/config callbacks, or in
>>>>>>>>     *    parallel with being added/removed.
>>>>>>>> + *    Returns 0 on success or error status.
>>>>>>>>     * @find_vqs: find virtqueues and instantiate them.
>>>>>>>>     *    vdev: the virtio_device
>>>>>>>>     *    nvqs: the number of virtqueues to find
>>>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>>>        u32 (*generation)(struct virtio_device *vdev);
>>>>>>>>        u8 (*get_status)(struct virtio_device *vdev);
>>>>>>>>        void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>>>        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>>>                struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>>>>                const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
>> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
>>> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>> The reset device operation, usually is an operation that might fail from
>>>>>> various reasons. For example, the controller might be in a bad state and
>>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>>> devices always succeed in reset operation but this is not the case for
>>>>>> HW based virtio devices.
>>>>> I would like to know under what condition that the reset operation may
>>>>> fail (except for the case of a bugg guest).
>>>> The controller might not be ready or stuck. This is a real use case for many
>>>> PCI devices.
>>>>
>>>> For real devices the FW might be in a bad state and it can happen also for
>>>> paravirt device if you have a bug in the controller code or if you entered
>>>> some error flow (Out of memory).
>>>>
>>>> You don't want to be stuck because of one bad device.
>>> OK so maybe we can do more to detect the bad device.
>>> Won't we get all 1's on a read in this case?
>> No. how can we guarantee it ?
>>
> Well this is what you tend to get if e.g. you disable device memory.
>
> Anyway, you know about hardware, I don't ... It's not returning 0 after
> reset as it should ... what does it return? Hopefully not random noise -
> I don't think it's very practical to write a driver for a device that
> starts doing that at random times ...

The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.

For paravirt devices, think of a situation that you can't allocate some 
internal buffers (malloc failed) and you want to wait for few seconds 
until the system memory will free some pages.

So you may return NEEDS_RESET that indicates some error state of the 
device. Once the system memory freed by other application for example, 
your internal virtio device malloc succeeded and you may return 0.

In this case, you don't want to stall the other virtio devices to probe 
(they might be real HW devices that driven by the same driver), right ?


>
>>>
>>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>>> resetting virtio devices.
>>>>>>
>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>> ---
>>>>>>
>>>>>> changes from v1:
>>>>>>     - update virtio_ccw.c (Cornelia)
>>>>>>     - update virtio_uml.c
>>>>>>     - update mlxbf-tmfifo.c
>>>>> Note that virtio driver may call reset, so you probably need to convert
>>>>> them.
>>>> I'm sure I understand.
>>>>
>>>> Convert to what ?
>>>>
>>>> Thanks.
>>>>
>>>>> Thanks
>>>>>
>>>>>
>>>>>> ---
>>>>>>     arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>     drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>     drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>     drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>     drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>>>>     drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>     drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>     drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>     drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>     include/linux/virtio_config.h            |  5 +++--
>>>>>>     10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>
>>>>>> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
>>>>>> *vdev, u8 status)
>>>>>>         vu_dev->status = status;
>>>>>>     }
>>>>>>     -static void vu_reset(struct virtio_device *vdev)
>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>>           vu_dev->status = 0;
>>>>>> +
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       static void vu_del_vq(struct virtqueue *vq)
>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>     }
>>>>>>       /* Reset the device. Not much here for now. */
>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>>>           tm_vdev->status = 0;
>>>>>> +
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       /* Read the value of a configuration field. */
>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>>>> virtio_device *vdev, u8 status)
>>>>>>         dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>     }
>>>>>>     -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>         struct fw_rsc_vdev *rsc;
>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>>>> virtio_device *vdev)
>>>>>>           rsc->status = 0;
>>>>>>         dev_dbg(&vdev->dev, "reset !\n");
>>>>>> +
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       /* provide the vdev features as retrieved from the firmware */
>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>         return ret;
>>>>>>     }
>>>>>>     -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>         struct ccw1 *ccw;
>>>>>> +    int ret;
>>>>>>           ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>>         if (!ccw)
>>>>>> -        return;
>>>>>> +        return -ENOMEM;
>>>>>>           /* Zero status bits. */
>>>>>>         vcdev->dma_area->status = 0;
>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>>>> virtio_device *vdev)
>>>>>>         ccw->flags = 0;
>>>>>>         ccw->count = 0;
>>>>>>         ccw->cda = 0;
>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>         ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>> +
>>>>>> +    return ret;
>>>>>>     }
>>>>>>       static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>> --- a/drivers/virtio/virtio.c
>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
>>>>>> *dev)
>>>>>>         /* Assign a unique device index and hence name. */
>>>>>>         err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>>         if (err < 0)
>>>>>> -        goto out;
>>>>>> +        goto out_err;
>>>>>>           dev->index = err;
>>>>>>         dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
>>>>>> *dev)
>>>>>>           /* We always start by resetting the device, in case a previous
>>>>>>          * driver messed it up.  This also tests that code path a
>>>>>> little. */
>>>>>> -    dev->config->reset(dev);
>>>>>> +    err = dev->config->reset(dev);
>>>>>> +    if (err)
>>>>>> +        goto out_ida;
>>>>>>           /* Acknowledge that we've seen the device. */
>>>>>>         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>>>> virtio_device *dev)
>>>>>>          */
>>>>>>         err = device_add(&dev->dev);
>>>>>>         if (err)
>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>> -out:
>>>>>> -    if (err)
>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>> +        goto out_ida;
>>>>>> +
>>>>>> +    return 0;
>>>>>> +
>>>>>> +out_ida:
>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>> +out_err:
>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>         return err;
>>>>>>     }
>>>>>>     EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>>>>>>           /* We always start by resetting the device, in case a previous
>>>>>>          * driver messed it up. */
>>>>>> -    dev->config->reset(dev);
>>>>>> +    ret = dev->config->reset(dev);
>>>>>> +    if (ret)
>>>>>> +        goto err;
>>>>>>           /* Acknowledge that we've seen the device. */
>>>>>>         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
>>>>>> *vdev, u8 status)
>>>>>>         writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>     }
>>>>>>     -static void vm_reset(struct virtio_device *vdev)
>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>>>>           /* 0 status means a reset. */
>>>>>>         writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>>>> *vdev, u8 status)
>>>>>>         iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>     }
>>>>>>     -static void vp_reset(struct virtio_device *vdev)
>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>         /* 0 status means a reset. */
>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>         ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>         /* Flush pending VQ/configuration callbacks. */
>>>>>>         vp_synchronize_vectors(vdev);
>>>>>> +
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>> u16 vector)
>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
>>>>>> *vdev, u8 status)
>>>>>>         vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>     }
>>>>>>     -static void vp_reset(struct virtio_device *vdev)
>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>         struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>             msleep(1);
>>>>>>         /* Flush pending VQ/configuration callbacks. */
>>>>>>         vp_synchronize_vectors(vdev);
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>> u16 vector)
>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>>>> virtio_device *vdev, u8 status)
>>>>>>         return ops->set_status(vdpa, status);
>>>>>>     }
>>>>>>     -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>     {
>>>>>>         struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>           vdpa_reset(vdpa);
>>>>>> +
>>>>>> +    return 0;
>>>>>>     }
>>>>>>       static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>> b/include/linux/virtio_config.h
>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>> --- a/include/linux/virtio_config.h
>>>>>> +++ b/include/linux/virtio_config.h
>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>      *    status: the new status byte
>>>>>>      * @reset: reset the device
>>>>>>      *    vdev: the virtio device
>>>>>> - *    After this, status and feature negotiation must be done again
>>>>>> + *    Upon success, status and feature negotiation must be done again
>>>>>>      *    Device must not be reset from its vq/config callbacks, or in
>>>>>>      *    parallel with being added/removed.
>>>>>> + *    Returns 0 on success or error status.
>>>>>>      * @find_vqs: find virtqueues and instantiate them.
>>>>>>      *    vdev: the virtio_device
>>>>>>      *    nvqs: the number of virtqueues to find
>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>         u32 (*generation)(struct virtio_device *vdev);
>>>>>>         u8 (*get_status)(struct virtio_device *vdev);
>>>>>>         void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>         int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>                 struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>>                 const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> 
> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > The reset device operation, usually is an operation that might fail from
> > > > > > > various reasons. For example, the controller might be in a bad state and
> > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > devices always succeed in reset operation but this is not the case for
> > > > > > > HW based virtio devices.
> > > > > > I would like to know under what condition that the reset operation may
> > > > > > fail (except for the case of a bugg guest).
> > > > > The controller might not be ready or stuck. This is a real use case for many
> > > > > PCI devices.
> > > > > 
> > > > > For real devices the FW might be in a bad state and it can happen also for
> > > > > paravirt device if you have a bug in the controller code or if you entered
> > > > > some error flow (Out of memory).
> > > > > 
> > > > > You don't want to be stuck because of one bad device.
> > > > OK so maybe we can do more to detect the bad device.
> > > > Won't we get all 1's on a read in this case?
> > > No. how can we guarantee it ?
> > > 
> > Well this is what you tend to get if e.g. you disable device memory.
> > 
> > Anyway, you know about hardware, I don't ... It's not returning 0 after
> > reset as it should ... what does it return? Hopefully not random noise -
> > I don't think it's very practical to write a driver for a device that
> > starts doing that at random times ...
> 
> The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
>
> For paravirt devices, think of a situation that you can't allocate some
> internal buffers (malloc failed) and you want to wait for few seconds until
> the system memory will free some pages.
> 
> So you may return NEEDS_RESET that indicates some error state of the device.
> Once the system memory freed by other application for example, your internal
> virtio device malloc succeeded and you may return 0.
> 
> In this case, you don't want to stall the other virtio devices to probe
> (they might be real HW devices that driven by the same driver), right ?

So the device is very busy then? Not sure it's smart to just assume
it's safe to free all memory allocated for it then ...

I guess the lesson is don't make device reset depend on malloc
of some memory?



> 
> > 
> > > > 
> > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > resetting virtio devices.
> > > > > > > 
> > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > ---
> > > > > > > 
> > > > > > > changes from v1:
> > > > > > >     - update virtio_ccw.c (Cornelia)
> > > > > > >     - update virtio_uml.c
> > > > > > >     - update mlxbf-tmfifo.c
> > > > > > Note that virtio driver may call reset, so you probably need to convert
> > > > > > them.
> > > > > I'm sure I understand.
> > > > > 
> > > > > Convert to what ?
> > > > > 
> > > > > Thanks.
> > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > ---
> > > > > > >     arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > >     drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > >     drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > >     drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > >     drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > > > >     drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > >     drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > >     drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > >     drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > >     include/linux/virtio_config.h            |  5 +++--
> > > > > > >     10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > 
> > > > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         vu_dev->status = status;
> > > > > > >     }
> > > > > > >     -static void vu_reset(struct virtio_device *vdev)
> > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > >           vu_dev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static void vu_del_vq(struct virtqueue *vq)
> > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > >     }
> > > > > > >       /* Reset the device. Not much here for now. */
> > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > >           tm_vdev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       /* Read the value of a configuration field. */
> > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >         dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > >     }
> > > > > > >     -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > >         struct fw_rsc_vdev *rsc;
> > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > virtio_device *vdev)
> > > > > > >           rsc->status = 0;
> > > > > > >         dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       /* provide the vdev features as retrieved from the firmware */
> > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > >         return ret;
> > > > > > >     }
> > > > > > >     -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > >         struct ccw1 *ccw;
> > > > > > > +    int ret;
> > > > > > >           ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > >         if (!ccw)
> > > > > > > -        return;
> > > > > > > +        return -ENOMEM;
> > > > > > >           /* Zero status bits. */
> > > > > > >         vcdev->dma_area->status = 0;
> > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > virtio_device *vdev)
> > > > > > >         ccw->flags = 0;
> > > > > > >         ccw->count = 0;
> > > > > > >         ccw->cda = 0;
> > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > >         ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > +
> > > > > > > +    return ret;
> > > > > > >     }
> > > > > > >       static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > > > *dev)
> > > > > > >         /* Assign a unique device index and hence name. */
> > > > > > >         err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > >         if (err < 0)
> > > > > > > -        goto out;
> > > > > > > +        goto out_err;
> > > > > > >           dev->index = err;
> > > > > > >         dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > > > *dev)
> > > > > > >           /* We always start by resetting the device, in case a previous
> > > > > > >          * driver messed it up.  This also tests that code path a
> > > > > > > little. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    err = dev->config->reset(dev);
> > > > > > > +    if (err)
> > > > > > > +        goto out_ida;
> > > > > > >           /* Acknowledge that we've seen the device. */
> > > > > > >         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > virtio_device *dev)
> > > > > > >          */
> > > > > > >         err = device_add(&dev->dev);
> > > > > > >         if (err)
> > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > -out:
> > > > > > > -    if (err)
> > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > +        goto out_ida;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > > +
> > > > > > > +out_ida:
> > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > +out_err:
> > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > >         return err;
> > > > > > >     }
> > > > > > >     EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > > > >           /* We always start by resetting the device, in case a previous
> > > > > > >          * driver messed it up. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > +    if (ret)
> > > > > > > +        goto err;
> > > > > > >           /* Acknowledge that we've seen the device. */
> > > > > > >         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > >     }
> > > > > > >     -static void vm_reset(struct virtio_device *vdev)
> > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > >           /* 0 status means a reset. */
> > > > > > >         writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >     }
> > > > > > >     -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >         /* 0 status means a reset. */
> > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >         ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >         /* Flush pending VQ/configuration callbacks. */
> > > > > > >         vp_synchronize_vectors(vdev);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > >     }
> > > > > > >     -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >         struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >             msleep(1);
> > > > > > >         /* Flush pending VQ/configuration callbacks. */
> > > > > > >         vp_synchronize_vectors(vdev);
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >         return ops->set_status(vdpa, status);
> > > > > > >     }
> > > > > > >     -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > >           vdpa_reset(vdpa);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > b/include/linux/virtio_config.h
> > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > >      *    status: the new status byte
> > > > > > >      * @reset: reset the device
> > > > > > >      *    vdev: the virtio device
> > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > + *    Upon success, status and feature negotiation must be done again
> > > > > > >      *    Device must not be reset from its vq/config callbacks, or in
> > > > > > >      *    parallel with being added/removed.
> > > > > > > + *    Returns 0 on success or error status.
> > > > > > >      * @find_vqs: find virtqueues and instantiate them.
> > > > > > >      *    vdev: the virtio_device
> > > > > > >      *    nvqs: the number of virtqueues to find
> > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > >         u32 (*generation)(struct virtio_device *vdev);
> > > > > > >         u8 (*get_status)(struct virtio_device *vdev);
> > > > > > >         void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > >         int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > >                 struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > >                 const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> 
> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > The reset device operation, usually is an operation that might fail from
> > > > > > > various reasons. For example, the controller might be in a bad state and
> > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > devices always succeed in reset operation but this is not the case for
> > > > > > > HW based virtio devices.
> > > > > > I would like to know under what condition that the reset operation may
> > > > > > fail (except for the case of a bugg guest).
> > > > > The controller might not be ready or stuck. This is a real use case for many
> > > > > PCI devices.
> > > > > 
> > > > > For real devices the FW might be in a bad state and it can happen also for
> > > > > paravirt device if you have a bug in the controller code or if you entered
> > > > > some error flow (Out of memory).
> > > > > 
> > > > > You don't want to be stuck because of one bad device.
> > > > OK so maybe we can do more to detect the bad device.
> > > > Won't we get all 1's on a read in this case?
> > > No. how can we guarantee it ?
> > > 
> > Well this is what you tend to get if e.g. you disable device memory.
> > 
> > Anyway, you know about hardware, I don't ... It's not returning 0 after
> > reset as it should ... what does it return? Hopefully not random noise -
> > I don't think it's very practical to write a driver for a device that
> > starts doing that at random times ...
> 
> The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
>
> For paravirt devices, think of a situation that you can't allocate some
> internal buffers (malloc failed) and you want to wait for few seconds until
> the system memory will free some pages.
> 
> So you may return NEEDS_RESET that indicates some error state of the device.
> Once the system memory freed by other application for example, your internal
> virtio device malloc succeeded and you may return 0.
> 
> In this case, you don't want to stall the other virtio devices to probe
> (they might be real HW devices that driven by the same driver), right ?

So the device is very busy then? Not sure it's smart to just assume
it's safe to free all memory allocated for it then ...

I guess the lesson is don't make device reset depend on malloc
of some memory?



> 
> > 
> > > > 
> > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > resetting virtio devices.
> > > > > > > 
> > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > ---
> > > > > > > 
> > > > > > > changes from v1:
> > > > > > >     - update virtio_ccw.c (Cornelia)
> > > > > > >     - update virtio_uml.c
> > > > > > >     - update mlxbf-tmfifo.c
> > > > > > Note that virtio driver may call reset, so you probably need to convert
> > > > > > them.
> > > > > I'm sure I understand.
> > > > > 
> > > > > Convert to what ?
> > > > > 
> > > > > Thanks.
> > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > ---
> > > > > > >     arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > >     drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > >     drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > >     drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > >     drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > > > >     drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > >     drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > >     drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > >     drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > >     include/linux/virtio_config.h            |  5 +++--
> > > > > > >     10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > 
> > > > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         vu_dev->status = status;
> > > > > > >     }
> > > > > > >     -static void vu_reset(struct virtio_device *vdev)
> > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > >           vu_dev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static void vu_del_vq(struct virtqueue *vq)
> > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > >     }
> > > > > > >       /* Reset the device. Not much here for now. */
> > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > >           tm_vdev->status = 0;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       /* Read the value of a configuration field. */
> > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >         dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > >     }
> > > > > > >     -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > >         struct fw_rsc_vdev *rsc;
> > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > virtio_device *vdev)
> > > > > > >           rsc->status = 0;
> > > > > > >         dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       /* provide the vdev features as retrieved from the firmware */
> > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > >         return ret;
> > > > > > >     }
> > > > > > >     -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > >         struct ccw1 *ccw;
> > > > > > > +    int ret;
> > > > > > >           ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > >         if (!ccw)
> > > > > > > -        return;
> > > > > > > +        return -ENOMEM;
> > > > > > >           /* Zero status bits. */
> > > > > > >         vcdev->dma_area->status = 0;
> > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > virtio_device *vdev)
> > > > > > >         ccw->flags = 0;
> > > > > > >         ccw->count = 0;
> > > > > > >         ccw->cda = 0;
> > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > >         ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > +
> > > > > > > +    return ret;
> > > > > > >     }
> > > > > > >       static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > > > *dev)
> > > > > > >         /* Assign a unique device index and hence name. */
> > > > > > >         err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > >         if (err < 0)
> > > > > > > -        goto out;
> > > > > > > +        goto out_err;
> > > > > > >           dev->index = err;
> > > > > > >         dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > > > *dev)
> > > > > > >           /* We always start by resetting the device, in case a previous
> > > > > > >          * driver messed it up.  This also tests that code path a
> > > > > > > little. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    err = dev->config->reset(dev);
> > > > > > > +    if (err)
> > > > > > > +        goto out_ida;
> > > > > > >           /* Acknowledge that we've seen the device. */
> > > > > > >         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > virtio_device *dev)
> > > > > > >          */
> > > > > > >         err = device_add(&dev->dev);
> > > > > > >         if (err)
> > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > -out:
> > > > > > > -    if (err)
> > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > +        goto out_ida;
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > > +
> > > > > > > +out_ida:
> > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > +out_err:
> > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > >         return err;
> > > > > > >     }
> > > > > > >     EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > > > >           /* We always start by resetting the device, in case a previous
> > > > > > >          * driver messed it up. */
> > > > > > > -    dev->config->reset(dev);
> > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > +    if (ret)
> > > > > > > +        goto err;
> > > > > > >           /* Acknowledge that we've seen the device. */
> > > > > > >         virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > >     }
> > > > > > >     -static void vm_reset(struct virtio_device *vdev)
> > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > >           /* 0 status means a reset. */
> > > > > > >         writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >     }
> > > > > > >     -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >         /* 0 status means a reset. */
> > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >         ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > >         /* Flush pending VQ/configuration callbacks. */
> > > > > > >         vp_synchronize_vectors(vdev);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > *vdev, u8 status)
> > > > > > >         vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > >     }
> > > > > > >     -static void vp_reset(struct virtio_device *vdev)
> > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > >         struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > >             msleep(1);
> > > > > > >         /* Flush pending VQ/configuration callbacks. */
> > > > > > >         vp_synchronize_vectors(vdev);
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > u16 vector)
> > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > virtio_device *vdev, u8 status)
> > > > > > >         return ops->set_status(vdpa, status);
> > > > > > >     }
> > > > > > >     -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > >     {
> > > > > > >         struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > >           vdpa_reset(vdpa);
> > > > > > > +
> > > > > > > +    return 0;
> > > > > > >     }
> > > > > > >       static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > b/include/linux/virtio_config.h
> > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > >      *    status: the new status byte
> > > > > > >      * @reset: reset the device
> > > > > > >      *    vdev: the virtio device
> > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > + *    Upon success, status and feature negotiation must be done again
> > > > > > >      *    Device must not be reset from its vq/config callbacks, or in
> > > > > > >      *    parallel with being added/removed.
> > > > > > > + *    Returns 0 on success or error status.
> > > > > > >      * @find_vqs: find virtqueues and instantiate them.
> > > > > > >      *    vdev: the virtio_device
> > > > > > >      *    nvqs: the number of virtqueues to find
> > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > >         u32 (*generation)(struct virtio_device *vdev);
> > > > > > >         u8 (*get_status)(struct virtio_device *vdev);
> > > > > > >         void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > >         int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > >                 struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > >                 const char * const names[], const bool *ctx,

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 03:17:02PM +0300, Max Gurtovoy wrote:
> 
> On 4/12/2021 12:07 PM, Michael S. Tsirkin wrote:
> > On Sun, Apr 11, 2021 at 12:50:22PM +0300, Max Gurtovoy wrote:
> > > On 4/9/2021 8:22 AM, Jason Wang wrote:
> > > > 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
> > > > > On 4/8/2021 4:14 PM, Jason Wang wrote:
> > > > > > 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
> > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > The reset device operation, usually is an operation
> > > > > > > > > that might fail from
> > > > > > > > > various reasons. For example, the controller might
> > > > > > > > > be in a bad state and
> > > > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > > > devices always succeed in reset operation but this
> > > > > > > > > is not the case for
> > > > > > > > > HW based virtio devices.
> > > > > > > > 
> > > > > > > > I would like to know under what condition that the reset
> > > > > > > > operation may fail (except for the case of a bugg
> > > > > > > > guest).
> > > > > > > The controller might not be ready or stuck. This is a real
> > > > > > > use case for many PCI devices.
> > > > > > > 
> > > > > > > For real devices the FW might be in a bad state and it can
> > > > > > > happen also for paravirt device if you have a bug in the
> > > > > > > controller code or if you entered some error flow (Out of
> > > > > > > memory).
> > > > > > > 
> > > > > > > You don't want to be stuck because of one bad device.
> > > > > > 
> > > > > > So the buggy driver can damage the host through various ways,
> > > > > > I'm not sure doing such workaround is worthwhile.
> > > > > do you mean device ?
> > > > 
> > > > Yes.
> > > > 
> > > > 
> > > > > sometimes you need to replace device FW and it will work.
> > > > > 
> > > > > I don't think it's a workaround. Other protocols, such as NVMe,
> > > > > solved this in the specification.
> > > > > 
> > > > > PCI config space and PCI controller are sometimes 2 different
> > > > > components and sometimes the controller is not active, although the
> > > > > device is plugged and seen by the PCI subsystem.
> > > > 
> > > > So I think we need patch to spec to see if it works first.
> > > We can't leave the driver to loop forever without allowing next "good"
> > > virtio devices to probe.
> > > 
> > > We can in parallel look for spec fixes but for now we must fix the driver.
> > I'd like to narrow the case though. the proper thing
> > would probably be for device to clear the status.
> > Provided it entered some state where it can not do it -
> > is there anything special about
> > the device state that *can* be detected?
> > If yes what is it?
> 
> We don't have anything in the spec to detect it.
> 
> Different devices can have different behavior.

I mean ... is this trying to address a practical or a theoretical
problem?

> The only thing we can count on is that the device didn't present a 0 in
> device_status once that was requested.

So for example, it's quite easy for a hypervisor to present 0 if it
wants to even if device isn't fully reset yet ... the fact it doesn't
seems to indicate it does not want the driver to be deleted yet.
I don't yet understand why overriding that from guest is a good idea.
What does guest know that hypervisor doesn't?


> 
> > 
> > > The fix can be using the mechanism introduced in this series or adding an
> > > async probing mechanism.
> > What would that be?
> 
> just schedule some work using queue_work/schedule_work.
> 
> And wait for it in the virtio_pci_remove
> 
> > > In both solutions, we can't allow looping forever.
> > Multiple minute downtime isn't much better either. I'd like a
> > much shorter timer or even no timer at all, instead
> > verifying that device is alive every X iterations.
> 
> I added an option to configure the module parameter.
> 
> I think 2-3 minutes to wait is not so bad and one can configure it for any
> value that fits.

What motivates all this work though? Figuring out the real usecase will help
us converge on a solution. Given you work for a hardware vendor I'm
guessing there's something that triggers it with a specific card
you are trying to support better.


> 
> > 
> > > > 
> > > > > 
> > > > > > Note that this driver has been used for real hardware PCI
> > > > > > devices for many years. We don't receive any report of this
> > > > > > before.
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > > 
> > > > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > > > resetting virtio devices.
> > > > > > > > > 
> > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > ---
> > > > > > > > > 
> > > > > > > > > changes from v1:
> > > > > > > > >    - update virtio_ccw.c (Cornelia)
> > > > > > > > >    - update virtio_uml.c
> > > > > > > > >    - update mlxbf-tmfifo.c
> > > > > > > > 
> > > > > > > > Note that virtio driver may call reset, so you probably
> > > > > > > > need to convert them.
> > > > > > > I'm sure I understand.
> > > > > > > 
> > > > > > > Convert to what ?
> > > > > > 
> > > > > > Convert to deal with the possible rest failure. E.g in
> > > > > > virtblk_freeze() we had:
> > > > > > 
> > > > > > static int virtblk_freeze(struct virtio_device *vdev)
> > > > > > {
> > > > > >          struct virtio_blk *vblk = vdev->priv;
> > > > > > 
> > > > > >          /* Ensure we don't receive any more interrupts */
> > > > > >          vdev->config->reset(vdev);
> > > > > > ...
> > > > > > 
> > > > > > We need fail the freeze here.
> > > > > 
> > > > > Agree.
> > > > > 
> > > > > 
> > > > > > Another example is the driver remove which is not expected to be
> > > > > > fail. A lot of virtio drivers tries to call reset there. I'm not
> > > > > > sure how hard to deal with the failure in the path of remove
> > > > > > (e.g __device_release_driver tends to ignore the return value of
> > > > > > bus->remove().)
> > > > > I think it can stay as-is and ignore the ->reset return value and
> > > > > continue free the other resources to avoid leakage.
> > > > 
> > > > The problem is that it's unclear that what kind of behaviour would the
> > > > device do, e.g can it still send interrupts?
> > > > 
> > > > That's why we need to formalize the bahviour first if necessary.
> > > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > Thanks.
> > > > > > > 
> > > > > > > > Thanks
> > > > > > > > 
> > > > > > > > 
> > > > > > > > > ---
> > > > > > > > >    arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > >    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > >    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > >    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > >    drivers/virtio/virtio.c                  | 22
> > > > > > > > > +++++++++++++++-------
> > > > > > > > >    drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > >    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > >    drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > >    drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > >    include/linux/virtio_config.h            |  5 +++--
> > > > > > > > >    10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/arch/um/drivers/virtio_uml.c
> > > > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > > > vu_set_status(struct virtio_device *vdev, u8 status)
> > > > > > > > >        vu_dev->status = status;
> > > > > > > > >    }
> > > > > > > > >    -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > >          vu_dev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > diff --git
> > > > > > > > > a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device
> > > > > > > > > *vdev,
> > > > > > > > >    }
> > > > > > > > >      /* Reset the device. Not much here for now. */
> > > > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct mlxbf_tmfifo_vdev *tm_vdev =
> > > > > > > > > mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > >          tm_vdev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      /* Read the value of a configuration field. */
> > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > @@ -191,7 +191,7 @@ static void
> > > > > > > > > rproc_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > > u8 status)
> > > > > > > > >        dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > >    }
> > > > > > > > >    -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > >        struct fw_rsc_vdev *rsc;
> > > > > > > > > @@ -200,6 +200,8 @@ static void
> > > > > > > > > rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >          rsc->status = 0;
> > > > > > > > >        dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      /* provide the vdev features as retrieved from the firmware */
> > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > @@ -732,14 +732,15 @@ static int
> > > > > > > > > virtio_ccw_find_vqs(struct virtio_device *vdev,
> > > > > > > > > unsigned nvqs,
> > > > > > > > >        return ret;
> > > > > > > > >    }
> > > > > > > > >    -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > >        struct ccw1 *ccw;
> > > > > > > > > +    int ret;
> > > > > > > > >          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > > > >        if (!ccw)
> > > > > > > > > -        return;
> > > > > > > > > +        return -ENOMEM;
> > > > > > > > >          /* Zero status bits. */
> > > > > > > > >        vcdev->dma_area->status = 0;
> > > > > > > > > @@ -749,8 +750,10 @@ static void
> > > > > > > > > virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >        ccw->flags = 0;
> > > > > > > > >        ccw->count = 0;
> > > > > > > > >        ccw->cda = 0;
> > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > >        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > +
> > > > > > > > > +    return ret;
> > > > > > > > >    }
> > > > > > > > >      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >        /* Assign a unique device index and hence name. */
> > > > > > > > >        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > > > >        if (err < 0)
> > > > > > > > > -        goto out;
> > > > > > > > > +        goto out_err;
> > > > > > > > >          dev->index = err;
> > > > > > > > >        dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >          /* We always start by resetting the device,
> > > > > > > > > in case a previous
> > > > > > > > >         * driver messed it up.  This also tests that
> > > > > > > > > code path a little. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > +    if (err)
> > > > > > > > > +        goto out_ida;
> > > > > > > > >          /* Acknowledge that we've seen the device. */
> > > > > > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > @@ -362,10 +364,14 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >         */
> > > > > > > > >        err = device_add(&dev->dev);
> > > > > > > > >        if (err)
> > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > -out:
> > > > > > > > > -    if (err)
> > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > +        goto out_ida;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > > +
> > > > > > > > > +out_ida:
> > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > +out_err:
> > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > >        return err;
> > > > > > > > >    }
> > > > > > > > >    EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct
> > > > > > > > > virtio_device *dev)
> > > > > > > > >          /* We always start by resetting the device,
> > > > > > > > > in case a previous
> > > > > > > > >         * driver messed it up. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > +    if (ret)
> > > > > > > > > +        goto err;
> > > > > > > > >          /* Acknowledge that we've seen the device. */
> > > > > > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > diff --git a/drivers/virtio/virtio_mmio.c
> > > > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > > > vm_set_status(struct virtio_device *vdev, u8 status)
> > > > > > > > >        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > >    }
> > > > > > > > >    -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_mmio_device *vm_dev =
> > > > > > > > > to_virtio_mmio_device(vdev);
> > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > >        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >    }
> > > > > > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >        /* 0 status means a reset. */
> > > > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >        vp_synchronize_vectors(vdev);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static u16 vp_config_vector(struct
> > > > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >        vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > >    }
> > > > > > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >            msleep(1);
> > > > > > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >        vp_synchronize_vectors(vdev);
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static u16 vp_config_vector(struct
> > > > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > @@ -97,11 +97,13 @@ static void
> > > > > > > > > virtio_vdpa_set_status(struct virtio_device *vdev,
> > > > > > > > > u8 status)
> > > > > > > > >        return ops->set_status(vdpa, status);
> > > > > > > > >    }
> > > > > > > > >    -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > >          vdpa_reset(vdpa);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > >     *    status: the new status byte
> > > > > > > > >     * @reset: reset the device
> > > > > > > > >     *    vdev: the virtio device
> > > > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > > > + *    Upon success, status and feature negotiation
> > > > > > > > > must be done again
> > > > > > > > >     *    Device must not be reset from its vq/config callbacks, or in
> > > > > > > > >     *    parallel with being added/removed.
> > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > >     * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > >     *    vdev: the virtio_device
> > > > > > > > >     *    nvqs: the number of virtqueues to find
> > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > >        u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > >        u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > >        void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > >        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > >                struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > > > >                const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Mon, Apr 12, 2021 at 03:17:02PM +0300, Max Gurtovoy wrote:
> 
> On 4/12/2021 12:07 PM, Michael S. Tsirkin wrote:
> > On Sun, Apr 11, 2021 at 12:50:22PM +0300, Max Gurtovoy wrote:
> > > On 4/9/2021 8:22 AM, Jason Wang wrote:
> > > > 在 2021/4/8 下午10:24, Max Gurtovoy 写道:
> > > > > On 4/8/2021 4:14 PM, Jason Wang wrote:
> > > > > > 在 2021/4/8 下午5:56, Max Gurtovoy 写道:
> > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > The reset device operation, usually is an operation
> > > > > > > > > that might fail from
> > > > > > > > > various reasons. For example, the controller might
> > > > > > > > > be in a bad state and
> > > > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > > > devices always succeed in reset operation but this
> > > > > > > > > is not the case for
> > > > > > > > > HW based virtio devices.
> > > > > > > > 
> > > > > > > > I would like to know under what condition that the reset
> > > > > > > > operation may fail (except for the case of a bugg
> > > > > > > > guest).
> > > > > > > The controller might not be ready or stuck. This is a real
> > > > > > > use case for many PCI devices.
> > > > > > > 
> > > > > > > For real devices the FW might be in a bad state and it can
> > > > > > > happen also for paravirt device if you have a bug in the
> > > > > > > controller code or if you entered some error flow (Out of
> > > > > > > memory).
> > > > > > > 
> > > > > > > You don't want to be stuck because of one bad device.
> > > > > > 
> > > > > > So the buggy driver can damage the host through various ways,
> > > > > > I'm not sure doing such workaround is worthwhile.
> > > > > do you mean device ?
> > > > 
> > > > Yes.
> > > > 
> > > > 
> > > > > sometimes you need to replace device FW and it will work.
> > > > > 
> > > > > I don't think it's a workaround. Other protocols, such as NVMe,
> > > > > solved this in the specification.
> > > > > 
> > > > > PCI config space and PCI controller are sometimes 2 different
> > > > > components and sometimes the controller is not active, although the
> > > > > device is plugged and seen by the PCI subsystem.
> > > > 
> > > > So I think we need patch to spec to see if it works first.
> > > We can't leave the driver to loop forever without allowing next "good"
> > > virtio devices to probe.
> > > 
> > > We can in parallel look for spec fixes but for now we must fix the driver.
> > I'd like to narrow the case though. the proper thing
> > would probably be for device to clear the status.
> > Provided it entered some state where it can not do it -
> > is there anything special about
> > the device state that *can* be detected?
> > If yes what is it?
> 
> We don't have anything in the spec to detect it.
> 
> Different devices can have different behavior.

I mean ... is this trying to address a practical or a theoretical
problem?

> The only thing we can count on is that the device didn't present a 0 in
> device_status once that was requested.

So for example, it's quite easy for a hypervisor to present 0 if it
wants to even if device isn't fully reset yet ... the fact it doesn't
seems to indicate it does not want the driver to be deleted yet.
I don't yet understand why overriding that from guest is a good idea.
What does guest know that hypervisor doesn't?


> 
> > 
> > > The fix can be using the mechanism introduced in this series or adding an
> > > async probing mechanism.
> > What would that be?
> 
> just schedule some work using queue_work/schedule_work.
> 
> And wait for it in the virtio_pci_remove
> 
> > > In both solutions, we can't allow looping forever.
> > Multiple minute downtime isn't much better either. I'd like a
> > much shorter timer or even no timer at all, instead
> > verifying that device is alive every X iterations.
> 
> I added an option to configure the module parameter.
> 
> I think 2-3 minutes to wait is not so bad and one can configure it for any
> value that fits.

What motivates all this work though? Figuring out the real usecase will help
us converge on a solution. Given you work for a hardware vendor I'm
guessing there's something that triggers it with a specific card
you are trying to support better.


> 
> > 
> > > > 
> > > > > 
> > > > > > Note that this driver has been used for real hardware PCI
> > > > > > devices for many years. We don't receive any report of this
> > > > > > before.
> > > > > > 
> > > > > > 
> > > > > > > 
> > > > > > > > 
> > > > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > > > resetting virtio devices.
> > > > > > > > > 
> > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > ---
> > > > > > > > > 
> > > > > > > > > changes from v1:
> > > > > > > > >    - update virtio_ccw.c (Cornelia)
> > > > > > > > >    - update virtio_uml.c
> > > > > > > > >    - update mlxbf-tmfifo.c
> > > > > > > > 
> > > > > > > > Note that virtio driver may call reset, so you probably
> > > > > > > > need to convert them.
> > > > > > > I'm sure I understand.
> > > > > > > 
> > > > > > > Convert to what ?
> > > > > > 
> > > > > > Convert to deal with the possible rest failure. E.g in
> > > > > > virtblk_freeze() we had:
> > > > > > 
> > > > > > static int virtblk_freeze(struct virtio_device *vdev)
> > > > > > {
> > > > > >          struct virtio_blk *vblk = vdev->priv;
> > > > > > 
> > > > > >          /* Ensure we don't receive any more interrupts */
> > > > > >          vdev->config->reset(vdev);
> > > > > > ...
> > > > > > 
> > > > > > We need fail the freeze here.
> > > > > 
> > > > > Agree.
> > > > > 
> > > > > 
> > > > > > Another example is the driver remove which is not expected to be
> > > > > > fail. A lot of virtio drivers tries to call reset there. I'm not
> > > > > > sure how hard to deal with the failure in the path of remove
> > > > > > (e.g __device_release_driver tends to ignore the return value of
> > > > > > bus->remove().)
> > > > > I think it can stay as-is and ignore the ->reset return value and
> > > > > continue free the other resources to avoid leakage.
> > > > 
> > > > The problem is that it's unclear that what kind of behaviour would the
> > > > device do, e.g can it still send interrupts?
> > > > 
> > > > That's why we need to formalize the bahviour first if necessary.
> > > > 
> > > > Thanks
> > > > 
> > > > 
> > > > > 
> > > > > > Thanks
> > > > > > 
> > > > > > 
> > > > > > > Thanks.
> > > > > > > 
> > > > > > > > Thanks
> > > > > > > > 
> > > > > > > > 
> > > > > > > > > ---
> > > > > > > > >    arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > >    drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > >    drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > >    drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > >    drivers/virtio/virtio.c                  | 22
> > > > > > > > > +++++++++++++++-------
> > > > > > > > >    drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > >    drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > >    drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > >    drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > >    include/linux/virtio_config.h            |  5 +++--
> > > > > > > > >    10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/arch/um/drivers/virtio_uml.c
> > > > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > > > vu_set_status(struct virtio_device *vdev, u8 status)
> > > > > > > > >        vu_dev->status = status;
> > > > > > > > >    }
> > > > > > > > >    -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > >          vu_dev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > diff --git
> > > > > > > > > a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device
> > > > > > > > > *vdev,
> > > > > > > > >    }
> > > > > > > > >      /* Reset the device. Not much here for now. */
> > > > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct mlxbf_tmfifo_vdev *tm_vdev =
> > > > > > > > > mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > >          tm_vdev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      /* Read the value of a configuration field. */
> > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > @@ -191,7 +191,7 @@ static void
> > > > > > > > > rproc_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > > u8 status)
> > > > > > > > >        dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > >    }
> > > > > > > > >    -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > >        struct fw_rsc_vdev *rsc;
> > > > > > > > > @@ -200,6 +200,8 @@ static void
> > > > > > > > > rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >          rsc->status = 0;
> > > > > > > > >        dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      /* provide the vdev features as retrieved from the firmware */
> > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > @@ -732,14 +732,15 @@ static int
> > > > > > > > > virtio_ccw_find_vqs(struct virtio_device *vdev,
> > > > > > > > > unsigned nvqs,
> > > > > > > > >        return ret;
> > > > > > > > >    }
> > > > > > > > >    -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > >        struct ccw1 *ccw;
> > > > > > > > > +    int ret;
> > > > > > > > >          ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > > > >        if (!ccw)
> > > > > > > > > -        return;
> > > > > > > > > +        return -ENOMEM;
> > > > > > > > >          /* Zero status bits. */
> > > > > > > > >        vcdev->dma_area->status = 0;
> > > > > > > > > @@ -749,8 +750,10 @@ static void
> > > > > > > > > virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >        ccw->flags = 0;
> > > > > > > > >        ccw->count = 0;
> > > > > > > > >        ccw->cda = 0;
> > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > >        ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > +
> > > > > > > > > +    return ret;
> > > > > > > > >    }
> > > > > > > > >      static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >        /* Assign a unique device index and hence name. */
> > > > > > > > >        err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > > > >        if (err < 0)
> > > > > > > > > -        goto out;
> > > > > > > > > +        goto out_err;
> > > > > > > > >          dev->index = err;
> > > > > > > > >        dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >          /* We always start by resetting the device,
> > > > > > > > > in case a previous
> > > > > > > > >         * driver messed it up.  This also tests that
> > > > > > > > > code path a little. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > +    if (err)
> > > > > > > > > +        goto out_ida;
> > > > > > > > >          /* Acknowledge that we've seen the device. */
> > > > > > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > @@ -362,10 +364,14 @@ int
> > > > > > > > > register_virtio_device(struct virtio_device *dev)
> > > > > > > > >         */
> > > > > > > > >        err = device_add(&dev->dev);
> > > > > > > > >        if (err)
> > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > -out:
> > > > > > > > > -    if (err)
> > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > +        goto out_ida;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > > +
> > > > > > > > > +out_ida:
> > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > +out_err:
> > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > >        return err;
> > > > > > > > >    }
> > > > > > > > >    EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct
> > > > > > > > > virtio_device *dev)
> > > > > > > > >          /* We always start by resetting the device,
> > > > > > > > > in case a previous
> > > > > > > > >         * driver messed it up. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > +    if (ret)
> > > > > > > > > +        goto err;
> > > > > > > > >          /* Acknowledge that we've seen the device. */
> > > > > > > > >        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > diff --git a/drivers/virtio/virtio_mmio.c
> > > > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > > > vm_set_status(struct virtio_device *vdev, u8 status)
> > > > > > > > >        writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > >    }
> > > > > > > > >    -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_mmio_device *vm_dev =
> > > > > > > > > to_virtio_mmio_device(vdev);
> > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > >        writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >    }
> > > > > > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >        /* 0 status means a reset. */
> > > > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >        vp_synchronize_vectors(vdev);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static u16 vp_config_vector(struct
> > > > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >        vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > >    }
> > > > > > > > >    -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >            msleep(1);
> > > > > > > > >        /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >        vp_synchronize_vectors(vdev);
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static u16 vp_config_vector(struct
> > > > > > > > > virtio_pci_device *vp_dev, u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > @@ -97,11 +97,13 @@ static void
> > > > > > > > > virtio_vdpa_set_status(struct virtio_device *vdev,
> > > > > > > > > u8 status)
> > > > > > > > >        return ops->set_status(vdpa, status);
> > > > > > > > >    }
> > > > > > > > >    -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > >    {
> > > > > > > > >        struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > >          vdpa_reset(vdpa);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >    }
> > > > > > > > >      static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > >     *    status: the new status byte
> > > > > > > > >     * @reset: reset the device
> > > > > > > > >     *    vdev: the virtio device
> > > > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > > > + *    Upon success, status and feature negotiation
> > > > > > > > > must be done again
> > > > > > > > >     *    Device must not be reset from its vq/config callbacks, or in
> > > > > > > > >     *    parallel with being added/removed.
> > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > >     * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > >     *    vdev: the virtio_device
> > > > > > > > >     *    nvqs: the number of virtqueues to find
> > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > >        u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > >        u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > >        void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > >        int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > >                struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > > > >                const char * const names[], const bool *ctx,

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
> On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
>> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
>>> On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
>>>> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
>>>>> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>>>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>>>> The reset device operation, usually is an operation that might fail from
>>>>>>>> various reasons. For example, the controller might be in a bad state and
>>>>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>>>>> devices always succeed in reset operation but this is not the case for
>>>>>>>> HW based virtio devices.
>>>>>>> I would like to know under what condition that the reset operation may
>>>>>>> fail (except for the case of a bugg guest).
>>>>>> The controller might not be ready or stuck. This is a real use case for many
>>>>>> PCI devices.
>>>>>>
>>>>>> For real devices the FW might be in a bad state and it can happen also for
>>>>>> paravirt device if you have a bug in the controller code or if you entered
>>>>>> some error flow (Out of memory).
>>>>>>
>>>>>> You don't want to be stuck because of one bad device.
>>>>> OK so maybe we can do more to detect the bad device.
>>>>> Won't we get all 1's on a read in this case?
>>>> No. how can we guarantee it ?
>>>>
>>> Well this is what you tend to get if e.g. you disable device memory.
>>>
>>> Anyway, you know about hardware, I don't ... It's not returning 0 after
>>> reset as it should ... what does it return? Hopefully not random noise -
>>> I don't think it's very practical to write a driver for a device that
>>> starts doing that at random times ...
>> The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
>>
>> For paravirt devices, think of a situation that you can't allocate some
>> internal buffers (malloc failed) and you want to wait for few seconds until
>> the system memory will free some pages.
>>
>> So you may return NEEDS_RESET that indicates some error state of the device.
>> Once the system memory freed by other application for example, your internal
>> virtio device malloc succeeded and you may return 0.
>>
>> In this case, you don't want to stall the other virtio devices to probe
>> (they might be real HW devices that driven by the same driver), right ?
> So the device is very busy then? Not sure it's smart to just assume
> it's safe to free all memory allocated for it then ...
>
> I guess the lesson is don't make device reset depend on malloc
> of some memory?

The device is not ready yet. And the malloc is just one example I gave 
you to emphasize the case.

Another example can be a bad FW installed.

The host/guest driver is trying to enable the device but the device is 
not ready. This is the real life use case and I gave many examples for 
reasons for device not to be ready. For paravirt and HW devices.

Endless loop and stalling next devices probe is not the way to go. PCI 
drivers can't allow this to happen.

Think of a situation that host it's booting from virtio-blk device but 
it has another virtio-net device that has bad FW or other internal error 
(it doesn't matter what it is for the example) and the virtio pci driver 
is probing virtio-net device first.

The host will never boot successfully. This is fatal.

Error flows are critical when working with real PCI HW and I understand 
that in paravirt devices with strong hypervisor you will be ok in 99% of 
the time but you need to be aware also for bugs and error flows in both 
paravirt and HW world.

So first we need to handle this endless loop (with this patch set or 
with async probing mechanism) and later we should update the specification.

The virtio world now is not only guest and paravirt devices. Bare metal 
hosts start using virtio devices and drivers more and more.

>
>
>>>>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>>>>> resetting virtio devices.
>>>>>>>>
>>>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>>>> ---
>>>>>>>>
>>>>>>>> changes from v1:
>>>>>>>>      - update virtio_ccw.c (Cornelia)
>>>>>>>>      - update virtio_uml.c
>>>>>>>>      - update mlxbf-tmfifo.c
>>>>>>> Note that virtio driver may call reset, so you probably need to convert
>>>>>>> them.
>>>>>> I'm sure I understand.
>>>>>>
>>>>>> Convert to what ?
>>>>>>
>>>>>> Thanks.
>>>>>>
>>>>>>> Thanks
>>>>>>>
>>>>>>>
>>>>>>>> ---
>>>>>>>>      arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>>>      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>>>      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>>>      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>>>      drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>>>>>>      drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>>>      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>>>      drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>>>      drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>>>      include/linux/virtio_config.h            |  5 +++--
>>>>>>>>      10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
>>>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
>>>>>>>> *vdev, u8 status)
>>>>>>>>          vu_dev->status = status;
>>>>>>>>      }
>>>>>>>>      -static void vu_reset(struct virtio_device *vdev)
>>>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>>>>            vu_dev->status = 0;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        static void vu_del_vq(struct virtqueue *vq)
>>>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>>>      }
>>>>>>>>        /* Reset the device. Not much here for now. */
>>>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>>>>>            tm_vdev->status = 0;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        /* Read the value of a configuration field. */
>>>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>          dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>>>      }
>>>>>>>>      -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>>>          struct fw_rsc_vdev *rsc;
>>>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>>>>>> virtio_device *vdev)
>>>>>>>>            rsc->status = 0;
>>>>>>>>          dev_dbg(&vdev->dev, "reset !\n");
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        /* provide the vdev features as retrieved from the firmware */
>>>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>>>          return ret;
>>>>>>>>      }
>>>>>>>>      -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>>>          struct ccw1 *ccw;
>>>>>>>> +    int ret;
>>>>>>>>            ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>>>>          if (!ccw)
>>>>>>>> -        return;
>>>>>>>> +        return -ENOMEM;
>>>>>>>>            /* Zero status bits. */
>>>>>>>>          vcdev->dma_area->status = 0;
>>>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>>>>>> virtio_device *vdev)
>>>>>>>>          ccw->flags = 0;
>>>>>>>>          ccw->count = 0;
>>>>>>>>          ccw->cda = 0;
>>>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>>>> +
>>>>>>>> +    return ret;
>>>>>>>>      }
>>>>>>>>        static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>>>> --- a/drivers/virtio/virtio.c
>>>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
>>>>>>>> *dev)
>>>>>>>>          /* Assign a unique device index and hence name. */
>>>>>>>>          err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>>>>          if (err < 0)
>>>>>>>> -        goto out;
>>>>>>>> +        goto out_err;
>>>>>>>>            dev->index = err;
>>>>>>>>          dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
>>>>>>>> *dev)
>>>>>>>>            /* We always start by resetting the device, in case a previous
>>>>>>>>           * driver messed it up.  This also tests that code path a
>>>>>>>> little. */
>>>>>>>> -    dev->config->reset(dev);
>>>>>>>> +    err = dev->config->reset(dev);
>>>>>>>> +    if (err)
>>>>>>>> +        goto out_ida;
>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>>>>>> virtio_device *dev)
>>>>>>>>           */
>>>>>>>>          err = device_add(&dev->dev);
>>>>>>>>          if (err)
>>>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>> -out:
>>>>>>>> -    if (err)
>>>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>> +        goto out_ida;
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>> +
>>>>>>>> +out_ida:
>>>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>> +out_err:
>>>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>          return err;
>>>>>>>>      }
>>>>>>>>      EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>>>>>>>>            /* We always start by resetting the device, in case a previous
>>>>>>>>           * driver messed it up. */
>>>>>>>> -    dev->config->reset(dev);
>>>>>>>> +    ret = dev->config->reset(dev);
>>>>>>>> +    if (ret)
>>>>>>>> +        goto err;
>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
>>>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
>>>>>>>> *vdev, u8 status)
>>>>>>>>          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>      }
>>>>>>>>      -static void vm_reset(struct virtio_device *vdev)
>>>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>>>>>>            /* 0 status means a reset. */
>>>>>>>>          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>> *vdev, u8 status)
>>>>>>>>          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>      }
>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>          /* 0 status means a reset. */
>>>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>>>> u16 vector)
>>>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>> *vdev, u8 status)
>>>>>>>>          vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>>>      }
>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>              msleep(1);
>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>>>> u16 vector)
>>>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
>>>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>          return ops->set_status(vdpa, status);
>>>>>>>>      }
>>>>>>>>      -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>      {
>>>>>>>>          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>>>            vdpa_reset(vdpa);
>>>>>>>> +
>>>>>>>> +    return 0;
>>>>>>>>      }
>>>>>>>>        static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>>>> b/include/linux/virtio_config.h
>>>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>>>> --- a/include/linux/virtio_config.h
>>>>>>>> +++ b/include/linux/virtio_config.h
>>>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>>>       *    status: the new status byte
>>>>>>>>       * @reset: reset the device
>>>>>>>>       *    vdev: the virtio device
>>>>>>>> - *    After this, status and feature negotiation must be done again
>>>>>>>> + *    Upon success, status and feature negotiation must be done again
>>>>>>>>       *    Device must not be reset from its vq/config callbacks, or in
>>>>>>>>       *    parallel with being added/removed.
>>>>>>>> + *    Returns 0 on success or error status.
>>>>>>>>       * @find_vqs: find virtqueues and instantiate them.
>>>>>>>>       *    vdev: the virtio_device
>>>>>>>>       *    nvqs: the number of virtqueues to find
>>>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>>>          u32 (*generation)(struct virtio_device *vdev);
>>>>>>>>          u8 (*get_status)(struct virtio_device *vdev);
>>>>>>>>          void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>>>          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>>>                  struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>>>>                  const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/13 上午6:53, Max Gurtovoy 写道:
>
> On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
>> On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
>>> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
>>>> On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
>>>>> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
>>>>>> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>>>>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>>>>> The reset device operation, usually is an operation that might 
>>>>>>>>> fail from
>>>>>>>>> various reasons. For example, the controller might be in a bad 
>>>>>>>>> state and
>>>>>>>>> can't answer to any request. Usually, the paravirt SW based 
>>>>>>>>> virtio
>>>>>>>>> devices always succeed in reset operation but this is not the 
>>>>>>>>> case for
>>>>>>>>> HW based virtio devices.
>>>>>>>> I would like to know under what condition that the reset 
>>>>>>>> operation may
>>>>>>>> fail (except for the case of a bugg guest).
>>>>>>> The controller might not be ready or stuck. This is a real use 
>>>>>>> case for many
>>>>>>> PCI devices.
>>>>>>>
>>>>>>> For real devices the FW might be in a bad state and it can 
>>>>>>> happen also for
>>>>>>> paravirt device if you have a bug in the controller code or if 
>>>>>>> you entered
>>>>>>> some error flow (Out of memory).
>>>>>>>
>>>>>>> You don't want to be stuck because of one bad device.
>>>>>> OK so maybe we can do more to detect the bad device.
>>>>>> Won't we get all 1's on a read in this case?
>>>>> No. how can we guarantee it ?
>>>>>
>>>> Well this is what you tend to get if e.g. you disable device memory.
>>>>
>>>> Anyway, you know about hardware, I don't ... It's not returning 0 
>>>> after
>>>> reset as it should ... what does it return? Hopefully not random 
>>>> noise -
>>>> I don't think it's very practical to write a driver for a device that
>>>> starts doing that at random times ...
>>> The device may return 0x40 (NEEDS_RESET). It doesn't have to return 
>>> all 1's.
>>>
>>> For paravirt devices, think of a situation that you can't allocate some
>>> internal buffers (malloc failed) and you want to wait for few 
>>> seconds until
>>> the system memory will free some pages.
>>>
>>> So you may return NEEDS_RESET that indicates some error state of the 
>>> device.
>>> Once the system memory freed by other application for example, your 
>>> internal
>>> virtio device malloc succeeded and you may return 0.
>>>
>>> In this case, you don't want to stall the other virtio devices to probe
>>> (they might be real HW devices that driven by the same driver), right ?
>> So the device is very busy then? Not sure it's smart to just assume
>> it's safe to free all memory allocated for it then ...
>>
>> I guess the lesson is don't make device reset depend on malloc
>> of some memory?
>
> The device is not ready yet. And the malloc is just one example I gave 
> you to emphasize the case.
>
> Another example can be a bad FW installed.
>
> The host/guest driver is trying to enable the device but the device is 
> not ready. This is the real life use case and I gave many examples for 
> reasons for device not to be ready. For paravirt and HW devices.
>
> Endless loop and stalling next devices probe is not the way to go. PCI 
> drivers can't allow this to happen.
>
> Think of a situation that host it's booting from virtio-blk device but 
> it has another virtio-net device that has bad FW or other internal 
> error (it doesn't matter what it is for the example) and the virtio 
> pci driver is probing virtio-net device first.
>
> The host will never boot successfully. This is fatal.


The issue is that there's not clear definition in the spec about how 
device is expected to behave when it is being reset. E.g can the device 
still modify virtqueue in this case? If yes, stall the host is much more 
safe than simply ignore it.

In another way, I wonder whether we can do transport level reset (e.g 
PCI) in this case.


>
> Error flows are critical when working with real PCI HW and I 
> understand that in paravirt devices with strong hypervisor you will be 
> ok in 99% of the time but you need to be aware also for bugs and error 
> flows in both paravirt and HW world.
>
> So first we need to handle this endless loop (with this patch set or 
> with async probing mechanism) and later we should update the 
> specification.
>
> The virtio world now is not only guest and paravirt devices. Bare 
> metal hosts start using virtio devices and drivers more and more.


So as mentioned in previous reply, this driver has been used for real 
virtio hardware for many years. One well known example is the Ali Cloud 
ECS bare metal instance.

Thanks


>
>>
>>
>>>>>>>>> This commit is also a preparation for adding a timeout 
>>>>>>>>> mechanism for
>>>>>>>>> resetting virtio devices.
>>>>>>>>>
>>>>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>>>>> ---
>>>>>>>>>
>>>>>>>>> changes from v1:
>>>>>>>>>      - update virtio_ccw.c (Cornelia)
>>>>>>>>>      - update virtio_uml.c
>>>>>>>>>      - update mlxbf-tmfifo.c
>>>>>>>> Note that virtio driver may call reset, so you probably need to 
>>>>>>>> convert
>>>>>>>> them.
>>>>>>> I'm sure I understand.
>>>>>>>
>>>>>>> Convert to what ?
>>>>>>>
>>>>>>> Thanks.
>>>>>>>
>>>>>>>> Thanks
>>>>>>>>
>>>>>>>>
>>>>>>>>> ---
>>>>>>>>>      arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>>>>      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>>>>      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>>>>      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>>>>      drivers/virtio/virtio.c                  | 22 
>>>>>>>>> +++++++++++++++-------
>>>>>>>>>      drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>>>>      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>>>>      drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>>>>      drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>>>>      include/linux/virtio_config.h            |  5 +++--
>>>>>>>>>      10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>>>>>>> b/arch/um/drivers/virtio_uml.c
>>>>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          vu_dev->status = status;
>>>>>>>>>      }
>>>>>>>>>      -static void vu_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_uml_device *vu_dev = 
>>>>>>>>> to_virtio_uml_device(vdev);
>>>>>>>>>            vu_dev->status = 0;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static void vu_del_vq(struct virtqueue *vq)
>>>>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>>>>      }
>>>>>>>>>        /* Reset the device. Not much here for now. */
>>>>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct mlxbf_tmfifo_vdev *tm_vdev = 
>>>>>>>>> mlxbf_vdev_to_tmfifo(vdev);
>>>>>>>>>            tm_vdev->status = 0;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        /* Read the value of a configuration field. */
>>>>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>          dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>>>>      }
>>>>>>>>>      -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>>>>          struct fw_rsc_vdev *rsc;
>>>>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>>>>>>> virtio_device *vdev)
>>>>>>>>>            rsc->status = 0;
>>>>>>>>>          dev_dbg(&vdev->dev, "reset !\n");
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        /* provide the vdev features as retrieved from the 
>>>>>>>>> firmware */
>>>>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>>>>          return ret;
>>>>>>>>>      }
>>>>>>>>>      -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>>>>          struct ccw1 *ccw;
>>>>>>>>> +    int ret;
>>>>>>>>>            ccw = ccw_device_dma_zalloc(vcdev->cdev, 
>>>>>>>>> sizeof(*ccw));
>>>>>>>>>          if (!ccw)
>>>>>>>>> -        return;
>>>>>>>>> +        return -ENOMEM;
>>>>>>>>>            /* Zero status bits. */
>>>>>>>>>          vcdev->dma_area->status = 0;
>>>>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>>>>>>> virtio_device *vdev)
>>>>>>>>>          ccw->flags = 0;
>>>>>>>>>          ccw->count = 0;
>>>>>>>>>          ccw->cda = 0;
>>>>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>>          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>>>>> +
>>>>>>>>> +    return ret;
>>>>>>>>>      }
>>>>>>>>>        static u64 virtio_ccw_get_features(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>>>>> --- a/drivers/virtio/virtio.c
>>>>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *dev)
>>>>>>>>>          /* Assign a unique device index and hence name. */
>>>>>>>>>          err = ida_simple_get(&virtio_index_ida, 0, 0, 
>>>>>>>>> GFP_KERNEL);
>>>>>>>>>          if (err < 0)
>>>>>>>>> -        goto out;
>>>>>>>>> +        goto out_err;
>>>>>>>>>            dev->index = err;
>>>>>>>>>          dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *dev)
>>>>>>>>>            /* We always start by resetting the device, in case 
>>>>>>>>> a previous
>>>>>>>>>           * driver messed it up.  This also tests that code 
>>>>>>>>> path a
>>>>>>>>> little. */
>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>> +    err = dev->config->reset(dev);
>>>>>>>>> +    if (err)
>>>>>>>>> +        goto out_ida;
>>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>>>>>>> virtio_device *dev)
>>>>>>>>>           */
>>>>>>>>>          err = device_add(&dev->dev);
>>>>>>>>>          if (err)
>>>>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>> -out:
>>>>>>>>> -    if (err)
>>>>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>> +        goto out_ida;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>> +
>>>>>>>>> +out_ida:
>>>>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>> +out_err:
>>>>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>>          return err;
>>>>>>>>>      }
>>>>>>>>>      EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct 
>>>>>>>>> virtio_device *dev)
>>>>>>>>>            /* We always start by resetting the device, in case 
>>>>>>>>> a previous
>>>>>>>>>           * driver messed it up. */
>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>> +    ret = dev->config->reset(dev);
>>>>>>>>> +    if (ret)
>>>>>>>>> +        goto err;
>>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>>>>>>> b/drivers/virtio/virtio_mmio.c
>>>>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>>      }
>>>>>>>>>      -static void vm_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_mmio_device *vm_dev = 
>>>>>>>>> to_virtio_mmio_device(vdev);
>>>>>>>>>            /* 0 status means a reset. */
>>>>>>>>>          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>      }
>>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>          /* 0 status means a reset. */
>>>>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>>          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device 
>>>>>>>>> *vp_dev,
>>>>>>>>> u16 vector)
>>>>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>> vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>>>>      }
>>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>>              msleep(1);
>>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device 
>>>>>>>>> *vp_dev,
>>>>>>>>> u16 vector)
>>>>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>>>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>          return ops->set_status(vdpa, status);
>>>>>>>>>      }
>>>>>>>>>      -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>>>>            vdpa_reset(vdpa);
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>>>>> b/include/linux/virtio_config.h
>>>>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>>>>> --- a/include/linux/virtio_config.h
>>>>>>>>> +++ b/include/linux/virtio_config.h
>>>>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>>>>       *    status: the new status byte
>>>>>>>>>       * @reset: reset the device
>>>>>>>>>       *    vdev: the virtio device
>>>>>>>>> - *    After this, status and feature negotiation must be done 
>>>>>>>>> again
>>>>>>>>> + *    Upon success, status and feature negotiation must be 
>>>>>>>>> done again
>>>>>>>>>       *    Device must not be reset from its vq/config 
>>>>>>>>> callbacks, or in
>>>>>>>>>       *    parallel with being added/removed.
>>>>>>>>> + *    Returns 0 on success or error status.
>>>>>>>>>       * @find_vqs: find virtqueues and instantiate them.
>>>>>>>>>       *    vdev: the virtio_device
>>>>>>>>>       *    nvqs: the number of virtqueues to find
>>>>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>>>>          u32 (*generation)(struct virtio_device *vdev);
>>>>>>>>>          u8 (*get_status)(struct virtio_device *vdev);
>>>>>>>>>          void (*set_status)(struct virtio_device *vdev, u8 
>>>>>>>>> status);
>>>>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>>>>          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>>>>                  struct virtqueue *vqs[], vq_callback_t 
>>>>>>>>> *callbacks[],
>>>>>>>>>                  const char * const names[], const bool *ctx,
>

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Jason Wang]

在 2021/4/13 上午6:53, Max Gurtovoy 写道:
>
> On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
>> On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
>>> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
>>>> On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
>>>>> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
>>>>>> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>>>>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>>>>> The reset device operation, usually is an operation that might 
>>>>>>>>> fail from
>>>>>>>>> various reasons. For example, the controller might be in a bad 
>>>>>>>>> state and
>>>>>>>>> can't answer to any request. Usually, the paravirt SW based 
>>>>>>>>> virtio
>>>>>>>>> devices always succeed in reset operation but this is not the 
>>>>>>>>> case for
>>>>>>>>> HW based virtio devices.
>>>>>>>> I would like to know under what condition that the reset 
>>>>>>>> operation may
>>>>>>>> fail (except for the case of a bugg guest).
>>>>>>> The controller might not be ready or stuck. This is a real use 
>>>>>>> case for many
>>>>>>> PCI devices.
>>>>>>>
>>>>>>> For real devices the FW might be in a bad state and it can 
>>>>>>> happen also for
>>>>>>> paravirt device if you have a bug in the controller code or if 
>>>>>>> you entered
>>>>>>> some error flow (Out of memory).
>>>>>>>
>>>>>>> You don't want to be stuck because of one bad device.
>>>>>> OK so maybe we can do more to detect the bad device.
>>>>>> Won't we get all 1's on a read in this case?
>>>>> No. how can we guarantee it ?
>>>>>
>>>> Well this is what you tend to get if e.g. you disable device memory.
>>>>
>>>> Anyway, you know about hardware, I don't ... It's not returning 0 
>>>> after
>>>> reset as it should ... what does it return? Hopefully not random 
>>>> noise -
>>>> I don't think it's very practical to write a driver for a device that
>>>> starts doing that at random times ...
>>> The device may return 0x40 (NEEDS_RESET). It doesn't have to return 
>>> all 1's.
>>>
>>> For paravirt devices, think of a situation that you can't allocate some
>>> internal buffers (malloc failed) and you want to wait for few 
>>> seconds until
>>> the system memory will free some pages.
>>>
>>> So you may return NEEDS_RESET that indicates some error state of the 
>>> device.
>>> Once the system memory freed by other application for example, your 
>>> internal
>>> virtio device malloc succeeded and you may return 0.
>>>
>>> In this case, you don't want to stall the other virtio devices to probe
>>> (they might be real HW devices that driven by the same driver), right ?
>> So the device is very busy then? Not sure it's smart to just assume
>> it's safe to free all memory allocated for it then ...
>>
>> I guess the lesson is don't make device reset depend on malloc
>> of some memory?
>
> The device is not ready yet. And the malloc is just one example I gave 
> you to emphasize the case.
>
> Another example can be a bad FW installed.
>
> The host/guest driver is trying to enable the device but the device is 
> not ready. This is the real life use case and I gave many examples for 
> reasons for device not to be ready. For paravirt and HW devices.
>
> Endless loop and stalling next devices probe is not the way to go. PCI 
> drivers can't allow this to happen.
>
> Think of a situation that host it's booting from virtio-blk device but 
> it has another virtio-net device that has bad FW or other internal 
> error (it doesn't matter what it is for the example) and the virtio 
> pci driver is probing virtio-net device first.
>
> The host will never boot successfully. This is fatal.


The issue is that there's not clear definition in the spec about how 
device is expected to behave when it is being reset. E.g can the device 
still modify virtqueue in this case? If yes, stall the host is much more 
safe than simply ignore it.

In another way, I wonder whether we can do transport level reset (e.g 
PCI) in this case.


>
> Error flows are critical when working with real PCI HW and I 
> understand that in paravirt devices with strong hypervisor you will be 
> ok in 99% of the time but you need to be aware also for bugs and error 
> flows in both paravirt and HW world.
>
> So first we need to handle this endless loop (with this patch set or 
> with async probing mechanism) and later we should update the 
> specification.
>
> The virtio world now is not only guest and paravirt devices. Bare 
> metal hosts start using virtio devices and drivers more and more.


So as mentioned in previous reply, this driver has been used for real 
virtio hardware for many years. One well known example is the Ali Cloud 
ECS bare metal instance.

Thanks


>
>>
>>
>>>>>>>>> This commit is also a preparation for adding a timeout 
>>>>>>>>> mechanism for
>>>>>>>>> resetting virtio devices.
>>>>>>>>>
>>>>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>>>>> ---
>>>>>>>>>
>>>>>>>>> changes from v1:
>>>>>>>>>      - update virtio_ccw.c (Cornelia)
>>>>>>>>>      - update virtio_uml.c
>>>>>>>>>      - update mlxbf-tmfifo.c
>>>>>>>> Note that virtio driver may call reset, so you probably need to 
>>>>>>>> convert
>>>>>>>> them.
>>>>>>> I'm sure I understand.
>>>>>>>
>>>>>>> Convert to what ?
>>>>>>>
>>>>>>> Thanks.
>>>>>>>
>>>>>>>> Thanks
>>>>>>>>
>>>>>>>>
>>>>>>>>> ---
>>>>>>>>>      arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>>>>      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>>>>      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>>>>      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>>>>      drivers/virtio/virtio.c                  | 22 
>>>>>>>>> +++++++++++++++-------
>>>>>>>>>      drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>>>>      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>>>>      drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>>>>      drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>>>>      include/linux/virtio_config.h            |  5 +++--
>>>>>>>>>      10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>>>>
>>>>>>>>> diff --git a/arch/um/drivers/virtio_uml.c 
>>>>>>>>> b/arch/um/drivers/virtio_uml.c
>>>>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          vu_dev->status = status;
>>>>>>>>>      }
>>>>>>>>>      -static void vu_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_uml_device *vu_dev = 
>>>>>>>>> to_virtio_uml_device(vdev);
>>>>>>>>>            vu_dev->status = 0;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static void vu_del_vq(struct virtqueue *vq)
>>>>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>>>>      }
>>>>>>>>>        /* Reset the device. Not much here for now. */
>>>>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct mlxbf_tmfifo_vdev *tm_vdev = 
>>>>>>>>> mlxbf_vdev_to_tmfifo(vdev);
>>>>>>>>>            tm_vdev->status = 0;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        /* Read the value of a configuration field. */
>>>>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>          dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>>>>      }
>>>>>>>>>      -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>>>>          struct fw_rsc_vdev *rsc;
>>>>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>>>>>>> virtio_device *vdev)
>>>>>>>>>            rsc->status = 0;
>>>>>>>>>          dev_dbg(&vdev->dev, "reset !\n");
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        /* provide the vdev features as retrieved from the 
>>>>>>>>> firmware */
>>>>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>>>>          return ret;
>>>>>>>>>      }
>>>>>>>>>      -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>>>>          struct ccw1 *ccw;
>>>>>>>>> +    int ret;
>>>>>>>>>            ccw = ccw_device_dma_zalloc(vcdev->cdev, 
>>>>>>>>> sizeof(*ccw));
>>>>>>>>>          if (!ccw)
>>>>>>>>> -        return;
>>>>>>>>> +        return -ENOMEM;
>>>>>>>>>            /* Zero status bits. */
>>>>>>>>>          vcdev->dma_area->status = 0;
>>>>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>>>>>>> virtio_device *vdev)
>>>>>>>>>          ccw->flags = 0;
>>>>>>>>>          ccw->count = 0;
>>>>>>>>>          ccw->cda = 0;
>>>>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>>          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>>>>> +
>>>>>>>>> +    return ret;
>>>>>>>>>      }
>>>>>>>>>        static u64 virtio_ccw_get_features(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>>>>> --- a/drivers/virtio/virtio.c
>>>>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *dev)
>>>>>>>>>          /* Assign a unique device index and hence name. */
>>>>>>>>>          err = ida_simple_get(&virtio_index_ida, 0, 0, 
>>>>>>>>> GFP_KERNEL);
>>>>>>>>>          if (err < 0)
>>>>>>>>> -        goto out;
>>>>>>>>> +        goto out_err;
>>>>>>>>>            dev->index = err;
>>>>>>>>>          dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *dev)
>>>>>>>>>            /* We always start by resetting the device, in case 
>>>>>>>>> a previous
>>>>>>>>>           * driver messed it up.  This also tests that code 
>>>>>>>>> path a
>>>>>>>>> little. */
>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>> +    err = dev->config->reset(dev);
>>>>>>>>> +    if (err)
>>>>>>>>> +        goto out_ida;
>>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>>>>>>> virtio_device *dev)
>>>>>>>>>           */
>>>>>>>>>          err = device_add(&dev->dev);
>>>>>>>>>          if (err)
>>>>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>> -out:
>>>>>>>>> -    if (err)
>>>>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>> +        goto out_ida;
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>> +
>>>>>>>>> +out_ida:
>>>>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>> +out_err:
>>>>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>>          return err;
>>>>>>>>>      }
>>>>>>>>>      EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct 
>>>>>>>>> virtio_device *dev)
>>>>>>>>>            /* We always start by resetting the device, in case 
>>>>>>>>> a previous
>>>>>>>>>           * driver messed it up. */
>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>> +    ret = dev->config->reset(dev);
>>>>>>>>> +    if (ret)
>>>>>>>>> +        goto err;
>>>>>>>>>            /* Acknowledge that we've seen the device. */
>>>>>>>>>          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>> diff --git a/drivers/virtio/virtio_mmio.c 
>>>>>>>>> b/drivers/virtio/virtio_mmio.c
>>>>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>>      }
>>>>>>>>>      -static void vm_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_mmio_device *vm_dev = 
>>>>>>>>> to_virtio_mmio_device(vdev);
>>>>>>>>>            /* 0 status means a reset. */
>>>>>>>>>          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>>          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>      }
>>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>          /* 0 status means a reset. */
>>>>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>>          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device 
>>>>>>>>> *vp_dev,
>>>>>>>>> u16 vector)
>>>>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct 
>>>>>>>>> virtio_device
>>>>>>>>> *vdev, u8 status)
>>>>>>>>> vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>>>>      }
>>>>>>>>>      -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device 
>>>>>>>>> *vdev)
>>>>>>>>>              msleep(1);
>>>>>>>>>          /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>          vp_synchronize_vectors(vdev);
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static u16 vp_config_vector(struct virtio_pci_device 
>>>>>>>>> *vp_dev,
>>>>>>>>> u16 vector)
>>>>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c 
>>>>>>>>> b/drivers/virtio/virtio_vdpa.c
>>>>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>          return ops->set_status(vdpa, status);
>>>>>>>>>      }
>>>>>>>>>      -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>>      {
>>>>>>>>>          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>>>>            vdpa_reset(vdpa);
>>>>>>>>> +
>>>>>>>>> +    return 0;
>>>>>>>>>      }
>>>>>>>>>        static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>>>>> b/include/linux/virtio_config.h
>>>>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>>>>> --- a/include/linux/virtio_config.h
>>>>>>>>> +++ b/include/linux/virtio_config.h
>>>>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>>>>       *    status: the new status byte
>>>>>>>>>       * @reset: reset the device
>>>>>>>>>       *    vdev: the virtio device
>>>>>>>>> - *    After this, status and feature negotiation must be done 
>>>>>>>>> again
>>>>>>>>> + *    Upon success, status and feature negotiation must be 
>>>>>>>>> done again
>>>>>>>>>       *    Device must not be reset from its vq/config 
>>>>>>>>> callbacks, or in
>>>>>>>>>       *    parallel with being added/removed.
>>>>>>>>> + *    Returns 0 on success or error status.
>>>>>>>>>       * @find_vqs: find virtqueues and instantiate them.
>>>>>>>>>       *    vdev: the virtio_device
>>>>>>>>>       *    nvqs: the number of virtqueues to find
>>>>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>>>>          u32 (*generation)(struct virtio_device *vdev);
>>>>>>>>>          u8 (*get_status)(struct virtio_device *vdev);
>>>>>>>>>          void (*set_status)(struct virtio_device *vdev, u8 
>>>>>>>>> status);
>>>>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>>>>          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>>>>                  struct virtqueue *vqs[], vq_callback_t 
>>>>>>>>> *callbacks[],
>>>>>>>>>                  const char * const names[], const bool *ctx,
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Tue, Apr 13, 2021 at 01:53:02AM +0300, Max Gurtovoy wrote:
> 
> On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
> > On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> > > On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > > > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > The reset device operation, usually is an operation that might fail from
> > > > > > > > > various reasons. For example, the controller might be in a bad state and
> > > > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > > > devices always succeed in reset operation but this is not the case for
> > > > > > > > > HW based virtio devices.
> > > > > > > > I would like to know under what condition that the reset operation may
> > > > > > > > fail (except for the case of a bugg guest).
> > > > > > > The controller might not be ready or stuck. This is a real use case for many
> > > > > > > PCI devices.
> > > > > > > 
> > > > > > > For real devices the FW might be in a bad state and it can happen also for
> > > > > > > paravirt device if you have a bug in the controller code or if you entered
> > > > > > > some error flow (Out of memory).
> > > > > > > 
> > > > > > > You don't want to be stuck because of one bad device.
> > > > > > OK so maybe we can do more to detect the bad device.
> > > > > > Won't we get all 1's on a read in this case?
> > > > > No. how can we guarantee it ?
> > > > > 
> > > > Well this is what you tend to get if e.g. you disable device memory.
> > > > 
> > > > Anyway, you know about hardware, I don't ... It's not returning 0 after
> > > > reset as it should ... what does it return? Hopefully not random noise -
> > > > I don't think it's very practical to write a driver for a device that
> > > > starts doing that at random times ...
> > > The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
> > > 
> > > For paravirt devices, think of a situation that you can't allocate some
> > > internal buffers (malloc failed) and you want to wait for few seconds until
> > > the system memory will free some pages.
> > > 
> > > So you may return NEEDS_RESET that indicates some error state of the device.
> > > Once the system memory freed by other application for example, your internal
> > > virtio device malloc succeeded and you may return 0.
> > > 
> > > In this case, you don't want to stall the other virtio devices to probe
> > > (they might be real HW devices that driven by the same driver), right ?
> > So the device is very busy then? Not sure it's smart to just assume
> > it's safe to free all memory allocated for it then ...
> > 
> > I guess the lesson is don't make device reset depend on malloc
> > of some memory?
> 
> The device is not ready yet. And the malloc is just one example I gave you
> to emphasize the case.
> 
> Another example can be a bad FW installed.

Surely this will wedge the host device to the point where vdpa won't
even be probed?

> The host/guest driver is trying to enable the device but the device is not
> ready. This is the real life use case and I gave many examples for reasons
> for device not to be ready. For paravirt and HW devices.
> 
> Endless loop and stalling next devices probe is not the way to go. PCI
> drivers can't allow this to happen.
> 
> Think of a situation that host it's booting from virtio-blk device but it
> has another virtio-net device that has bad FW or other internal error (it
> doesn't matter what it is for the example) and the virtio pci driver is
> probing virtio-net device first.
> 
> The host will never boot successfully. This is fatal.

It's not nice for sure. But broken hardware is broken hardware.
Try not to have it is still best advice.
Making very sure this will never regress any useful flows is high
priority though.  Thus the focus on addressing specific issues
not just falling over the moment something seems a bit fishy.

> Error flows are critical when working with real PCI HW and I understand that
> in paravirt devices with strong hypervisor you will be ok in 99% of the time
> but you need to be aware also for bugs and error flows in both paravirt and
> HW world.
> 
> So first we need to handle this endless loop (with this patch set or with
> async probing mechanism) and later we should update the specification.
> 
> The virtio world now is not only guest and paravirt devices. Bare metal
> hosts start using virtio devices and drivers more and more.

Excellent so can we please stop talking hypotheticals and discuss how
does actual hardware behave, and how best to recover in each case?
Recovering from weird scenarious like bad firmware is nice to have but
not at risk of e.g. corrupting kernel memory. Which blindly removing the
driver with an active device has a good chance to do.

Just to stress the importance of addressing actual issues,
how was this patch even tested? Worth mentioning in the commit log.

And by the way, there are actually use-cases I am aware of where we spin
forever and really should not, for example it would be super nice to
support surprise removal so host can drop the driver after device is
gone without waiting for guest. This allows allocating the device to
another VM.  In that case too, an out of the blue timeout and blindly
assuming all is well is not the way to go, you need to probe device to
check whether it's dead or alive.
Maybe your error recovery can fit in such a framework.


> > 
> > 
> > > > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > > > resetting virtio devices.
> > > > > > > > > 
> > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > ---
> > > > > > > > > 
> > > > > > > > > changes from v1:
> > > > > > > > >      - update virtio_ccw.c (Cornelia)
> > > > > > > > >      - update virtio_uml.c
> > > > > > > > >      - update mlxbf-tmfifo.c
> > > > > > > > Note that virtio driver may call reset, so you probably need to convert
> > > > > > > > them.
> > > > > > > I'm sure I understand.
> > > > > > > 
> > > > > > > Convert to what ?
> > > > > > > 
> > > > > > > Thanks.
> > > > > > > 
> > > > > > > > Thanks
> > > > > > > > 
> > > > > > > > 
> > > > > > > > > ---
> > > > > > > > >      arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > >      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > >      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > >      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > >      drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > > > > > >      drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > >      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > >      drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > >      drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > >      include/linux/virtio_config.h            |  5 +++--
> > > > > > > > >      10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          vu_dev->status = status;
> > > > > > > > >      }
> > > > > > > > >      -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > >            vu_dev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > >      }
> > > > > > > > >        /* Reset the device. Not much here for now. */
> > > > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > >            tm_vdev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        /* Read the value of a configuration field. */
> > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >          dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > >      }
> > > > > > > > >      -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > >          struct fw_rsc_vdev *rsc;
> > > > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > > > virtio_device *vdev)
> > > > > > > > >            rsc->status = 0;
> > > > > > > > >          dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        /* provide the vdev features as retrieved from the firmware */
> > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > > > >          return ret;
> > > > > > > > >      }
> > > > > > > > >      -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > >          struct ccw1 *ccw;
> > > > > > > > > +    int ret;
> > > > > > > > >            ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > > > >          if (!ccw)
> > > > > > > > > -        return;
> > > > > > > > > +        return -ENOMEM;
> > > > > > > > >            /* Zero status bits. */
> > > > > > > > >          vcdev->dma_area->status = 0;
> > > > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > > > virtio_device *vdev)
> > > > > > > > >          ccw->flags = 0;
> > > > > > > > >          ccw->count = 0;
> > > > > > > > >          ccw->cda = 0;
> > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > >          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > +
> > > > > > > > > +    return ret;
> > > > > > > > >      }
> > > > > > > > >        static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > > > > > *dev)
> > > > > > > > >          /* Assign a unique device index and hence name. */
> > > > > > > > >          err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > > > >          if (err < 0)
> > > > > > > > > -        goto out;
> > > > > > > > > +        goto out_err;
> > > > > > > > >            dev->index = err;
> > > > > > > > >          dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > > > > > *dev)
> > > > > > > > >            /* We always start by resetting the device, in case a previous
> > > > > > > > >           * driver messed it up.  This also tests that code path a
> > > > > > > > > little. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > +    if (err)
> > > > > > > > > +        goto out_ida;
> > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > > > virtio_device *dev)
> > > > > > > > >           */
> > > > > > > > >          err = device_add(&dev->dev);
> > > > > > > > >          if (err)
> > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > -out:
> > > > > > > > > -    if (err)
> > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > +        goto out_ida;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > > +
> > > > > > > > > +out_ida:
> > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > +out_err:
> > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > >          return err;
> > > > > > > > >      }
> > > > > > > > >      EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > > > > > >            /* We always start by resetting the device, in case a previous
> > > > > > > > >           * driver messed it up. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > +    if (ret)
> > > > > > > > > +        goto err;
> > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > >      }
> > > > > > > > >      -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > > > >            /* 0 status means a reset. */
> > > > > > > > >          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >      }
> > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > > > u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > >      }
> > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >              msleep(1);
> > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > > > u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >          return ops->set_status(vdpa, status);
> > > > > > > > >      }
> > > > > > > > >      -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > >            vdpa_reset(vdpa);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > >       *    status: the new status byte
> > > > > > > > >       * @reset: reset the device
> > > > > > > > >       *    vdev: the virtio device
> > > > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > > > + *    Upon success, status and feature negotiation must be done again
> > > > > > > > >       *    Device must not be reset from its vq/config callbacks, or in
> > > > > > > > >       *    parallel with being added/removed.
> > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > >       * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > >       *    vdev: the virtio_device
> > > > > > > > >       *    nvqs: the number of virtqueues to find
> > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > >          u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > >          u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > >          void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > >          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > >                  struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > > > >                  const char * const names[], const bool *ctx,

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Tue, Apr 13, 2021 at 01:53:02AM +0300, Max Gurtovoy wrote:
> 
> On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
> > On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> > > On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > > > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > The reset device operation, usually is an operation that might fail from
> > > > > > > > > various reasons. For example, the controller might be in a bad state and
> > > > > > > > > can't answer to any request. Usually, the paravirt SW based virtio
> > > > > > > > > devices always succeed in reset operation but this is not the case for
> > > > > > > > > HW based virtio devices.
> > > > > > > > I would like to know under what condition that the reset operation may
> > > > > > > > fail (except for the case of a bugg guest).
> > > > > > > The controller might not be ready or stuck. This is a real use case for many
> > > > > > > PCI devices.
> > > > > > > 
> > > > > > > For real devices the FW might be in a bad state and it can happen also for
> > > > > > > paravirt device if you have a bug in the controller code or if you entered
> > > > > > > some error flow (Out of memory).
> > > > > > > 
> > > > > > > You don't want to be stuck because of one bad device.
> > > > > > OK so maybe we can do more to detect the bad device.
> > > > > > Won't we get all 1's on a read in this case?
> > > > > No. how can we guarantee it ?
> > > > > 
> > > > Well this is what you tend to get if e.g. you disable device memory.
> > > > 
> > > > Anyway, you know about hardware, I don't ... It's not returning 0 after
> > > > reset as it should ... what does it return? Hopefully not random noise -
> > > > I don't think it's very practical to write a driver for a device that
> > > > starts doing that at random times ...
> > > The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
> > > 
> > > For paravirt devices, think of a situation that you can't allocate some
> > > internal buffers (malloc failed) and you want to wait for few seconds until
> > > the system memory will free some pages.
> > > 
> > > So you may return NEEDS_RESET that indicates some error state of the device.
> > > Once the system memory freed by other application for example, your internal
> > > virtio device malloc succeeded and you may return 0.
> > > 
> > > In this case, you don't want to stall the other virtio devices to probe
> > > (they might be real HW devices that driven by the same driver), right ?
> > So the device is very busy then? Not sure it's smart to just assume
> > it's safe to free all memory allocated for it then ...
> > 
> > I guess the lesson is don't make device reset depend on malloc
> > of some memory?
> 
> The device is not ready yet. And the malloc is just one example I gave you
> to emphasize the case.
> 
> Another example can be a bad FW installed.

Surely this will wedge the host device to the point where vdpa won't
even be probed?

> The host/guest driver is trying to enable the device but the device is not
> ready. This is the real life use case and I gave many examples for reasons
> for device not to be ready. For paravirt and HW devices.
> 
> Endless loop and stalling next devices probe is not the way to go. PCI
> drivers can't allow this to happen.
> 
> Think of a situation that host it's booting from virtio-blk device but it
> has another virtio-net device that has bad FW or other internal error (it
> doesn't matter what it is for the example) and the virtio pci driver is
> probing virtio-net device first.
> 
> The host will never boot successfully. This is fatal.

It's not nice for sure. But broken hardware is broken hardware.
Try not to have it is still best advice.
Making very sure this will never regress any useful flows is high
priority though.  Thus the focus on addressing specific issues
not just falling over the moment something seems a bit fishy.

> Error flows are critical when working with real PCI HW and I understand that
> in paravirt devices with strong hypervisor you will be ok in 99% of the time
> but you need to be aware also for bugs and error flows in both paravirt and
> HW world.
> 
> So first we need to handle this endless loop (with this patch set or with
> async probing mechanism) and later we should update the specification.
> 
> The virtio world now is not only guest and paravirt devices. Bare metal
> hosts start using virtio devices and drivers more and more.

Excellent so can we please stop talking hypotheticals and discuss how
does actual hardware behave, and how best to recover in each case?
Recovering from weird scenarious like bad firmware is nice to have but
not at risk of e.g. corrupting kernel memory. Which blindly removing the
driver with an active device has a good chance to do.

Just to stress the importance of addressing actual issues,
how was this patch even tested? Worth mentioning in the commit log.

And by the way, there are actually use-cases I am aware of where we spin
forever and really should not, for example it would be super nice to
support surprise removal so host can drop the driver after device is
gone without waiting for guest. This allows allocating the device to
another VM.  In that case too, an out of the blue timeout and blindly
assuming all is well is not the way to go, you need to probe device to
check whether it's dead or alive.
Maybe your error recovery can fit in such a framework.


> > 
> > 
> > > > > > > > > This commit is also a preparation for adding a timeout mechanism for
> > > > > > > > > resetting virtio devices.
> > > > > > > > > 
> > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > ---
> > > > > > > > > 
> > > > > > > > > changes from v1:
> > > > > > > > >      - update virtio_ccw.c (Cornelia)
> > > > > > > > >      - update virtio_uml.c
> > > > > > > > >      - update mlxbf-tmfifo.c
> > > > > > > > Note that virtio driver may call reset, so you probably need to convert
> > > > > > > > them.
> > > > > > > I'm sure I understand.
> > > > > > > 
> > > > > > > Convert to what ?
> > > > > > > 
> > > > > > > Thanks.
> > > > > > > 
> > > > > > > > Thanks
> > > > > > > > 
> > > > > > > > 
> > > > > > > > > ---
> > > > > > > > >      arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > >      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > >      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > >      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > >      drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
> > > > > > > > >      drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > >      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > >      drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > >      drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > >      include/linux/virtio_config.h            |  5 +++--
> > > > > > > > >      10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
> > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          vu_dev->status = status;
> > > > > > > > >      }
> > > > > > > > >      -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > >            vu_dev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > >      }
> > > > > > > > >        /* Reset the device. Not much here for now. */
> > > > > > > > > -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > >            tm_vdev->status = 0;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        /* Read the value of a configuration field. */
> > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >          dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > >      }
> > > > > > > > >      -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > >          struct fw_rsc_vdev *rsc;
> > > > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > > > virtio_device *vdev)
> > > > > > > > >            rsc->status = 0;
> > > > > > > > >          dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        /* provide the vdev features as retrieved from the firmware */
> > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > > > >          return ret;
> > > > > > > > >      }
> > > > > > > > >      -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > >          struct ccw1 *ccw;
> > > > > > > > > +    int ret;
> > > > > > > > >            ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
> > > > > > > > >          if (!ccw)
> > > > > > > > > -        return;
> > > > > > > > > +        return -ENOMEM;
> > > > > > > > >            /* Zero status bits. */
> > > > > > > > >          vcdev->dma_area->status = 0;
> > > > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > > > virtio_device *vdev)
> > > > > > > > >          ccw->flags = 0;
> > > > > > > > >          ccw->count = 0;
> > > > > > > > >          ccw->cda = 0;
> > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > >          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > +
> > > > > > > > > +    return ret;
> > > > > > > > >      }
> > > > > > > > >        static u64 virtio_ccw_get_features(struct virtio_device *vdev)
> > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
> > > > > > > > > *dev)
> > > > > > > > >          /* Assign a unique device index and hence name. */
> > > > > > > > >          err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
> > > > > > > > >          if (err < 0)
> > > > > > > > > -        goto out;
> > > > > > > > > +        goto out_err;
> > > > > > > > >            dev->index = err;
> > > > > > > > >          dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
> > > > > > > > > *dev)
> > > > > > > > >            /* We always start by resetting the device, in case a previous
> > > > > > > > >           * driver messed it up.  This also tests that code path a
> > > > > > > > > little. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > +    if (err)
> > > > > > > > > +        goto out_ida;
> > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > > > virtio_device *dev)
> > > > > > > > >           */
> > > > > > > > >          err = device_add(&dev->dev);
> > > > > > > > >          if (err)
> > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > -out:
> > > > > > > > > -    if (err)
> > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > +        goto out_ida;
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > > +
> > > > > > > > > +out_ida:
> > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > +out_err:
> > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > >          return err;
> > > > > > > > >      }
> > > > > > > > >      EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
> > > > > > > > >            /* We always start by resetting the device, in case a previous
> > > > > > > > >           * driver messed it up. */
> > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > +    if (ret)
> > > > > > > > > +        goto err;
> > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
> > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > >      }
> > > > > > > > >      -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > > > >            /* 0 status means a reset. */
> > > > > > > > >          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >      }
> > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > > @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > > > u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > *vdev, u8 status)
> > > > > > > > >          vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > >      }
> > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > >          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
> > > > > > > > >              msleep(1);
> > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
> > > > > > > > > u16 vector)
> > > > > > > > > diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > >          return ops->set_status(vdpa, status);
> > > > > > > > >      }
> > > > > > > > >      -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > >      {
> > > > > > > > >          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > >            vdpa_reset(vdpa);
> > > > > > > > > +
> > > > > > > > > +    return 0;
> > > > > > > > >      }
> > > > > > > > >        static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > >       *    status: the new status byte
> > > > > > > > >       * @reset: reset the device
> > > > > > > > >       *    vdev: the virtio device
> > > > > > > > > - *    After this, status and feature negotiation must be done again
> > > > > > > > > + *    Upon success, status and feature negotiation must be done again
> > > > > > > > >       *    Device must not be reset from its vq/config callbacks, or in
> > > > > > > > >       *    parallel with being added/removed.
> > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > >       * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > >       *    vdev: the virtio_device
> > > > > > > > >       *    nvqs: the number of virtqueues to find
> > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > >          u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > >          u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > >          void (*set_status)(struct virtio_device *vdev, u8 status);
> > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > >          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > >                  struct virtqueue *vqs[], vq_callback_t *callbacks[],
> > > > > > > > >                  const char * const names[], const bool *ctx,

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Tue, Apr 13, 2021 at 10:42:56AM +0800, Jason Wang wrote:
> 
> 在 2021/4/13 上午6:53, Max Gurtovoy 写道:
> > 
> > On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
> > > On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> > > > On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > > > > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > > > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > > The reset device operation, usually is
> > > > > > > > > > an operation that might fail from
> > > > > > > > > > various reasons. For example, the
> > > > > > > > > > controller might be in a bad state and
> > > > > > > > > > can't answer to any request. Usually,
> > > > > > > > > > the paravirt SW based virtio
> > > > > > > > > > devices always succeed in reset
> > > > > > > > > > operation but this is not the case for
> > > > > > > > > > HW based virtio devices.
> > > > > > > > > I would like to know under what condition
> > > > > > > > > that the reset operation may
> > > > > > > > > fail (except for the case of a bugg guest).
> > > > > > > > The controller might not be ready or stuck. This
> > > > > > > > is a real use case for many
> > > > > > > > PCI devices.
> > > > > > > > 
> > > > > > > > For real devices the FW might be in a bad state
> > > > > > > > and it can happen also for
> > > > > > > > paravirt device if you have a bug in the
> > > > > > > > controller code or if you entered
> > > > > > > > some error flow (Out of memory).
> > > > > > > > 
> > > > > > > > You don't want to be stuck because of one bad device.
> > > > > > > OK so maybe we can do more to detect the bad device.
> > > > > > > Won't we get all 1's on a read in this case?
> > > > > > No. how can we guarantee it ?
> > > > > > 
> > > > > Well this is what you tend to get if e.g. you disable device memory.
> > > > > 
> > > > > Anyway, you know about hardware, I don't ... It's not
> > > > > returning 0 after
> > > > > reset as it should ... what does it return? Hopefully not
> > > > > random noise -
> > > > > I don't think it's very practical to write a driver for a device that
> > > > > starts doing that at random times ...
> > > > The device may return 0x40 (NEEDS_RESET). It doesn't have to
> > > > return all 1's.
> > > > 
> > > > For paravirt devices, think of a situation that you can't allocate some
> > > > internal buffers (malloc failed) and you want to wait for few
> > > > seconds until
> > > > the system memory will free some pages.
> > > > 
> > > > So you may return NEEDS_RESET that indicates some error state of
> > > > the device.
> > > > Once the system memory freed by other application for example,
> > > > your internal
> > > > virtio device malloc succeeded and you may return 0.
> > > > 
> > > > In this case, you don't want to stall the other virtio devices to probe
> > > > (they might be real HW devices that driven by the same driver), right ?
> > > So the device is very busy then? Not sure it's smart to just assume
> > > it's safe to free all memory allocated for it then ...
> > > 
> > > I guess the lesson is don't make device reset depend on malloc
> > > of some memory?
> > 
> > The device is not ready yet. And the malloc is just one example I gave
> > you to emphasize the case.
> > 
> > Another example can be a bad FW installed.
> > 
> > The host/guest driver is trying to enable the device but the device is
> > not ready. This is the real life use case and I gave many examples for
> > reasons for device not to be ready. For paravirt and HW devices.
> > 
> > Endless loop and stalling next devices probe is not the way to go. PCI
> > drivers can't allow this to happen.
> > 
> > Think of a situation that host it's booting from virtio-blk device but
> > it has another virtio-net device that has bad FW or other internal error
> > (it doesn't matter what it is for the example) and the virtio pci driver
> > is probing virtio-net device first.
> > 
> > The host will never boot successfully. This is fatal.
> 
> 
> The issue is that there's not clear definition in the spec about how device
> is expected to behave when it is being reset. E.g can the device still
> modify virtqueue in this case? If yes, stall the host is much more safe than
> simply ignore it.
> 
> In another way, I wonder whether we can do transport level reset (e.g PCI)
> in this case.
> 

Or a bus reset. Yes. That would be much more reasonable.
That will give you a reasonable timeout too since the pci spec
actually tells you how long is FLR supposed to take.
So we can spin until we get to within that order of magnitude of time.


> > 
> > Error flows are critical when working with real PCI HW and I understand
> > that in paravirt devices with strong hypervisor you will be ok in 99% of
> > the time but you need to be aware also for bugs and error flows in both
> > paravirt and HW world.
> > 
> > So first we need to handle this endless loop (with this patch set or
> > with async probing mechanism) and later we should update the
> > specification.
> > 
> > The virtio world now is not only guest and paravirt devices. Bare metal
> > hosts start using virtio devices and drivers more and more.
> 
> 
> So as mentioned in previous reply, this driver has been used for real virtio
> hardware for many years. One well known example is the Ali Cloud ECS bare
> metal instance.
> 
> Thanks
> 
> 
> > 
> > > 
> > > 
> > > > > > > > > > This commit is also a preparation for
> > > > > > > > > > adding a timeout mechanism for
> > > > > > > > > > resetting virtio devices.
> > > > > > > > > > 
> > > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > > ---
> > > > > > > > > > 
> > > > > > > > > > changes from v1:
> > > > > > > > > >      - update virtio_ccw.c (Cornelia)
> > > > > > > > > >      - update virtio_uml.c
> > > > > > > > > >      - update mlxbf-tmfifo.c
> > > > > > > > > Note that virtio driver may call reset, so
> > > > > > > > > you probably need to convert
> > > > > > > > > them.
> > > > > > > > I'm sure I understand.
> > > > > > > > 
> > > > > > > > Convert to what ?
> > > > > > > > 
> > > > > > > > Thanks.
> > > > > > > > 
> > > > > > > > > Thanks
> > > > > > > > > 
> > > > > > > > > 
> > > > > > > > > > ---
> > > > > > > > > >      arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > > >      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > > >      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > > >      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > > >     
> > > > > > > > > > drivers/virtio/virtio.c                 
> > > > > > > > > > | 22 +++++++++++++++-------
> > > > > > > > > >      drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > > >      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > > >      drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > > >      drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > > >      include/linux/virtio_config.h            |  5 +++--
> > > > > > > > > >      10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > > 
> > > > > > > > > > diff --git
> > > > > > > > > > a/arch/um/drivers/virtio_uml.c
> > > > > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > > > > vu_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          vu_dev->status = status;
> > > > > > > > > >      }
> > > > > > > > > >      -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_uml_device
> > > > > > > > > > *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > > >            vu_dev->status = 0;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > > >      }
> > > > > > > > > >        /* Reset the device. Not much here for now. */
> > > > > > > > > > -static void
> > > > > > > > > > mlxbf_tmfifo_virtio_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct mlxbf_tmfifo_vdev
> > > > > > > > > > *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > > >            tm_vdev->status = 0;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        /* Read the value of a configuration field. */
> > > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > > >          dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > > >          struct fw_rsc_vdev *rsc;
> > > > > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > >            rsc->status = 0;
> > > > > > > > > >          dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        /* provide the vdev features as
> > > > > > > > > > retrieved from the firmware */
> > > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > > > > >          return ret;
> > > > > > > > > >      }
> > > > > > > > > >      -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > > >          struct ccw1 *ccw;
> > > > > > > > > > +    int ret;
> > > > > > > > > >            ccw =
> > > > > > > > > > ccw_device_dma_zalloc(vcdev->cdev,
> > > > > > > > > > sizeof(*ccw));
> > > > > > > > > >          if (!ccw)
> > > > > > > > > > -        return;
> > > > > > > > > > +        return -ENOMEM;
> > > > > > > > > >            /* Zero status bits. */
> > > > > > > > > >          vcdev->dma_area->status = 0;
> > > > > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > >          ccw->flags = 0;
> > > > > > > > > >          ccw->count = 0;
> > > > > > > > > >          ccw->cda = 0;
> > > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > >          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > > +
> > > > > > > > > > +    return ret;
> > > > > > > > > >      }
> > > > > > > > > >        static u64
> > > > > > > > > > virtio_ccw_get_features(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > > > > register_virtio_device(struct
> > > > > > > > > > virtio_device
> > > > > > > > > > *dev)
> > > > > > > > > >          /* Assign a unique device index and hence name. */
> > > > > > > > > >          err =
> > > > > > > > > > ida_simple_get(&virtio_index_ida, 0, 0,
> > > > > > > > > > GFP_KERNEL);
> > > > > > > > > >          if (err < 0)
> > > > > > > > > > -        goto out;
> > > > > > > > > > +        goto out_err;
> > > > > > > > > >            dev->index = err;
> > > > > > > > > >          dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > > > > register_virtio_device(struct
> > > > > > > > > > virtio_device
> > > > > > > > > > *dev)
> > > > > > > > > >            /* We always start by
> > > > > > > > > > resetting the device, in case a previous
> > > > > > > > > >           * driver messed it up.  This
> > > > > > > > > > also tests that code path a
> > > > > > > > > > little. */
> > > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > > +    if (err)
> > > > > > > > > > +        goto out_ida;
> > > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > > > > virtio_device *dev)
> > > > > > > > > >           */
> > > > > > > > > >          err = device_add(&dev->dev);
> > > > > > > > > >          if (err)
> > > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > > -out:
> > > > > > > > > > -    if (err)
> > > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > > +        goto out_ida;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > > +
> > > > > > > > > > +out_ida:
> > > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > > +out_err:
> > > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > >          return err;
> > > > > > > > > >      }
> > > > > > > > > >      EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > > @@ -408,7 +414,9 @@ int
> > > > > > > > > > virtio_device_restore(struct
> > > > > > > > > > virtio_device *dev)
> > > > > > > > > >            /* We always start by
> > > > > > > > > > resetting the device, in case a previous
> > > > > > > > > >           * driver messed it up. */
> > > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > > +    if (ret)
> > > > > > > > > > +        goto err;
> > > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/virtio/virtio_mmio.c
> > > > > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > > > > vm_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_mmio_device
> > > > > > > > > > *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > > > > >            /* 0 status means a reset. */
> > > > > > > > > >          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > > > @@ -99,6 +99,8 @@ static void
> > > > > > > > > > vp_reset(struct virtio_device *vdev)
> > > > > > > > > >          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static u16
> > > > > > > > > > vp_config_vector(struct
> > > > > > > > > > virtio_pci_device *vp_dev,
> > > > > > > > > > u16 vector)
> > > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > @@ -158,7 +158,7 @@ static void
> > > > > > > > > > vp_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > > vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > > >          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > > @@ -174,6 +174,7 @@ static void
> > > > > > > > > > vp_reset(struct virtio_device *vdev)
> > > > > > > > > >              msleep(1);
> > > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static u16
> > > > > > > > > > vp_config_vector(struct
> > > > > > > > > > virtio_pci_device *vp_dev,
> > > > > > > > > > u16 vector)
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > > >          return ops->set_status(vdpa, status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > > >            vdpa_reset(vdpa);
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > > >       *    status: the new status byte
> > > > > > > > > >       * @reset: reset the device
> > > > > > > > > >       *    vdev: the virtio device
> > > > > > > > > > - *    After this, status and feature
> > > > > > > > > > negotiation must be done again
> > > > > > > > > > + *    Upon success, status and feature
> > > > > > > > > > negotiation must be done again
> > > > > > > > > >       *    Device must not be reset from
> > > > > > > > > > its vq/config callbacks, or in
> > > > > > > > > >       *    parallel with being added/removed.
> > > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > > >       * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > > >       *    vdev: the virtio_device
> > > > > > > > > >       *    nvqs: the number of virtqueues to find
> > > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > > >          u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > > >          u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > > >          void (*set_status)(struct
> > > > > > > > > > virtio_device *vdev, u8 status);
> > > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > > >          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > > >                  struct virtqueue
> > > > > > > > > > *vqs[], vq_callback_t *callbacks[],
> > > > > > > > > >                  const char * const names[], const bool *ctx,
> > 

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Michael S. Tsirkin]

On Tue, Apr 13, 2021 at 10:42:56AM +0800, Jason Wang wrote:
> 
> 在 2021/4/13 上午6:53, Max Gurtovoy 写道:
> > 
> > On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
> > > On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
> > > > On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
> > > > > On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
> > > > > > On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
> > > > > > > On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
> > > > > > > > On 4/8/2021 11:58 AM, Jason Wang wrote:
> > > > > > > > > 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
> > > > > > > > > > The reset device operation, usually is
> > > > > > > > > > an operation that might fail from
> > > > > > > > > > various reasons. For example, the
> > > > > > > > > > controller might be in a bad state and
> > > > > > > > > > can't answer to any request. Usually,
> > > > > > > > > > the paravirt SW based virtio
> > > > > > > > > > devices always succeed in reset
> > > > > > > > > > operation but this is not the case for
> > > > > > > > > > HW based virtio devices.
> > > > > > > > > I would like to know under what condition
> > > > > > > > > that the reset operation may
> > > > > > > > > fail (except for the case of a bugg guest).
> > > > > > > > The controller might not be ready or stuck. This
> > > > > > > > is a real use case for many
> > > > > > > > PCI devices.
> > > > > > > > 
> > > > > > > > For real devices the FW might be in a bad state
> > > > > > > > and it can happen also for
> > > > > > > > paravirt device if you have a bug in the
> > > > > > > > controller code or if you entered
> > > > > > > > some error flow (Out of memory).
> > > > > > > > 
> > > > > > > > You don't want to be stuck because of one bad device.
> > > > > > > OK so maybe we can do more to detect the bad device.
> > > > > > > Won't we get all 1's on a read in this case?
> > > > > > No. how can we guarantee it ?
> > > > > > 
> > > > > Well this is what you tend to get if e.g. you disable device memory.
> > > > > 
> > > > > Anyway, you know about hardware, I don't ... It's not
> > > > > returning 0 after
> > > > > reset as it should ... what does it return? Hopefully not
> > > > > random noise -
> > > > > I don't think it's very practical to write a driver for a device that
> > > > > starts doing that at random times ...
> > > > The device may return 0x40 (NEEDS_RESET). It doesn't have to
> > > > return all 1's.
> > > > 
> > > > For paravirt devices, think of a situation that you can't allocate some
> > > > internal buffers (malloc failed) and you want to wait for few
> > > > seconds until
> > > > the system memory will free some pages.
> > > > 
> > > > So you may return NEEDS_RESET that indicates some error state of
> > > > the device.
> > > > Once the system memory freed by other application for example,
> > > > your internal
> > > > virtio device malloc succeeded and you may return 0.
> > > > 
> > > > In this case, you don't want to stall the other virtio devices to probe
> > > > (they might be real HW devices that driven by the same driver), right ?
> > > So the device is very busy then? Not sure it's smart to just assume
> > > it's safe to free all memory allocated for it then ...
> > > 
> > > I guess the lesson is don't make device reset depend on malloc
> > > of some memory?
> > 
> > The device is not ready yet. And the malloc is just one example I gave
> > you to emphasize the case.
> > 
> > Another example can be a bad FW installed.
> > 
> > The host/guest driver is trying to enable the device but the device is
> > not ready. This is the real life use case and I gave many examples for
> > reasons for device not to be ready. For paravirt and HW devices.
> > 
> > Endless loop and stalling next devices probe is not the way to go. PCI
> > drivers can't allow this to happen.
> > 
> > Think of a situation that host it's booting from virtio-blk device but
> > it has another virtio-net device that has bad FW or other internal error
> > (it doesn't matter what it is for the example) and the virtio pci driver
> > is probing virtio-net device first.
> > 
> > The host will never boot successfully. This is fatal.
> 
> 
> The issue is that there's not clear definition in the spec about how device
> is expected to behave when it is being reset. E.g can the device still
> modify virtqueue in this case? If yes, stall the host is much more safe than
> simply ignore it.
> 
> In another way, I wonder whether we can do transport level reset (e.g PCI)
> in this case.
> 

Or a bus reset. Yes. That would be much more reasonable.
That will give you a reasonable timeout too since the pci spec
actually tells you how long is FLR supposed to take.
So we can spin until we get to within that order of magnitude of time.


> > 
> > Error flows are critical when working with real PCI HW and I understand
> > that in paravirt devices with strong hypervisor you will be ok in 99% of
> > the time but you need to be aware also for bugs and error flows in both
> > paravirt and HW world.
> > 
> > So first we need to handle this endless loop (with this patch set or
> > with async probing mechanism) and later we should update the
> > specification.
> > 
> > The virtio world now is not only guest and paravirt devices. Bare metal
> > hosts start using virtio devices and drivers more and more.
> 
> 
> So as mentioned in previous reply, this driver has been used for real virtio
> hardware for many years. One well known example is the Ali Cloud ECS bare
> metal instance.
> 
> Thanks
> 
> 
> > 
> > > 
> > > 
> > > > > > > > > > This commit is also a preparation for
> > > > > > > > > > adding a timeout mechanism for
> > > > > > > > > > resetting virtio devices.
> > > > > > > > > > 
> > > > > > > > > > Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
> > > > > > > > > > ---
> > > > > > > > > > 
> > > > > > > > > > changes from v1:
> > > > > > > > > >      - update virtio_ccw.c (Cornelia)
> > > > > > > > > >      - update virtio_uml.c
> > > > > > > > > >      - update mlxbf-tmfifo.c
> > > > > > > > > Note that virtio driver may call reset, so
> > > > > > > > > you probably need to convert
> > > > > > > > > them.
> > > > > > > > I'm sure I understand.
> > > > > > > > 
> > > > > > > > Convert to what ?
> > > > > > > > 
> > > > > > > > Thanks.
> > > > > > > > 
> > > > > > > > > Thanks
> > > > > > > > > 
> > > > > > > > > 
> > > > > > > > > > ---
> > > > > > > > > >      arch/um/drivers/virtio_uml.c             |  4 +++-
> > > > > > > > > >      drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
> > > > > > > > > >      drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
> > > > > > > > > >      drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
> > > > > > > > > >     
> > > > > > > > > > drivers/virtio/virtio.c                 
> > > > > > > > > > | 22 +++++++++++++++-------
> > > > > > > > > >      drivers/virtio/virtio_mmio.c             |  3 ++-
> > > > > > > > > >      drivers/virtio/virtio_pci_legacy.c       |  4 +++-
> > > > > > > > > >      drivers/virtio/virtio_pci_modern.c       |  3 ++-
> > > > > > > > > >      drivers/virtio/virtio_vdpa.c             |  4 +++-
> > > > > > > > > >      include/linux/virtio_config.h            |  5 +++--
> > > > > > > > > >      10 files changed, 43 insertions(+), 19 deletions(-)
> > > > > > > > > > 
> > > > > > > > > > diff --git
> > > > > > > > > > a/arch/um/drivers/virtio_uml.c
> > > > > > > > > > b/arch/um/drivers/virtio_uml.c
> > > > > > > > > > index 91ddf74ca888..b6e66265ed32 100644
> > > > > > > > > > --- a/arch/um/drivers/virtio_uml.c
> > > > > > > > > > +++ b/arch/um/drivers/virtio_uml.c
> > > > > > > > > > @@ -827,11 +827,13 @@ static void
> > > > > > > > > > vu_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          vu_dev->status = status;
> > > > > > > > > >      }
> > > > > > > > > >      -static void vu_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vu_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_uml_device
> > > > > > > > > > *vu_dev = to_virtio_uml_device(vdev);
> > > > > > > > > >            vu_dev->status = 0;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static void vu_del_vq(struct virtqueue *vq)
> > > > > > > > > > diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > index bbc4e71a16ff..c192b8ac5d9e 100644
> > > > > > > > > > --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
> > > > > > > > > > @@ -980,11 +980,13 @@ static void
> > > > > > > > > > mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
> > > > > > > > > >      }
> > > > > > > > > >        /* Reset the device. Not much here for now. */
> > > > > > > > > > -static void
> > > > > > > > > > mlxbf_tmfifo_virtio_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > > +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct mlxbf_tmfifo_vdev
> > > > > > > > > > *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
> > > > > > > > > >            tm_vdev->status = 0;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        /* Read the value of a configuration field. */
> > > > > > > > > > diff --git a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > index 0cc617f76068..ca9573c62c3d 100644
> > > > > > > > > > --- a/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > +++ b/drivers/remoteproc/remoteproc_virtio.c
> > > > > > > > > > @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
> > > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > > >          dev_dbg(&vdev->dev, "status: %d\n", status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int rproc_virtio_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
> > > > > > > > > >          struct fw_rsc_vdev *rsc;
> > > > > > > > > > @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > >            rsc->status = 0;
> > > > > > > > > >          dev_dbg(&vdev->dev, "reset !\n");
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        /* provide the vdev features as
> > > > > > > > > > retrieved from the firmware */
> > > > > > > > > > diff --git a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > index 54e686dca6de..52b32555e746 100644
> > > > > > > > > > --- a/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > +++ b/drivers/s390/virtio/virtio_ccw.c
> > > > > > > > > > @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
> > > > > > > > > > virtio_device *vdev, unsigned nvqs,
> > > > > > > > > >          return ret;
> > > > > > > > > >      }
> > > > > > > > > >      -static void virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int virtio_ccw_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_ccw_device *vcdev = to_vc_device(vdev);
> > > > > > > > > >          struct ccw1 *ccw;
> > > > > > > > > > +    int ret;
> > > > > > > > > >            ccw =
> > > > > > > > > > ccw_device_dma_zalloc(vcdev->cdev,
> > > > > > > > > > sizeof(*ccw));
> > > > > > > > > >          if (!ccw)
> > > > > > > > > > -        return;
> > > > > > > > > > +        return -ENOMEM;
> > > > > > > > > >            /* Zero status bits. */
> > > > > > > > > >          vcdev->dma_area->status = 0;
> > > > > > > > > > @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > >          ccw->flags = 0;
> > > > > > > > > >          ccw->count = 0;
> > > > > > > > > >          ccw->cda = 0;
> > > > > > > > > > -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > > +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
> > > > > > > > > >          ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
> > > > > > > > > > +
> > > > > > > > > > +    return ret;
> > > > > > > > > >      }
> > > > > > > > > >        static u64
> > > > > > > > > > virtio_ccw_get_features(struct
> > > > > > > > > > virtio_device *vdev)
> > > > > > > > > > diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
> > > > > > > > > > index 4b15c00c0a0a..ddbfd5b5f3bd 100644
> > > > > > > > > > --- a/drivers/virtio/virtio.c
> > > > > > > > > > +++ b/drivers/virtio/virtio.c
> > > > > > > > > > @@ -338,7 +338,7 @@ int
> > > > > > > > > > register_virtio_device(struct
> > > > > > > > > > virtio_device
> > > > > > > > > > *dev)
> > > > > > > > > >          /* Assign a unique device index and hence name. */
> > > > > > > > > >          err =
> > > > > > > > > > ida_simple_get(&virtio_index_ida, 0, 0,
> > > > > > > > > > GFP_KERNEL);
> > > > > > > > > >          if (err < 0)
> > > > > > > > > > -        goto out;
> > > > > > > > > > +        goto out_err;
> > > > > > > > > >            dev->index = err;
> > > > > > > > > >          dev_set_name(&dev->dev, "virtio%u", dev->index);
> > > > > > > > > > @@ -349,7 +349,9 @@ int
> > > > > > > > > > register_virtio_device(struct
> > > > > > > > > > virtio_device
> > > > > > > > > > *dev)
> > > > > > > > > >            /* We always start by
> > > > > > > > > > resetting the device, in case a previous
> > > > > > > > > >           * driver messed it up.  This
> > > > > > > > > > also tests that code path a
> > > > > > > > > > little. */
> > > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > > +    err = dev->config->reset(dev);
> > > > > > > > > > +    if (err)
> > > > > > > > > > +        goto out_ida;
> > > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > > @@ -362,10 +364,14 @@ int register_virtio_device(struct
> > > > > > > > > > virtio_device *dev)
> > > > > > > > > >           */
> > > > > > > > > >          err = device_add(&dev->dev);
> > > > > > > > > >          if (err)
> > > > > > > > > > -        ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > > -out:
> > > > > > > > > > -    if (err)
> > > > > > > > > > -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > > +        goto out_ida;
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > > +
> > > > > > > > > > +out_ida:
> > > > > > > > > > +    ida_simple_remove(&virtio_index_ida, dev->index);
> > > > > > > > > > +out_err:
> > > > > > > > > > +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> > > > > > > > > >          return err;
> > > > > > > > > >      }
> > > > > > > > > >      EXPORT_SYMBOL_GPL(register_virtio_device);
> > > > > > > > > > @@ -408,7 +414,9 @@ int
> > > > > > > > > > virtio_device_restore(struct
> > > > > > > > > > virtio_device *dev)
> > > > > > > > > >            /* We always start by
> > > > > > > > > > resetting the device, in case a previous
> > > > > > > > > >           * driver messed it up. */
> > > > > > > > > > -    dev->config->reset(dev);
> > > > > > > > > > +    ret = dev->config->reset(dev);
> > > > > > > > > > +    if (ret)
> > > > > > > > > > +        goto err;
> > > > > > > > > >            /* Acknowledge that we've seen the device. */
> > > > > > > > > >          virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/virtio/virtio_mmio.c
> > > > > > > > > > b/drivers/virtio/virtio_mmio.c
> > > > > > > > > > index 56128b9c46eb..12b8f048c48d 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_mmio.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_mmio.c
> > > > > > > > > > @@ -256,12 +256,13 @@ static void
> > > > > > > > > > vm_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vm_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vm_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_mmio_device
> > > > > > > > > > *vm_dev = to_virtio_mmio_device(vdev);
> > > > > > > > > >            /* 0 status means a reset. */
> > > > > > > > > >          writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        diff --git a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > index d62e9835aeec..0b5d95e3efa1 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_pci_legacy.c
> > > > > > > > > > @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > >          iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > > >          /* 0 status means a reset. */
> > > > > > > > > > @@ -99,6 +99,8 @@ static void
> > > > > > > > > > vp_reset(struct virtio_device *vdev)
> > > > > > > > > >          ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
> > > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static u16
> > > > > > > > > > vp_config_vector(struct
> > > > > > > > > > virtio_pci_device *vp_dev,
> > > > > > > > > > u16 vector)
> > > > > > > > > > diff --git a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > index fbd4ebc00eb6..cc3412a96a17 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_pci_modern.c
> > > > > > > > > > @@ -158,7 +158,7 @@ static void
> > > > > > > > > > vp_set_status(struct virtio_device
> > > > > > > > > > *vdev, u8 status)
> > > > > > > > > > vp_modern_set_status(&vp_dev->mdev, status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void vp_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int vp_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct virtio_pci_device *vp_dev = to_vp_device(vdev);
> > > > > > > > > >          struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
> > > > > > > > > > @@ -174,6 +174,7 @@ static void
> > > > > > > > > > vp_reset(struct virtio_device *vdev)
> > > > > > > > > >              msleep(1);
> > > > > > > > > >          /* Flush pending VQ/configuration callbacks. */
> > > > > > > > > >          vp_synchronize_vectors(vdev);
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static u16
> > > > > > > > > > vp_config_vector(struct
> > > > > > > > > > virtio_pci_device *vp_dev,
> > > > > > > > > > u16 vector)
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > index e28acf482e0c..5fd4e627a9b0 100644
> > > > > > > > > > --- a/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > +++ b/drivers/virtio/virtio_vdpa.c
> > > > > > > > > > @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
> > > > > > > > > > virtio_device *vdev, u8 status)
> > > > > > > > > >          return ops->set_status(vdpa, status);
> > > > > > > > > >      }
> > > > > > > > > >      -static void virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > > +static int virtio_vdpa_reset(struct virtio_device *vdev)
> > > > > > > > > >      {
> > > > > > > > > >          struct vdpa_device *vdpa = vd_get_vdpa(vdev);
> > > > > > > > > >            vdpa_reset(vdpa);
> > > > > > > > > > +
> > > > > > > > > > +    return 0;
> > > > > > > > > >      }
> > > > > > > > > >        static bool virtio_vdpa_notify(struct virtqueue *vq)
> > > > > > > > > > diff --git a/include/linux/virtio_config.h
> > > > > > > > > > b/include/linux/virtio_config.h
> > > > > > > > > > index 8519b3ae5d52..d2b0f1699a75 100644
> > > > > > > > > > --- a/include/linux/virtio_config.h
> > > > > > > > > > +++ b/include/linux/virtio_config.h
> > > > > > > > > > @@ -44,9 +44,10 @@ struct virtio_shm_region {
> > > > > > > > > >       *    status: the new status byte
> > > > > > > > > >       * @reset: reset the device
> > > > > > > > > >       *    vdev: the virtio device
> > > > > > > > > > - *    After this, status and feature
> > > > > > > > > > negotiation must be done again
> > > > > > > > > > + *    Upon success, status and feature
> > > > > > > > > > negotiation must be done again
> > > > > > > > > >       *    Device must not be reset from
> > > > > > > > > > its vq/config callbacks, or in
> > > > > > > > > >       *    parallel with being added/removed.
> > > > > > > > > > + *    Returns 0 on success or error status.
> > > > > > > > > >       * @find_vqs: find virtqueues and instantiate them.
> > > > > > > > > >       *    vdev: the virtio_device
> > > > > > > > > >       *    nvqs: the number of virtqueues to find
> > > > > > > > > > @@ -82,7 +83,7 @@ struct virtio_config_ops {
> > > > > > > > > >          u32 (*generation)(struct virtio_device *vdev);
> > > > > > > > > >          u8 (*get_status)(struct virtio_device *vdev);
> > > > > > > > > >          void (*set_status)(struct
> > > > > > > > > > virtio_device *vdev, u8 status);
> > > > > > > > > > -    void (*reset)(struct virtio_device *vdev);
> > > > > > > > > > +    int (*reset)(struct virtio_device *vdev);
> > > > > > > > > >          int (*find_vqs)(struct virtio_device *, unsigned nvqs,
> > > > > > > > > >                  struct virtqueue
> > > > > > > > > > *vqs[], vq_callback_t *callbacks[],
> > > > > > > > > >                  const char * const names[], const bool *ctx,
> > 

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH v2 1/3] virtio: update reset callback to return status

[Max Gurtovoy]

On 4/13/2021 7:42 AM, Michael S. Tsirkin wrote:
> On Tue, Apr 13, 2021 at 01:53:02AM +0300, Max Gurtovoy wrote:
>> On 4/13/2021 12:23 AM, Michael S. Tsirkin wrote:
>>> On Mon, Apr 12, 2021 at 04:03:02PM +0300, Max Gurtovoy wrote:
>>>> On 4/12/2021 3:04 PM, Michael S. Tsirkin wrote:
>>>>> On Mon, Apr 12, 2021 at 02:55:27PM +0300, Max Gurtovoy wrote:
>>>>>> On 4/8/2021 6:56 PM, Michael S. Tsirkin wrote:
>>>>>>> On Thu, Apr 08, 2021 at 12:56:52PM +0300, Max Gurtovoy wrote:
>>>>>>>> On 4/8/2021 11:58 AM, Jason Wang wrote:
>>>>>>>>> 在 2021/4/8 下午4:11, Max Gurtovoy 写道:
>>>>>>>>>> The reset device operation, usually is an operation that might fail from
>>>>>>>>>> various reasons. For example, the controller might be in a bad state and
>>>>>>>>>> can't answer to any request. Usually, the paravirt SW based virtio
>>>>>>>>>> devices always succeed in reset operation but this is not the case for
>>>>>>>>>> HW based virtio devices.
>>>>>>>>> I would like to know under what condition that the reset operation may
>>>>>>>>> fail (except for the case of a bugg guest).
>>>>>>>> The controller might not be ready or stuck. This is a real use case for many
>>>>>>>> PCI devices.
>>>>>>>>
>>>>>>>> For real devices the FW might be in a bad state and it can happen also for
>>>>>>>> paravirt device if you have a bug in the controller code or if you entered
>>>>>>>> some error flow (Out of memory).
>>>>>>>>
>>>>>>>> You don't want to be stuck because of one bad device.
>>>>>>> OK so maybe we can do more to detect the bad device.
>>>>>>> Won't we get all 1's on a read in this case?
>>>>>> No. how can we guarantee it ?
>>>>>>
>>>>> Well this is what you tend to get if e.g. you disable device memory.
>>>>>
>>>>> Anyway, you know about hardware, I don't ... It's not returning 0 after
>>>>> reset as it should ... what does it return? Hopefully not random noise -
>>>>> I don't think it's very practical to write a driver for a device that
>>>>> starts doing that at random times ...
>>>> The device may return 0x40 (NEEDS_RESET). It doesn't have to return all 1's.
>>>>
>>>> For paravirt devices, think of a situation that you can't allocate some
>>>> internal buffers (malloc failed) and you want to wait for few seconds until
>>>> the system memory will free some pages.
>>>>
>>>> So you may return NEEDS_RESET that indicates some error state of the device.
>>>> Once the system memory freed by other application for example, your internal
>>>> virtio device malloc succeeded and you may return 0.
>>>>
>>>> In this case, you don't want to stall the other virtio devices to probe
>>>> (they might be real HW devices that driven by the same driver), right ?
>>> So the device is very busy then? Not sure it's smart to just assume
>>> it's safe to free all memory allocated for it then ...
>>>
>>> I guess the lesson is don't make device reset depend on malloc
>>> of some memory?
>> The device is not ready yet. And the malloc is just one example I gave you
>> to emphasize the case.
>>
>> Another example can be a bad FW installed.
> Surely this will wedge the host device to the point where vdpa won't
> even be probed?

what VDPA device ?


>
>> The host/guest driver is trying to enable the device but the device is not
>> ready. This is the real life use case and I gave many examples for reasons
>> for device not to be ready. For paravirt and HW devices.
>>
>> Endless loop and stalling next devices probe is not the way to go. PCI
>> drivers can't allow this to happen.
>>
>> Think of a situation that host it's booting from virtio-blk device but it
>> has another virtio-net device that has bad FW or other internal error (it
>> doesn't matter what it is for the example) and the virtio pci driver is
>> probing virtio-net device first.
>>
>> The host will never boot successfully. This is fatal.
> It's not nice for sure. But broken hardware is broken hardware.

what does it mean ?

broken SW is broken SW as well.

And maybe it's not something that can't be solved manually. why do you 
insist on breaking application in case of error flows ?

what will happen if, for example, virtiofsd process will be killed ? do 
you want the entire guest to suffer ?


> Try not to have it is still best advice.
> Making very sure this will never regress any useful flows is high
> priority though.  Thus the focus on addressing specific issues
> not just falling over the moment something seems a bit fishy.
>
>> Error flows are critical when working with real PCI HW and I understand that
>> in paravirt devices with strong hypervisor you will be ok in 99% of the time
>> but you need to be aware also for bugs and error flows in both paravirt and
>> HW world.
>>
>> So first we need to handle this endless loop (with this patch set or with
>> async probing mechanism) and later we should update the specification.
>>
>> The virtio world now is not only guest and paravirt devices. Bare metal
>> hosts start using virtio devices and drivers more and more.
> Excellent so can we please stop talking hypotheticals and discuss how
> does actual hardware behave, and how best to recover in each case?

Why hypothetical ? if you say that SW bugs in paravirt controllers or 
some system stress is hypothetical so I dis-agree.

For HW devices, I gave you an example of broken FW.

In DPU solutions you enter a world with entire matrix of errors that can 
happen.

Error sometimes are recoverable, but virtio driver probes in sync way 
and this is wrong.

I gave you an example of booting from virtio-blk device that might be fatal.


> Recovering from weird scenarious like bad firmware is nice to have but
> not at risk of e.g. corrupting kernel memory. Which blindly removing the
> driver with an active device has a good chance to do.

weird ? This is the reality. If we will ignore it, this will not be fixed.

Error flows are not nice to have. They are the building blocks of 
developing robust solutions.


>
> Just to stress the importance of addressing actual issues,
> how was this patch even tested? Worth mentioning in the commit log.

With NVIDIA virtio-blk SNAP device.

I plugged virtio-blk pci function to host pci subsystem, and I entered 
the controller to error state.

Then I plugged another function in good state but it was never probed 
without this patch set.

With this patch set, after a timeout the first device failed to probed 
and cleaned all needed resources and the second device probed successfully.

later on, I moved the first controller from error state to good state 
and re-scanned the pci bus to re-probe it (There are other ways to cause 
the driver to re-probe it).

>
> And by the way, there are actually use-cases I am aware of where we spin
> forever and really should not, for example it would be super nice to
> support surprise removal so host can drop the driver after device is
> gone without waiting for guest. This allows allocating the device to
> another VM.  In that case too, an out of the blue timeout and blindly
> assuming all is well is not the way to go, you need to probe device to
> check whether it's dead or alive.
> Maybe your error recovery can fit in such a framework.
>
Spinning forever in kernel is not a good practice and we can think of 
many scenarios. But we should start somewhere and continue improving the 
stack.


>>>
>>>>>>>>>> This commit is also a preparation for adding a timeout mechanism for
>>>>>>>>>> resetting virtio devices.
>>>>>>>>>>
>>>>>>>>>> Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
>>>>>>>>>> ---
>>>>>>>>>>
>>>>>>>>>> changes from v1:
>>>>>>>>>>       - update virtio_ccw.c (Cornelia)
>>>>>>>>>>       - update virtio_uml.c
>>>>>>>>>>       - update mlxbf-tmfifo.c
>>>>>>>>> Note that virtio driver may call reset, so you probably need to convert
>>>>>>>>> them.
>>>>>>>> I'm sure I understand.
>>>>>>>>
>>>>>>>> Convert to what ?
>>>>>>>>
>>>>>>>> Thanks.
>>>>>>>>
>>>>>>>>> Thanks
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>> ---
>>>>>>>>>>       arch/um/drivers/virtio_uml.c             |  4 +++-
>>>>>>>>>>       drivers/platform/mellanox/mlxbf-tmfifo.c |  4 +++-
>>>>>>>>>>       drivers/remoteproc/remoteproc_virtio.c   |  4 +++-
>>>>>>>>>>       drivers/s390/virtio/virtio_ccw.c         |  9 ++++++---
>>>>>>>>>>       drivers/virtio/virtio.c                  | 22 +++++++++++++++-------
>>>>>>>>>>       drivers/virtio/virtio_mmio.c             |  3 ++-
>>>>>>>>>>       drivers/virtio/virtio_pci_legacy.c       |  4 +++-
>>>>>>>>>>       drivers/virtio/virtio_pci_modern.c       |  3 ++-
>>>>>>>>>>       drivers/virtio/virtio_vdpa.c             |  4 +++-
>>>>>>>>>>       include/linux/virtio_config.h            |  5 +++--
>>>>>>>>>>       10 files changed, 43 insertions(+), 19 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
>>>>>>>>>> index 91ddf74ca888..b6e66265ed32 100644
>>>>>>>>>> --- a/arch/um/drivers/virtio_uml.c
>>>>>>>>>> +++ b/arch/um/drivers/virtio_uml.c
>>>>>>>>>> @@ -827,11 +827,13 @@ static void vu_set_status(struct virtio_device
>>>>>>>>>> *vdev, u8 status)
>>>>>>>>>>           vu_dev->status = status;
>>>>>>>>>>       }
>>>>>>>>>>       -static void vu_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int vu_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
>>>>>>>>>>             vu_dev->status = 0;
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         static void vu_del_vq(struct virtqueue *vq)
>>>>>>>>>> diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>>> b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>>> index bbc4e71a16ff..c192b8ac5d9e 100644
>>>>>>>>>> --- a/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>>> +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
>>>>>>>>>> @@ -980,11 +980,13 @@ static void
>>>>>>>>>> mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev,
>>>>>>>>>>       }
>>>>>>>>>>         /* Reset the device. Not much here for now. */
>>>>>>>>>> -static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
>>>>>>>>>>             tm_vdev->status = 0;
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         /* Read the value of a configuration field. */
>>>>>>>>>> diff --git a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>>> b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>>> index 0cc617f76068..ca9573c62c3d 100644
>>>>>>>>>> --- a/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>>> +++ b/drivers/remoteproc/remoteproc_virtio.c
>>>>>>>>>> @@ -191,7 +191,7 @@ static void rproc_virtio_set_status(struct
>>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>>           dev_dbg(&vdev->dev, "status: %d\n", status);
>>>>>>>>>>       }
>>>>>>>>>>       -static void rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int rproc_virtio_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
>>>>>>>>>>           struct fw_rsc_vdev *rsc;
>>>>>>>>>> @@ -200,6 +200,8 @@ static void rproc_virtio_reset(struct
>>>>>>>>>> virtio_device *vdev)
>>>>>>>>>>             rsc->status = 0;
>>>>>>>>>>           dev_dbg(&vdev->dev, "reset !\n");
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         /* provide the vdev features as retrieved from the firmware */
>>>>>>>>>> diff --git a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>>> b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>>> index 54e686dca6de..52b32555e746 100644
>>>>>>>>>> --- a/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>>> +++ b/drivers/s390/virtio/virtio_ccw.c
>>>>>>>>>> @@ -732,14 +732,15 @@ static int virtio_ccw_find_vqs(struct
>>>>>>>>>> virtio_device *vdev, unsigned nvqs,
>>>>>>>>>>           return ret;
>>>>>>>>>>       }
>>>>>>>>>>       -static void virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int virtio_ccw_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct virtio_ccw_device *vcdev = to_vc_device(vdev);
>>>>>>>>>>           struct ccw1 *ccw;
>>>>>>>>>> +    int ret;
>>>>>>>>>>             ccw = ccw_device_dma_zalloc(vcdev->cdev, sizeof(*ccw));
>>>>>>>>>>           if (!ccw)
>>>>>>>>>> -        return;
>>>>>>>>>> +        return -ENOMEM;
>>>>>>>>>>             /* Zero status bits. */
>>>>>>>>>>           vcdev->dma_area->status = 0;
>>>>>>>>>> @@ -749,8 +750,10 @@ static void virtio_ccw_reset(struct
>>>>>>>>>> virtio_device *vdev)
>>>>>>>>>>           ccw->flags = 0;
>>>>>>>>>>           ccw->count = 0;
>>>>>>>>>>           ccw->cda = 0;
>>>>>>>>>> -    ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>>> +    ret = ccw_io_helper(vcdev, ccw, VIRTIO_CCW_DOING_RESET);
>>>>>>>>>>           ccw_device_dma_free(vcdev->cdev, ccw, sizeof(*ccw));
>>>>>>>>>> +
>>>>>>>>>> +    return ret;
>>>>>>>>>>       }
>>>>>>>>>>         static u64 virtio_ccw_get_features(struct virtio_device *vdev)
>>>>>>>>>> diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
>>>>>>>>>> index 4b15c00c0a0a..ddbfd5b5f3bd 100644
>>>>>>>>>> --- a/drivers/virtio/virtio.c
>>>>>>>>>> +++ b/drivers/virtio/virtio.c
>>>>>>>>>> @@ -338,7 +338,7 @@ int register_virtio_device(struct virtio_device
>>>>>>>>>> *dev)
>>>>>>>>>>           /* Assign a unique device index and hence name. */
>>>>>>>>>>           err = ida_simple_get(&virtio_index_ida, 0, 0, GFP_KERNEL);
>>>>>>>>>>           if (err < 0)
>>>>>>>>>> -        goto out;
>>>>>>>>>> +        goto out_err;
>>>>>>>>>>             dev->index = err;
>>>>>>>>>>           dev_set_name(&dev->dev, "virtio%u", dev->index);
>>>>>>>>>> @@ -349,7 +349,9 @@ int register_virtio_device(struct virtio_device
>>>>>>>>>> *dev)
>>>>>>>>>>             /* We always start by resetting the device, in case a previous
>>>>>>>>>>            * driver messed it up.  This also tests that code path a
>>>>>>>>>> little. */
>>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>>> +    err = dev->config->reset(dev);
>>>>>>>>>> +    if (err)
>>>>>>>>>> +        goto out_ida;
>>>>>>>>>>             /* Acknowledge that we've seen the device. */
>>>>>>>>>>           virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>>> @@ -362,10 +364,14 @@ int register_virtio_device(struct
>>>>>>>>>> virtio_device *dev)
>>>>>>>>>>            */
>>>>>>>>>>           err = device_add(&dev->dev);
>>>>>>>>>>           if (err)
>>>>>>>>>> -        ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>>> -out:
>>>>>>>>>> -    if (err)
>>>>>>>>>> -        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>>> +        goto out_ida;
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>> +
>>>>>>>>>> +out_ida:
>>>>>>>>>> +    ida_simple_remove(&virtio_index_ida, dev->index);
>>>>>>>>>> +out_err:
>>>>>>>>>> +    virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>>>>>>>>>>           return err;
>>>>>>>>>>       }
>>>>>>>>>>       EXPORT_SYMBOL_GPL(register_virtio_device);
>>>>>>>>>> @@ -408,7 +414,9 @@ int virtio_device_restore(struct virtio_device *dev)
>>>>>>>>>>             /* We always start by resetting the device, in case a previous
>>>>>>>>>>            * driver messed it up. */
>>>>>>>>>> -    dev->config->reset(dev);
>>>>>>>>>> +    ret = dev->config->reset(dev);
>>>>>>>>>> +    if (ret)
>>>>>>>>>> +        goto err;
>>>>>>>>>>             /* Acknowledge that we've seen the device. */
>>>>>>>>>>           virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>>>>>>>>>> diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
>>>>>>>>>> index 56128b9c46eb..12b8f048c48d 100644
>>>>>>>>>> --- a/drivers/virtio/virtio_mmio.c
>>>>>>>>>> +++ b/drivers/virtio/virtio_mmio.c
>>>>>>>>>> @@ -256,12 +256,13 @@ static void vm_set_status(struct virtio_device
>>>>>>>>>> *vdev, u8 status)
>>>>>>>>>>           writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>>>       }
>>>>>>>>>>       -static void vm_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int vm_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
>>>>>>>>>>             /* 0 status means a reset. */
>>>>>>>>>>           writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         diff --git a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>>> b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>>> index d62e9835aeec..0b5d95e3efa1 100644
>>>>>>>>>> --- a/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>>> +++ b/drivers/virtio/virtio_pci_legacy.c
>>>>>>>>>> @@ -89,7 +89,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>>>> *vdev, u8 status)
>>>>>>>>>>           iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>>       }
>>>>>>>>>>       -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>>           /* 0 status means a reset. */
>>>>>>>>>> @@ -99,6 +99,8 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>>>           ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
>>>>>>>>>>           /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>>           vp_synchronize_vectors(vdev);
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>>>>>> u16 vector)
>>>>>>>>>> diff --git a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>>> b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>>> index fbd4ebc00eb6..cc3412a96a17 100644
>>>>>>>>>> --- a/drivers/virtio/virtio_pci_modern.c
>>>>>>>>>> +++ b/drivers/virtio/virtio_pci_modern.c
>>>>>>>>>> @@ -158,7 +158,7 @@ static void vp_set_status(struct virtio_device
>>>>>>>>>> *vdev, u8 status)
>>>>>>>>>>           vp_modern_set_status(&vp_dev->mdev, status);
>>>>>>>>>>       }
>>>>>>>>>>       -static void vp_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int vp_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct virtio_pci_device *vp_dev = to_vp_device(vdev);
>>>>>>>>>>           struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
>>>>>>>>>> @@ -174,6 +174,7 @@ static void vp_reset(struct virtio_device *vdev)
>>>>>>>>>>               msleep(1);
>>>>>>>>>>           /* Flush pending VQ/configuration callbacks. */
>>>>>>>>>>           vp_synchronize_vectors(vdev);
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         static u16 vp_config_vector(struct virtio_pci_device *vp_dev,
>>>>>>>>>> u16 vector)
>>>>>>>>>> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
>>>>>>>>>> index e28acf482e0c..5fd4e627a9b0 100644
>>>>>>>>>> --- a/drivers/virtio/virtio_vdpa.c
>>>>>>>>>> +++ b/drivers/virtio/virtio_vdpa.c
>>>>>>>>>> @@ -97,11 +97,13 @@ static void virtio_vdpa_set_status(struct
>>>>>>>>>> virtio_device *vdev, u8 status)
>>>>>>>>>>           return ops->set_status(vdpa, status);
>>>>>>>>>>       }
>>>>>>>>>>       -static void virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>>> +static int virtio_vdpa_reset(struct virtio_device *vdev)
>>>>>>>>>>       {
>>>>>>>>>>           struct vdpa_device *vdpa = vd_get_vdpa(vdev);
>>>>>>>>>>             vdpa_reset(vdpa);
>>>>>>>>>> +
>>>>>>>>>> +    return 0;
>>>>>>>>>>       }
>>>>>>>>>>         static bool virtio_vdpa_notify(struct virtqueue *vq)
>>>>>>>>>> diff --git a/include/linux/virtio_config.h
>>>>>>>>>> b/include/linux/virtio_config.h
>>>>>>>>>> index 8519b3ae5d52..d2b0f1699a75 100644
>>>>>>>>>> --- a/include/linux/virtio_config.h
>>>>>>>>>> +++ b/include/linux/virtio_config.h
>>>>>>>>>> @@ -44,9 +44,10 @@ struct virtio_shm_region {
>>>>>>>>>>        *    status: the new status byte
>>>>>>>>>>        * @reset: reset the device
>>>>>>>>>>        *    vdev: the virtio device
>>>>>>>>>> - *    After this, status and feature negotiation must be done again
>>>>>>>>>> + *    Upon success, status and feature negotiation must be done again
>>>>>>>>>>        *    Device must not be reset from its vq/config callbacks, or in
>>>>>>>>>>        *    parallel with being added/removed.
>>>>>>>>>> + *    Returns 0 on success or error status.
>>>>>>>>>>        * @find_vqs: find virtqueues and instantiate them.
>>>>>>>>>>        *    vdev: the virtio_device
>>>>>>>>>>        *    nvqs: the number of virtqueues to find
>>>>>>>>>> @@ -82,7 +83,7 @@ struct virtio_config_ops {
>>>>>>>>>>           u32 (*generation)(struct virtio_device *vdev);
>>>>>>>>>>           u8 (*get_status)(struct virtio_device *vdev);
>>>>>>>>>>           void (*set_status)(struct virtio_device *vdev, u8 status);
>>>>>>>>>> -    void (*reset)(struct virtio_device *vdev);
>>>>>>>>>> +    int (*reset)(struct virtio_device *vdev);
>>>>>>>>>>           int (*find_vqs)(struct virtio_device *, unsigned nvqs,
>>>>>>>>>>                   struct virtqueue *vqs[], vq_callback_t *callbacks[],
>>>>>>>>>>                   const char * const names[], const bool *ctx,