[PATCH] virtio: disable ioeventfd for record/replay

[PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

virtio devices support separate iothreads waiting for
events from file descriptors. These are asynchronous
events that can't be recorded and replayed, therefore
this patch disables ioeventfd for all devices when
record or replay is enabled.

Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
---
 hw/s390x/virtio-ccw.c   |    6 ++++++
 hw/virtio/virtio-mmio.c |    6 ++++++
 hw/virtio/virtio-pci.c  |    6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index 92b950e09a..bd8b9c5755 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -31,6 +31,7 @@
 #include "trace.h"
 #include "hw/s390x/css-bridge.h"
 #include "hw/s390x/s390-virtio-ccw.h"
+#include "sysemu/replay.h"
 
 #define NR_CLASSIC_INDICATOR_BITS 64
 
@@ -769,6 +770,11 @@ static void virtio_ccw_device_realize(VirtioCcwDevice *dev, Error **errp)
         dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
     }
 
+    /* fd-based ioevents can't be synchronized in record/replay */
+    if (replay_mode != REPLAY_MODE_NONE) {
+        dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
+    }
+
     if (k->realize) {
         k->realize(dev, &err);
         if (err) {
diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 5952471b38..1af48a1b04 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -29,6 +29,7 @@
 #include "qemu/host-utils.h"
 #include "qemu/module.h"
 #include "sysemu/kvm.h"
+#include "sysemu/replay.h"
 #include "hw/virtio/virtio-mmio.h"
 #include "qemu/error-report.h"
 #include "qemu/log.h"
@@ -740,6 +741,11 @@ static void virtio_mmio_realizefn(DeviceState *d, Error **errp)
         proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
     }
 
+    /* fd-based ioevents can't be synchronized in record/replay */
+    if (replay_mode != REPLAY_MODE_NONE) {
+        proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
+    }
+
     if (proxy->legacy) {
         memory_region_init_io(&proxy->iomem, OBJECT(d),
                               &virtio_legacy_mem_ops, proxy,
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index b321604d9b..f1e105fa52 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -37,6 +37,7 @@
 #include "qemu/range.h"
 #include "hw/virtio/virtio-bus.h"
 #include "qapi/visitor.h"
+#include "sysemu/replay.h"
 
 #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
 
@@ -1760,6 +1761,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
     }
 
+    /* fd-based ioevents can't be synchronized in record/replay */
+    if (replay_mode != REPLAY_MODE_NONE) {
+        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
+    }
+
     /*
      * virtio pci bar layout used by default.
      * subclasses can re-arrange things if needed.

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

ping

On 17.05.2021 16:06, Pavel Dovgalyuk wrote:
> virtio devices support separate iothreads waiting for
> events from file descriptors. These are asynchronous
> events that can't be recorded and replayed, therefore
> this patch disables ioeventfd for all devices when
> record or replay is enabled.
> 
> Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
> ---
>   hw/s390x/virtio-ccw.c   |    6 ++++++
>   hw/virtio/virtio-mmio.c |    6 ++++++
>   hw/virtio/virtio-pci.c  |    6 ++++++
>   3 files changed, 18 insertions(+)
> 
> diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
> index 92b950e09a..bd8b9c5755 100644
> --- a/hw/s390x/virtio-ccw.c
> +++ b/hw/s390x/virtio-ccw.c
> @@ -31,6 +31,7 @@
>   #include "trace.h"
>   #include "hw/s390x/css-bridge.h"
>   #include "hw/s390x/s390-virtio-ccw.h"
> +#include "sysemu/replay.h"
>   
>   #define NR_CLASSIC_INDICATOR_BITS 64
>   
> @@ -769,6 +770,11 @@ static void virtio_ccw_device_realize(VirtioCcwDevice *dev, Error **errp)
>           dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
> +    }
> +
>       if (k->realize) {
>           k->realize(dev, &err);
>           if (err) {
> diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
> index 5952471b38..1af48a1b04 100644
> --- a/hw/virtio/virtio-mmio.c
> +++ b/hw/virtio/virtio-mmio.c
> @@ -29,6 +29,7 @@
>   #include "qemu/host-utils.h"
>   #include "qemu/module.h"
>   #include "sysemu/kvm.h"
> +#include "sysemu/replay.h"
>   #include "hw/virtio/virtio-mmio.h"
>   #include "qemu/error-report.h"
>   #include "qemu/log.h"
> @@ -740,6 +741,11 @@ static void virtio_mmio_realizefn(DeviceState *d, Error **errp)
>           proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
> +    }
> +
>       if (proxy->legacy) {
>           memory_region_init_io(&proxy->iomem, OBJECT(d),
>                                 &virtio_legacy_mem_ops, proxy,
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index b321604d9b..f1e105fa52 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -37,6 +37,7 @@
>   #include "qemu/range.h"
>   #include "hw/virtio/virtio-bus.h"
>   #include "qapi/visitor.h"
> +#include "sysemu/replay.h"
>   
>   #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
>   
> @@ -1760,6 +1761,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
>           proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
> +    }
> +
>       /*
>        * virtio pci bar layout used by default.
>        * subclasses can re-arrange things if needed.
> 

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

ping

On 17.05.2021 16:06, Pavel Dovgalyuk wrote:
> virtio devices support separate iothreads waiting for
> events from file descriptors. These are asynchronous
> events that can't be recorded and replayed, therefore
> this patch disables ioeventfd for all devices when
> record or replay is enabled.
> 
> Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
> ---
>   hw/s390x/virtio-ccw.c   |    6 ++++++
>   hw/virtio/virtio-mmio.c |    6 ++++++
>   hw/virtio/virtio-pci.c  |    6 ++++++
>   3 files changed, 18 insertions(+)
> 
> diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
> index 92b950e09a..bd8b9c5755 100644
> --- a/hw/s390x/virtio-ccw.c
> +++ b/hw/s390x/virtio-ccw.c
> @@ -31,6 +31,7 @@
>   #include "trace.h"
>   #include "hw/s390x/css-bridge.h"
>   #include "hw/s390x/s390-virtio-ccw.h"
> +#include "sysemu/replay.h"
>   
>   #define NR_CLASSIC_INDICATOR_BITS 64
>   
> @@ -769,6 +770,11 @@ static void virtio_ccw_device_realize(VirtioCcwDevice *dev, Error **errp)
>           dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
> +    }
> +
>       if (k->realize) {
>           k->realize(dev, &err);
>           if (err) {
> diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
> index 5952471b38..1af48a1b04 100644
> --- a/hw/virtio/virtio-mmio.c
> +++ b/hw/virtio/virtio-mmio.c
> @@ -29,6 +29,7 @@
>   #include "qemu/host-utils.h"
>   #include "qemu/module.h"
>   #include "sysemu/kvm.h"
> +#include "sysemu/replay.h"
>   #include "hw/virtio/virtio-mmio.h"
>   #include "qemu/error-report.h"
>   #include "qemu/log.h"
> @@ -740,6 +741,11 @@ static void virtio_mmio_realizefn(DeviceState *d, Error **errp)
>           proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_IOMMIO_FLAG_USE_IOEVENTFD;
> +    }
> +
>       if (proxy->legacy) {
>           memory_region_init_io(&proxy->iomem, OBJECT(d),
>                                 &virtio_legacy_mem_ops, proxy,
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index b321604d9b..f1e105fa52 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -37,6 +37,7 @@
>   #include "qemu/range.h"
>   #include "hw/virtio/virtio-bus.h"
>   #include "qapi/visitor.h"
> +#include "sysemu/replay.h"
>   
>   #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
>   
> @@ -1760,6 +1761,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
>           proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>       }
>   
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
> +    }
> +
>       /*
>        * virtio pci bar layout used by default.
>        * subclasses can re-arrange things if needed.
> 

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Paolo Bonzini]

On 24/02/21 10:56, Pavel Dovgalyuk wrote:
>>
> 
> Thanks for pointing at these machines.
> But virtio-mmio does not support ioeventfd disabling as in PCI and CCW 
> devices.
> Should this option be added first?

Yes, that was a good idea.

Paolo

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

On 11.02.2021 12:05, Paolo Bonzini wrote:
> On 11/02/21 09:56, Pavel Dovgalyuk wrote:
>>
>> That's right.
>> It seems, that ioeventfd is used in virtio-ccw, virtio-mmio, and 
>> virtio-pci. The second one is related to KVM only, right?
> 
> No, it's used for TCG as well (with ARM "-M virt" or x86 "-M microvm").

Thanks for pointing at these machines.
But virtio-mmio does not support ioeventfd disabling as in PCI and CCW 
devices.
Should this option be added first?
Or the following code means that ioeventfd is used only in KVM mode?

static bool virtio_mmio_ioeventfd_enabled(DeviceState *d)
{
     return kvm_eventfds_enabled();
}

Pavel Dovgalyuk

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Paolo Bonzini]

On 11/02/21 09:58, Alex Bennée wrote:
>> virtio devices support separate iothreads waiting for
>> events from file descriptors. These are asynchronous
>> events that can't be recorded and replayed, therefore
>> this patch disables ioeventfd for all devices when
>> record or replay is enabled.
> Does it always have to be the case? Can a virtio device forgo iothreads
> and poll events from the main loop?

Yes and in fact it's the default, but ioeventfd has the same issue I 
think even if the polling happens from the main thread.

Paolo

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Paolo Bonzini]

On 11/02/21 09:56, Pavel Dovgalyuk wrote:
> 
> That's right.
> It seems, that ioeventfd is used in virtio-ccw, virtio-mmio, and 
> virtio-pci. The second one is related to KVM only, right?

No, it's used for TCG as well (with ARM "-M virt" or x86 "-M microvm").

Paolo

> And the first one should be updated too for replay.

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Alex Bennée]

Pavel Dovgalyuk <pavel.dovgalyuk@ispras.ru> writes:

> virtio devices support separate iothreads waiting for
> events from file descriptors. These are asynchronous
> events that can't be recorded and replayed, therefore
> this patch disables ioeventfd for all devices when
> record or replay is enabled.

Does it always have to be the case? Can a virtio device forgo iothreads
and poll events from the main loop?

>
> Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
> ---
>  hw/virtio/virtio-pci.c |    6 ++++++
>  1 file changed, 6 insertions(+)
>
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index 094c36aa3e..76fbc111ea 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -37,6 +37,7 @@
>  #include "qemu/range.h"
>  #include "hw/virtio/virtio-bus.h"
>  #include "qapi/visitor.h"
> +#include "sysemu/replay.h"
>  
>  #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
>  
> @@ -1746,6 +1747,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
>          proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>      }
>  
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
> +    }
> +
>      /*
>       * virtio pci bar layout used by default.
>       * subclasses can re-arrange things if needed.

I think we need a better solution for this. Are we going to end up
sprinkling replay annotations throughout the device backends for each an
every device supported by QEMU?

This seems to be it should be the other way around. We should identify
the devices (or classes of devices) that support the logging and
consistency requirements needed for replay and mark them with a QOM
parameter. Then we can walk the tree of devices and either claim
compatibility with the request to record/replay or report to the user
"can't use record/replay because device %s is not supported".

-- 
Alex Bennée

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

On 11.02.2021 11:44, Cornelia Huck wrote:
> On Thu, 11 Feb 2021 11:34:53 +0300
> Pavel Dovgalyuk <pavel.dovgalyuk@ispras.ru> wrote:
> 
>> virtio devices support separate iothreads waiting for
>> events from file descriptors. These are asynchronous
>> events that can't be recorded and replayed, therefore
>> this patch disables ioeventfd for all devices when
>> record or replay is enabled.
>>
>> Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
>> ---
>>   hw/virtio/virtio-pci.c |    6 ++++++
>>   1 file changed, 6 insertions(+)
>>
>> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
>> index 094c36aa3e..76fbc111ea 100644
>> --- a/hw/virtio/virtio-pci.c
>> +++ b/hw/virtio/virtio-pci.c
>> @@ -37,6 +37,7 @@
>>   #include "qemu/range.h"
>>   #include "hw/virtio/virtio-bus.h"
>>   #include "qapi/visitor.h"
>> +#include "sysemu/replay.h"
>>   
>>   #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
>>   
>> @@ -1746,6 +1747,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
>>           proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>>       }
>>   
>> +    /* fd-based ioevents can't be synchronized in record/replay */
>> +    if (replay_mode != REPLAY_MODE_NONE) {
>> +        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>> +    }
>> +
>>       /*
>>        * virtio pci bar layout used by default.
>>        * subclasses can re-arrange things if needed.
>>
>>
> 
> I think you would need to do the same for any device using ioeventfd,
> no?

That's right.
It seems, that ioeventfd is used in virtio-ccw, virtio-mmio, and 
virtio-pci. The second one is related to KVM only, right?

And the first one should be updated too for replay.

Re: [PATCH] virtio: disable ioeventfd for record/replay

[Cornelia Huck]

On Thu, 11 Feb 2021 11:34:53 +0300
Pavel Dovgalyuk <pavel.dovgalyuk@ispras.ru> wrote:

> virtio devices support separate iothreads waiting for
> events from file descriptors. These are asynchronous
> events that can't be recorded and replayed, therefore
> this patch disables ioeventfd for all devices when
> record or replay is enabled.
> 
> Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
> ---
>  hw/virtio/virtio-pci.c |    6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> index 094c36aa3e..76fbc111ea 100644
> --- a/hw/virtio/virtio-pci.c
> +++ b/hw/virtio/virtio-pci.c
> @@ -37,6 +37,7 @@
>  #include "qemu/range.h"
>  #include "hw/virtio/virtio-bus.h"
>  #include "qapi/visitor.h"
> +#include "sysemu/replay.h"
>  
>  #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
>  
> @@ -1746,6 +1747,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
>          proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
>      }
>  
> +    /* fd-based ioevents can't be synchronized in record/replay */
> +    if (replay_mode != REPLAY_MODE_NONE) {
> +        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
> +    }
> +
>      /*
>       * virtio pci bar layout used by default.
>       * subclasses can re-arrange things if needed.
> 
> 

I think you would need to do the same for any device using ioeventfd,
no?

[PATCH] virtio: disable ioeventfd for record/replay

[Pavel Dovgalyuk]

virtio devices support separate iothreads waiting for
events from file descriptors. These are asynchronous
events that can't be recorded and replayed, therefore
this patch disables ioeventfd for all devices when
record or replay is enabled.

Signed-off-by: Pavel Dovgalyuk <Pavel.Dovgalyuk@ispras.ru>
---
 hw/virtio/virtio-pci.c |    6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index 094c36aa3e..76fbc111ea 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -37,6 +37,7 @@
 #include "qemu/range.h"
 #include "hw/virtio/virtio-bus.h"
 #include "qapi/visitor.h"
+#include "sysemu/replay.h"
 
 #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
 
@@ -1746,6 +1747,11 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
     }
 
+    /* fd-based ioevents can't be synchronized in record/replay */
+    if (replay_mode != REPLAY_MODE_NONE) {
+        proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
+    }
+
     /*
      * virtio pci bar layout used by default.
      * subclasses can re-arrange things if needed.