[Qemu-devel] [PATCH 0/3 v2] virtio: improve virtio devices initialization time

[Qemu-devel] [PATCH 0/3 v2] virtio: improve virtio devices initialization time

[Gal Hammer]

A bug was reported about a very slow boot time and a 100% CPU usage of
both Windows and Linux guests when running a VM with multiple
virtio-serial devices (https://bugzilla.redhat.com/1528588).

For example, running a VM with 25 virtio-serial devices, each one with
max_ports=511, could have a boot time of around 30 minutes. With this
patch (and another patch to kvm) the boot time is reduced to
approximately 3 minutes.

The patch wraps all the changes made to the Memory Regions during the
eventfd registrations in a memory regions transaction. I had to add a
cleanup callback function to the EventNotifier struct, so it will be
possible to use a transaction in the shutdown code path as well.

Gal Hammer (3):
  qemu: add a cleanup callback function to EventNotifier
  virtio: postpone the execution of event_notifier_cleanup function
  virtio: improve virtio devices initialization time

 accel/kvm/kvm-all.c           |  4 ++++
 hw/virtio/virtio-bus.c        | 19 +++++++++++--------
 hw/virtio/virtio.c            |  5 +++++
 include/qemu/event_notifier.h |  1 +
 util/event_notifier-posix.c   |  5 ++++-
 util/event_notifier-win32.c   |  2 ++
 6 files changed, 27 insertions(+), 9 deletions(-)

-- 
2.7.5

[Qemu-devel] [PATCH 1/3] qemu: add a cleanup callback function to EventNotifier

[Gal Hammer]

Adding a cleanup callback function to the EventNotifier struct
which allows users to execute event_notifier_cleanup in a
different context.

Signed-off-by: Gal Hammer <ghammer@redhat.com>
---
 include/qemu/event_notifier.h | 1 +
 util/event_notifier-posix.c   | 5 ++++-
 util/event_notifier-win32.c   | 2 ++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/include/qemu/event_notifier.h b/include/qemu/event_notifier.h
index 599c99f..b30a454 100644
--- a/include/qemu/event_notifier.h
+++ b/include/qemu/event_notifier.h
@@ -26,6 +26,7 @@ struct EventNotifier {
     int rfd;
     int wfd;
 #endif
+    void (*cleanup)(EventNotifier *);
 };
 
 typedef void EventNotifierHandler(EventNotifier *);
diff --git a/util/event_notifier-posix.c b/util/event_notifier-posix.c
index 73c4046..6525666 100644
--- a/util/event_notifier-posix.c
+++ b/util/event_notifier-posix.c
@@ -29,6 +29,7 @@ void event_notifier_init_fd(EventNotifier *e, int fd)
 {
     e->rfd = fd;
     e->wfd = fd;
+    e->cleanup = NULL;
 }
 #endif
 
@@ -65,6 +66,7 @@ int event_notifier_init(EventNotifier *e, int active)
         e->rfd = fds[0];
         e->wfd = fds[1];
     }
+    e->cleanup = NULL;
     if (active) {
         event_notifier_set(e);
     }
@@ -80,10 +82,11 @@ void event_notifier_cleanup(EventNotifier *e)
 {
     if (e->rfd != e->wfd) {
         close(e->rfd);
-        e->rfd = -1;
     }
     close(e->wfd);
+    e->rfd = -1;
     e->wfd = -1;
+    e->cleanup = NULL;
 }
 
 int event_notifier_get_fd(const EventNotifier *e)
diff --git a/util/event_notifier-win32.c b/util/event_notifier-win32.c
index 62c53b0..eff8670 100644
--- a/util/event_notifier-win32.c
+++ b/util/event_notifier-win32.c
@@ -19,6 +19,7 @@ int event_notifier_init(EventNotifier *e, int active)
 {
     e->event = CreateEvent(NULL, TRUE, FALSE, NULL);
     assert(e->event);
+    e->cleanup = NULL;
     return 0;
 }
 
@@ -26,6 +27,7 @@ void event_notifier_cleanup(EventNotifier *e)
 {
     CloseHandle(e->event);
     e->event = NULL;
+    e->cleanup = NULL;
 }
 
 HANDLE event_notifier_get_handle(EventNotifier *e)
-- 
2.7.5

[Qemu-devel] [PATCH 2/3] virtio: postpone the execution of event_notifier_cleanup function

[Gal Hammer]

Use the EventNotifier's cleanup callback function to execute the
event_notifier_cleanup function after kvm unregistered the eventfd.

This change supports running the virtio_bus_set_host_notifier
function inside a memory region transaction. Otherwise, a closed
fd is sent to kvm, which results in a failure.

Signed-off-by: Gal Hammer <ghammer@redhat.com>
---
 accel/kvm/kvm-all.c    |  4 ++++
 hw/virtio/virtio-bus.c | 19 +++++++++++--------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index f290f48..071f4f5 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -812,6 +812,10 @@ static void kvm_mem_ioeventfd_del(MemoryListener *listener,
     if (r < 0) {
         abort();
     }
+
+    if (e->cleanup) {
+        e->cleanup(e);
+    }
 }
 
 static void kvm_io_ioeventfd_add(MemoryListener *listener,
diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
index 3042232..8106346 100644
--- a/hw/virtio/virtio-bus.c
+++ b/hw/virtio/virtio-bus.c
@@ -256,6 +256,15 @@ bool virtio_bus_ioeventfd_enabled(VirtioBusState *bus)
     return k->ioeventfd_assign && k->ioeventfd_enabled(proxy);
 }
 
+static void virtio_bus_cleanup_event_notifier(EventNotifier *notifier)
+{
+    /* Test and clear notifier after disabling event,
+     * in case poll callback didn't have time to run.
+     */
+    virtio_queue_host_notifier_read(notifier);
+    event_notifier_cleanup(notifier);
+}
+
 /*
  * This function switches ioeventfd on/off in the device.
  * The caller must set or clear the handlers for the EventNotifier.
@@ -283,19 +292,13 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
         r = k->ioeventfd_assign(proxy, notifier, n, true);
         if (r < 0) {
             error_report("%s: unable to assign ioeventfd: %d", __func__, r);
-            goto cleanup_event_notifier;
+            virtio_bus_cleanup_event_notifier(notifier);
         }
-        return 0;
     } else {
+        notifier->cleanup = virtio_bus_cleanup_event_notifier;
         k->ioeventfd_assign(proxy, notifier, n, false);
     }
 
-cleanup_event_notifier:
-    /* Test and clear notifier after disabling event,
-     * in case poll callback didn't have time to run.
-     */
-    virtio_queue_host_notifier_read(notifier);
-    event_notifier_cleanup(notifier);
     return r;
 }
 
-- 
2.7.5

[Qemu-devel] [PATCH 3/3] virtio: improve virtio devices initialization time

[Gal Hammer]

The loading time of a VM is quite significant when its virtio
devices use a large amount of virt-queues (e.g. a virtio-serial
device with max_ports=511). Most of the time is spend in the
creation of all the required event notifiers (ioeventfd and memory
regions).

This patch pack all the changes to the memory regions in a
single memory transaction.

Reported-by: Sitong Liu <siliu@redhat.com>
Reported-by: Xiaoling Gao <xiagao@redhat.com>
Signed-off-by: Gal Hammer <ghammer@redhat.com>
---
 hw/virtio/virtio.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index d6002ee..3ac3491 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -2574,6 +2574,7 @@ static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
     int n, r, err;
 
+    memory_region_transaction_begin();
     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
         VirtQueue *vq = &vdev->vq[n];
         if (!virtio_queue_get_num(vdev, n)) {
@@ -2596,6 +2597,7 @@ static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
         }
         event_notifier_set(&vq->host_notifier);
     }
+    memory_region_transaction_commit();
     return 0;
 
 assign_error:
@@ -2609,6 +2611,7 @@ assign_error:
         r = virtio_bus_set_host_notifier(qbus, n, false);
         assert(r >= 0);
     }
+    memory_region_transaction_commit();
     return err;
 }
 
@@ -2625,6 +2628,7 @@ static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
     int n, r;
 
+    memory_region_transaction_begin();
     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
         VirtQueue *vq = &vdev->vq[n];
 
@@ -2635,6 +2639,7 @@ static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
         r = virtio_bus_set_host_notifier(qbus, n, false);
         assert(r >= 0);
     }
+    memory_region_transaction_commit();
 }
 
 void virtio_device_stop_ioeventfd(VirtIODevice *vdev)
-- 
2.7.5

Re: [Qemu-devel] [PATCH 0/3 v2] virtio: improve virtio devices initialization time

[Kinsella, Ray]

Hi Gal,

Brilliant - will test this in the next day or two.
Hopefully this will help resolve the issues I reported last summer.

http://lists.nongnu.org/archive/html/qemu-devel/2017-07/msg05268.html

Ray K

On 14/01/2018 10:06, Gal Hammer wrote:
> A bug was reported about a very slow boot time and a 100% CPU usage of
> both Windows and Linux guests when running a VM with multiple
> virtio-serial devices (https://bugzilla.redhat.com/1528588).
>
> For example, running a VM with 25 virtio-serial devices, each one with
> max_ports=511, could have a boot time of around 30 minutes. With this
> patch (and another patch to kvm) the boot time is reduced to
> approximately 3 minutes.
>
> The patch wraps all the changes made to the Memory Regions during the
> eventfd registrations in a memory regions transaction. I had to add a
> cleanup callback function to the EventNotifier struct, so it will be
> possible to use a transaction in the shutdown code path as well.
>
> Gal Hammer (3):
>    qemu: add a cleanup callback function to EventNotifier
>    virtio: postpone the execution of event_notifier_cleanup function
>    virtio: improve virtio devices initialization time
>
>   accel/kvm/kvm-all.c           |  4 ++++
>   hw/virtio/virtio-bus.c        | 19 +++++++++++--------
>   hw/virtio/virtio.c            |  5 +++++
>   include/qemu/event_notifier.h |  1 +
>   util/event_notifier-posix.c   |  5 ++++-
>   util/event_notifier-win32.c   |  2 ++
>   6 files changed, 27 insertions(+), 9 deletions(-)
>

Re: [Qemu-devel] [PATCH 0/3 v2] virtio: improve virtio devices initialization time

[Gal Hammer]

Hi Ray,

On Tue, Jan 16, 2018 at 5:40 PM, Kinsella, Ray <mdr@ashroe.eu> wrote:
> Hi Gal,
>
> Brilliant - will test this in the next day or two.
> Hopefully this will help resolve the issues I reported last summer.
>
> http://lists.nongnu.org/archive/html/qemu-devel/2017-07/msg05268.html

Thanks for the compliment although a little over-tuned. ;-)

I'm not sure my patch will help with your problem of multiple PCI
devices (lots of devices). My patch improves the eventfd registration
path while it seems that the source of your problem is either
limitation enforced by the PCI specifications or by the QEMU's PCI
emulation code.

> Ray K

    Gal.

>
> On 14/01/2018 10:06, Gal Hammer wrote:
>>
>> A bug was reported about a very slow boot time and a 100% CPU usage of
>> both Windows and Linux guests when running a VM with multiple
>> virtio-serial devices (https://bugzilla.redhat.com/1528588).
>>
>> For example, running a VM with 25 virtio-serial devices, each one with
>> max_ports=511, could have a boot time of around 30 minutes. With this
>> patch (and another patch to kvm) the boot time is reduced to
>> approximately 3 minutes.
>>
>> The patch wraps all the changes made to the Memory Regions during the
>> eventfd registrations in a memory regions transaction. I had to add a
>> cleanup callback function to the EventNotifier struct, so it will be
>> possible to use a transaction in the shutdown code path as well.
>>
>> Gal Hammer (3):
>>    qemu: add a cleanup callback function to EventNotifier
>>    virtio: postpone the execution of event_notifier_cleanup function
>>    virtio: improve virtio devices initialization time
>>
>>   accel/kvm/kvm-all.c           |  4 ++++
>>   hw/virtio/virtio-bus.c        | 19 +++++++++++--------
>>   hw/virtio/virtio.c            |  5 +++++
>>   include/qemu/event_notifier.h |  1 +
>>   util/event_notifier-posix.c   |  5 ++++-
>>   util/event_notifier-win32.c   |  2 ++
>>   6 files changed, 27 insertions(+), 9 deletions(-)
>>
>

Re: [Qemu-devel] [PATCH 0/3 v2] virtio: improve virtio devices initialization time

[Kinsella, Ray]

[-- Attachment #1: Type: text/plain, Size: 991 bytes --]

Hi Gal,

So the good news is that before I applied your patch I found that 
initialization time has improved.

For 128 virtio-net devices it has gone from 137s the last time I tested 
it, to 40s today.

The bad news is that for 256 virtio-net devices - it now just hangs.

However when I apply your patch - it now core dumps for 128 virtio-devices.

You can find the output here - https://pastebin.com/W4DXZ6J5.

My config is attached to this email.


On 17/01/2018 09:28, Gal Hammer wrote:
> Hi Ray,
>
> On Tue, Jan 16, 2018 at 5:40 PM, Kinsella, Ray <mdr@ashroe.eu> wrote:
>> Hi Gal,
>>
>>
>> I'm not sure my patch will help with your problem of multiple PCI
>> devices (lots of devices). My patch improves the eventfd registration
>> path while it seems that the source of your problem is either
>> limitation enforced by the PCI specifications or by the QEMU's PCI
>> emulation code.
>>

Did you read down the thread to my timings?

https://marc.info/?l=qemu-devel&m=150100585427348&w=2

[-- Attachment #2: test_128.cfg --]
[-- Type: text/plain, Size: 43460 bytes --]


[device "pxb"]
driver = "pxb-pcie"
bus = "pcie.0"
bus_nr = "0x80"

[device "io0"]
driver = "ioh3420"
multifunction = "on"
addr = "0.0"
chassis = "0"
bus = "pxb"

[chardev "vhost0"]
backend = "socket"
path = "/tmp/vpp/vhost-user0.sock"
server = "off"

[netdev "netdev0"]
type = "vhost-user"
chardev = "vhost0"

[device "virtio0"]
driver = "virtio-net-pci"
netdev = "netdev0"
bus = "io0"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost1"]
backend = "socket"
path = "/tmp/vpp/vhost-user1.sock"
server = "off"

[netdev "netdev1"]
type = "vhost-user"
chardev = "vhost1"

[device "virtio1"]
driver = "virtio-net-pci"
netdev = "netdev1"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost2"]
backend = "socket"
path = "/tmp/vpp/vhost-user2.sock"
server = "off"

[netdev "netdev2"]
type = "vhost-user"
chardev = "vhost2"

[device "virtio2"]
driver = "virtio-net-pci"
netdev = "netdev2"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost3"]
backend = "socket"
path = "/tmp/vpp/vhost-user3.sock"
server = "off"

[netdev "netdev3"]
type = "vhost-user"
chardev = "vhost3"

[device "virtio3"]
driver = "virtio-net-pci"
netdev = "netdev3"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost4"]
backend = "socket"
path = "/tmp/vpp/vhost-user4.sock"
server = "off"

[netdev "netdev4"]
type = "vhost-user"
chardev = "vhost4"

[device "virtio4"]
driver = "virtio-net-pci"
netdev = "netdev4"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost5"]
backend = "socket"
path = "/tmp/vpp/vhost-user5.sock"
server = "off"

[netdev "netdev5"]
type = "vhost-user"
chardev = "vhost5"

[device "virtio5"]
driver = "virtio-net-pci"
netdev = "netdev5"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost6"]
backend = "socket"
path = "/tmp/vpp/vhost-user6.sock"
server = "off"

[netdev "netdev6"]
type = "vhost-user"
chardev = "vhost6"

[device "virtio6"]
driver = "virtio-net-pci"
netdev = "netdev6"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost7"]
backend = "socket"
path = "/tmp/vpp/vhost-user7.sock"
server = "off"

[netdev "netdev7"]
type = "vhost-user"
chardev = "vhost7"

[device "virtio7"]
driver = "virtio-net-pci"
netdev = "netdev7"
bus = "io0"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io1"]
driver = "ioh3420"
addr = "0.1"
chassis = "1"
bus = "pxb"

[chardev "vhost8"]
backend = "socket"
path = "/tmp/vpp/vhost-user8.sock"
server = "off"

[netdev "netdev8"]
type = "vhost-user"
chardev = "vhost8"

[device "virtio8"]
driver = "virtio-net-pci"
netdev = "netdev8"
bus = "io1"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost9"]
backend = "socket"
path = "/tmp/vpp/vhost-user9.sock"
server = "off"

[netdev "netdev9"]
type = "vhost-user"
chardev = "vhost9"

[device "virtio9"]
driver = "virtio-net-pci"
netdev = "netdev9"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost10"]
backend = "socket"
path = "/tmp/vpp/vhost-user10.sock"
server = "off"

[netdev "netdev10"]
type = "vhost-user"
chardev = "vhost10"

[device "virtio10"]
driver = "virtio-net-pci"
netdev = "netdev10"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost11"]
backend = "socket"
path = "/tmp/vpp/vhost-user11.sock"
server = "off"

[netdev "netdev11"]
type = "vhost-user"
chardev = "vhost11"

[device "virtio11"]
driver = "virtio-net-pci"
netdev = "netdev11"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost12"]
backend = "socket"
path = "/tmp/vpp/vhost-user12.sock"
server = "off"

[netdev "netdev12"]
type = "vhost-user"
chardev = "vhost12"

[device "virtio12"]
driver = "virtio-net-pci"
netdev = "netdev12"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost13"]
backend = "socket"
path = "/tmp/vpp/vhost-user13.sock"
server = "off"

[netdev "netdev13"]
type = "vhost-user"
chardev = "vhost13"

[device "virtio13"]
driver = "virtio-net-pci"
netdev = "netdev13"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost14"]
backend = "socket"
path = "/tmp/vpp/vhost-user14.sock"
server = "off"

[netdev "netdev14"]
type = "vhost-user"
chardev = "vhost14"

[device "virtio14"]
driver = "virtio-net-pci"
netdev = "netdev14"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost15"]
backend = "socket"
path = "/tmp/vpp/vhost-user15.sock"
server = "off"

[netdev "netdev15"]
type = "vhost-user"
chardev = "vhost15"

[device "virtio15"]
driver = "virtio-net-pci"
netdev = "netdev15"
bus = "io1"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io2"]
driver = "ioh3420"
addr = "0.2"
chassis = "2"
bus = "pxb"

[chardev "vhost16"]
backend = "socket"
path = "/tmp/vpp/vhost-user16.sock"
server = "off"

[netdev "netdev16"]
type = "vhost-user"
chardev = "vhost16"

[device "virtio16"]
driver = "virtio-net-pci"
netdev = "netdev16"
bus = "io2"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost17"]
backend = "socket"
path = "/tmp/vpp/vhost-user17.sock"
server = "off"

[netdev "netdev17"]
type = "vhost-user"
chardev = "vhost17"

[device "virtio17"]
driver = "virtio-net-pci"
netdev = "netdev17"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost18"]
backend = "socket"
path = "/tmp/vpp/vhost-user18.sock"
server = "off"

[netdev "netdev18"]
type = "vhost-user"
chardev = "vhost18"

[device "virtio18"]
driver = "virtio-net-pci"
netdev = "netdev18"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost19"]
backend = "socket"
path = "/tmp/vpp/vhost-user19.sock"
server = "off"

[netdev "netdev19"]
type = "vhost-user"
chardev = "vhost19"

[device "virtio19"]
driver = "virtio-net-pci"
netdev = "netdev19"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost20"]
backend = "socket"
path = "/tmp/vpp/vhost-user20.sock"
server = "off"

[netdev "netdev20"]
type = "vhost-user"
chardev = "vhost20"

[device "virtio20"]
driver = "virtio-net-pci"
netdev = "netdev20"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost21"]
backend = "socket"
path = "/tmp/vpp/vhost-user21.sock"
server = "off"

[netdev "netdev21"]
type = "vhost-user"
chardev = "vhost21"

[device "virtio21"]
driver = "virtio-net-pci"
netdev = "netdev21"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost22"]
backend = "socket"
path = "/tmp/vpp/vhost-user22.sock"
server = "off"

[netdev "netdev22"]
type = "vhost-user"
chardev = "vhost22"

[device "virtio22"]
driver = "virtio-net-pci"
netdev = "netdev22"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost23"]
backend = "socket"
path = "/tmp/vpp/vhost-user23.sock"
server = "off"

[netdev "netdev23"]
type = "vhost-user"
chardev = "vhost23"

[device "virtio23"]
driver = "virtio-net-pci"
netdev = "netdev23"
bus = "io2"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io3"]
driver = "ioh3420"
addr = "0.3"
chassis = "3"
bus = "pxb"

[chardev "vhost24"]
backend = "socket"
path = "/tmp/vpp/vhost-user24.sock"
server = "off"

[netdev "netdev24"]
type = "vhost-user"
chardev = "vhost24"

[device "virtio24"]
driver = "virtio-net-pci"
netdev = "netdev24"
bus = "io3"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost25"]
backend = "socket"
path = "/tmp/vpp/vhost-user25.sock"
server = "off"

[netdev "netdev25"]
type = "vhost-user"
chardev = "vhost25"

[device "virtio25"]
driver = "virtio-net-pci"
netdev = "netdev25"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost26"]
backend = "socket"
path = "/tmp/vpp/vhost-user26.sock"
server = "off"

[netdev "netdev26"]
type = "vhost-user"
chardev = "vhost26"

[device "virtio26"]
driver = "virtio-net-pci"
netdev = "netdev26"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost27"]
backend = "socket"
path = "/tmp/vpp/vhost-user27.sock"
server = "off"

[netdev "netdev27"]
type = "vhost-user"
chardev = "vhost27"

[device "virtio27"]
driver = "virtio-net-pci"
netdev = "netdev27"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost28"]
backend = "socket"
path = "/tmp/vpp/vhost-user28.sock"
server = "off"

[netdev "netdev28"]
type = "vhost-user"
chardev = "vhost28"

[device "virtio28"]
driver = "virtio-net-pci"
netdev = "netdev28"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost29"]
backend = "socket"
path = "/tmp/vpp/vhost-user29.sock"
server = "off"

[netdev "netdev29"]
type = "vhost-user"
chardev = "vhost29"

[device "virtio29"]
driver = "virtio-net-pci"
netdev = "netdev29"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost30"]
backend = "socket"
path = "/tmp/vpp/vhost-user30.sock"
server = "off"

[netdev "netdev30"]
type = "vhost-user"
chardev = "vhost30"

[device "virtio30"]
driver = "virtio-net-pci"
netdev = "netdev30"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost31"]
backend = "socket"
path = "/tmp/vpp/vhost-user31.sock"
server = "off"

[netdev "netdev31"]
type = "vhost-user"
chardev = "vhost31"

[device "virtio31"]
driver = "virtio-net-pci"
netdev = "netdev31"
bus = "io3"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io4"]
driver = "ioh3420"
addr = "0.4"
chassis = "4"
bus = "pxb"

[chardev "vhost32"]
backend = "socket"
path = "/tmp/vpp/vhost-user32.sock"
server = "off"

[netdev "netdev32"]
type = "vhost-user"
chardev = "vhost32"

[device "virtio32"]
driver = "virtio-net-pci"
netdev = "netdev32"
bus = "io4"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost33"]
backend = "socket"
path = "/tmp/vpp/vhost-user33.sock"
server = "off"

[netdev "netdev33"]
type = "vhost-user"
chardev = "vhost33"

[device "virtio33"]
driver = "virtio-net-pci"
netdev = "netdev33"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost34"]
backend = "socket"
path = "/tmp/vpp/vhost-user34.sock"
server = "off"

[netdev "netdev34"]
type = "vhost-user"
chardev = "vhost34"

[device "virtio34"]
driver = "virtio-net-pci"
netdev = "netdev34"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost35"]
backend = "socket"
path = "/tmp/vpp/vhost-user35.sock"
server = "off"

[netdev "netdev35"]
type = "vhost-user"
chardev = "vhost35"

[device "virtio35"]
driver = "virtio-net-pci"
netdev = "netdev35"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost36"]
backend = "socket"
path = "/tmp/vpp/vhost-user36.sock"
server = "off"

[netdev "netdev36"]
type = "vhost-user"
chardev = "vhost36"

[device "virtio36"]
driver = "virtio-net-pci"
netdev = "netdev36"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost37"]
backend = "socket"
path = "/tmp/vpp/vhost-user37.sock"
server = "off"

[netdev "netdev37"]
type = "vhost-user"
chardev = "vhost37"

[device "virtio37"]
driver = "virtio-net-pci"
netdev = "netdev37"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost38"]
backend = "socket"
path = "/tmp/vpp/vhost-user38.sock"
server = "off"

[netdev "netdev38"]
type = "vhost-user"
chardev = "vhost38"

[device "virtio38"]
driver = "virtio-net-pci"
netdev = "netdev38"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost39"]
backend = "socket"
path = "/tmp/vpp/vhost-user39.sock"
server = "off"

[netdev "netdev39"]
type = "vhost-user"
chardev = "vhost39"

[device "virtio39"]
driver = "virtio-net-pci"
netdev = "netdev39"
bus = "io4"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io5"]
driver = "ioh3420"
addr = "0.5"
chassis = "5"
bus = "pxb"

[chardev "vhost40"]
backend = "socket"
path = "/tmp/vpp/vhost-user40.sock"
server = "off"

[netdev "netdev40"]
type = "vhost-user"
chardev = "vhost40"

[device "virtio40"]
driver = "virtio-net-pci"
netdev = "netdev40"
bus = "io5"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost41"]
backend = "socket"
path = "/tmp/vpp/vhost-user41.sock"
server = "off"

[netdev "netdev41"]
type = "vhost-user"
chardev = "vhost41"

[device "virtio41"]
driver = "virtio-net-pci"
netdev = "netdev41"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost42"]
backend = "socket"
path = "/tmp/vpp/vhost-user42.sock"
server = "off"

[netdev "netdev42"]
type = "vhost-user"
chardev = "vhost42"

[device "virtio42"]
driver = "virtio-net-pci"
netdev = "netdev42"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost43"]
backend = "socket"
path = "/tmp/vpp/vhost-user43.sock"
server = "off"

[netdev "netdev43"]
type = "vhost-user"
chardev = "vhost43"

[device "virtio43"]
driver = "virtio-net-pci"
netdev = "netdev43"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost44"]
backend = "socket"
path = "/tmp/vpp/vhost-user44.sock"
server = "off"

[netdev "netdev44"]
type = "vhost-user"
chardev = "vhost44"

[device "virtio44"]
driver = "virtio-net-pci"
netdev = "netdev44"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost45"]
backend = "socket"
path = "/tmp/vpp/vhost-user45.sock"
server = "off"

[netdev "netdev45"]
type = "vhost-user"
chardev = "vhost45"

[device "virtio45"]
driver = "virtio-net-pci"
netdev = "netdev45"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost46"]
backend = "socket"
path = "/tmp/vpp/vhost-user46.sock"
server = "off"

[netdev "netdev46"]
type = "vhost-user"
chardev = "vhost46"

[device "virtio46"]
driver = "virtio-net-pci"
netdev = "netdev46"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost47"]
backend = "socket"
path = "/tmp/vpp/vhost-user47.sock"
server = "off"

[netdev "netdev47"]
type = "vhost-user"
chardev = "vhost47"

[device "virtio47"]
driver = "virtio-net-pci"
netdev = "netdev47"
bus = "io5"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io6"]
driver = "ioh3420"
addr = "0.6"
chassis = "6"
bus = "pxb"

[chardev "vhost48"]
backend = "socket"
path = "/tmp/vpp/vhost-user48.sock"
server = "off"

[netdev "netdev48"]
type = "vhost-user"
chardev = "vhost48"

[device "virtio48"]
driver = "virtio-net-pci"
netdev = "netdev48"
bus = "io6"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost49"]
backend = "socket"
path = "/tmp/vpp/vhost-user49.sock"
server = "off"

[netdev "netdev49"]
type = "vhost-user"
chardev = "vhost49"

[device "virtio49"]
driver = "virtio-net-pci"
netdev = "netdev49"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost50"]
backend = "socket"
path = "/tmp/vpp/vhost-user50.sock"
server = "off"

[netdev "netdev50"]
type = "vhost-user"
chardev = "vhost50"

[device "virtio50"]
driver = "virtio-net-pci"
netdev = "netdev50"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost51"]
backend = "socket"
path = "/tmp/vpp/vhost-user51.sock"
server = "off"

[netdev "netdev51"]
type = "vhost-user"
chardev = "vhost51"

[device "virtio51"]
driver = "virtio-net-pci"
netdev = "netdev51"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost52"]
backend = "socket"
path = "/tmp/vpp/vhost-user52.sock"
server = "off"

[netdev "netdev52"]
type = "vhost-user"
chardev = "vhost52"

[device "virtio52"]
driver = "virtio-net-pci"
netdev = "netdev52"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost53"]
backend = "socket"
path = "/tmp/vpp/vhost-user53.sock"
server = "off"

[netdev "netdev53"]
type = "vhost-user"
chardev = "vhost53"

[device "virtio53"]
driver = "virtio-net-pci"
netdev = "netdev53"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost54"]
backend = "socket"
path = "/tmp/vpp/vhost-user54.sock"
server = "off"

[netdev "netdev54"]
type = "vhost-user"
chardev = "vhost54"

[device "virtio54"]
driver = "virtio-net-pci"
netdev = "netdev54"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost55"]
backend = "socket"
path = "/tmp/vpp/vhost-user55.sock"
server = "off"

[netdev "netdev55"]
type = "vhost-user"
chardev = "vhost55"

[device "virtio55"]
driver = "virtio-net-pci"
netdev = "netdev55"
bus = "io6"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io7"]
driver = "ioh3420"
addr = "0.7"
chassis = "7"
bus = "pxb"

[chardev "vhost56"]
backend = "socket"
path = "/tmp/vpp/vhost-user56.sock"
server = "off"

[netdev "netdev56"]
type = "vhost-user"
chardev = "vhost56"

[device "virtio56"]
driver = "virtio-net-pci"
netdev = "netdev56"
bus = "io7"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost57"]
backend = "socket"
path = "/tmp/vpp/vhost-user57.sock"
server = "off"

[netdev "netdev57"]
type = "vhost-user"
chardev = "vhost57"

[device "virtio57"]
driver = "virtio-net-pci"
netdev = "netdev57"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost58"]
backend = "socket"
path = "/tmp/vpp/vhost-user58.sock"
server = "off"

[netdev "netdev58"]
type = "vhost-user"
chardev = "vhost58"

[device "virtio58"]
driver = "virtio-net-pci"
netdev = "netdev58"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost59"]
backend = "socket"
path = "/tmp/vpp/vhost-user59.sock"
server = "off"

[netdev "netdev59"]
type = "vhost-user"
chardev = "vhost59"

[device "virtio59"]
driver = "virtio-net-pci"
netdev = "netdev59"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost60"]
backend = "socket"
path = "/tmp/vpp/vhost-user60.sock"
server = "off"

[netdev "netdev60"]
type = "vhost-user"
chardev = "vhost60"

[device "virtio60"]
driver = "virtio-net-pci"
netdev = "netdev60"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost61"]
backend = "socket"
path = "/tmp/vpp/vhost-user61.sock"
server = "off"

[netdev "netdev61"]
type = "vhost-user"
chardev = "vhost61"

[device "virtio61"]
driver = "virtio-net-pci"
netdev = "netdev61"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost62"]
backend = "socket"
path = "/tmp/vpp/vhost-user62.sock"
server = "off"

[netdev "netdev62"]
type = "vhost-user"
chardev = "vhost62"

[device "virtio62"]
driver = "virtio-net-pci"
netdev = "netdev62"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost63"]
backend = "socket"
path = "/tmp/vpp/vhost-user63.sock"
server = "off"

[netdev "netdev63"]
type = "vhost-user"
chardev = "vhost63"

[device "virtio63"]
driver = "virtio-net-pci"
netdev = "netdev63"
bus = "io7"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io8"]
driver = "ioh3420"
multifunction = "on"
addr = "1.0"
chassis = "8"
bus = "pxb"

[chardev "vhost64"]
backend = "socket"
path = "/tmp/vpp/vhost-user64.sock"
server = "off"

[netdev "netdev64"]
type = "vhost-user"
chardev = "vhost64"

[device "virtio64"]
driver = "virtio-net-pci"
netdev = "netdev64"
bus = "io8"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost65"]
backend = "socket"
path = "/tmp/vpp/vhost-user65.sock"
server = "off"

[netdev "netdev65"]
type = "vhost-user"
chardev = "vhost65"

[device "virtio65"]
driver = "virtio-net-pci"
netdev = "netdev65"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost66"]
backend = "socket"
path = "/tmp/vpp/vhost-user66.sock"
server = "off"

[netdev "netdev66"]
type = "vhost-user"
chardev = "vhost66"

[device "virtio66"]
driver = "virtio-net-pci"
netdev = "netdev66"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost67"]
backend = "socket"
path = "/tmp/vpp/vhost-user67.sock"
server = "off"

[netdev "netdev67"]
type = "vhost-user"
chardev = "vhost67"

[device "virtio67"]
driver = "virtio-net-pci"
netdev = "netdev67"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost68"]
backend = "socket"
path = "/tmp/vpp/vhost-user68.sock"
server = "off"

[netdev "netdev68"]
type = "vhost-user"
chardev = "vhost68"

[device "virtio68"]
driver = "virtio-net-pci"
netdev = "netdev68"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost69"]
backend = "socket"
path = "/tmp/vpp/vhost-user69.sock"
server = "off"

[netdev "netdev69"]
type = "vhost-user"
chardev = "vhost69"

[device "virtio69"]
driver = "virtio-net-pci"
netdev = "netdev69"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost70"]
backend = "socket"
path = "/tmp/vpp/vhost-user70.sock"
server = "off"

[netdev "netdev70"]
type = "vhost-user"
chardev = "vhost70"

[device "virtio70"]
driver = "virtio-net-pci"
netdev = "netdev70"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost71"]
backend = "socket"
path = "/tmp/vpp/vhost-user71.sock"
server = "off"

[netdev "netdev71"]
type = "vhost-user"
chardev = "vhost71"

[device "virtio71"]
driver = "virtio-net-pci"
netdev = "netdev71"
bus = "io8"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io9"]
driver = "ioh3420"
addr = "1.1"
chassis = "9"
bus = "pxb"

[chardev "vhost72"]
backend = "socket"
path = "/tmp/vpp/vhost-user72.sock"
server = "off"

[netdev "netdev72"]
type = "vhost-user"
chardev = "vhost72"

[device "virtio72"]
driver = "virtio-net-pci"
netdev = "netdev72"
bus = "io9"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost73"]
backend = "socket"
path = "/tmp/vpp/vhost-user73.sock"
server = "off"

[netdev "netdev73"]
type = "vhost-user"
chardev = "vhost73"

[device "virtio73"]
driver = "virtio-net-pci"
netdev = "netdev73"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost74"]
backend = "socket"
path = "/tmp/vpp/vhost-user74.sock"
server = "off"

[netdev "netdev74"]
type = "vhost-user"
chardev = "vhost74"

[device "virtio74"]
driver = "virtio-net-pci"
netdev = "netdev74"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost75"]
backend = "socket"
path = "/tmp/vpp/vhost-user75.sock"
server = "off"

[netdev "netdev75"]
type = "vhost-user"
chardev = "vhost75"

[device "virtio75"]
driver = "virtio-net-pci"
netdev = "netdev75"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost76"]
backend = "socket"
path = "/tmp/vpp/vhost-user76.sock"
server = "off"

[netdev "netdev76"]
type = "vhost-user"
chardev = "vhost76"

[device "virtio76"]
driver = "virtio-net-pci"
netdev = "netdev76"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost77"]
backend = "socket"
path = "/tmp/vpp/vhost-user77.sock"
server = "off"

[netdev "netdev77"]
type = "vhost-user"
chardev = "vhost77"

[device "virtio77"]
driver = "virtio-net-pci"
netdev = "netdev77"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost78"]
backend = "socket"
path = "/tmp/vpp/vhost-user78.sock"
server = "off"

[netdev "netdev78"]
type = "vhost-user"
chardev = "vhost78"

[device "virtio78"]
driver = "virtio-net-pci"
netdev = "netdev78"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost79"]
backend = "socket"
path = "/tmp/vpp/vhost-user79.sock"
server = "off"

[netdev "netdev79"]
type = "vhost-user"
chardev = "vhost79"

[device "virtio79"]
driver = "virtio-net-pci"
netdev = "netdev79"
bus = "io9"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io10"]
driver = "ioh3420"
addr = "1.2"
chassis = "10"
bus = "pxb"

[chardev "vhost80"]
backend = "socket"
path = "/tmp/vpp/vhost-user80.sock"
server = "off"

[netdev "netdev80"]
type = "vhost-user"
chardev = "vhost80"

[device "virtio80"]
driver = "virtio-net-pci"
netdev = "netdev80"
bus = "io10"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost81"]
backend = "socket"
path = "/tmp/vpp/vhost-user81.sock"
server = "off"

[netdev "netdev81"]
type = "vhost-user"
chardev = "vhost81"

[device "virtio81"]
driver = "virtio-net-pci"
netdev = "netdev81"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost82"]
backend = "socket"
path = "/tmp/vpp/vhost-user82.sock"
server = "off"

[netdev "netdev82"]
type = "vhost-user"
chardev = "vhost82"

[device "virtio82"]
driver = "virtio-net-pci"
netdev = "netdev82"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost83"]
backend = "socket"
path = "/tmp/vpp/vhost-user83.sock"
server = "off"

[netdev "netdev83"]
type = "vhost-user"
chardev = "vhost83"

[device "virtio83"]
driver = "virtio-net-pci"
netdev = "netdev83"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost84"]
backend = "socket"
path = "/tmp/vpp/vhost-user84.sock"
server = "off"

[netdev "netdev84"]
type = "vhost-user"
chardev = "vhost84"

[device "virtio84"]
driver = "virtio-net-pci"
netdev = "netdev84"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost85"]
backend = "socket"
path = "/tmp/vpp/vhost-user85.sock"
server = "off"

[netdev "netdev85"]
type = "vhost-user"
chardev = "vhost85"

[device "virtio85"]
driver = "virtio-net-pci"
netdev = "netdev85"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost86"]
backend = "socket"
path = "/tmp/vpp/vhost-user86.sock"
server = "off"

[netdev "netdev86"]
type = "vhost-user"
chardev = "vhost86"

[device "virtio86"]
driver = "virtio-net-pci"
netdev = "netdev86"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost87"]
backend = "socket"
path = "/tmp/vpp/vhost-user87.sock"
server = "off"

[netdev "netdev87"]
type = "vhost-user"
chardev = "vhost87"

[device "virtio87"]
driver = "virtio-net-pci"
netdev = "netdev87"
bus = "io10"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io11"]
driver = "ioh3420"
addr = "1.3"
chassis = "11"
bus = "pxb"

[chardev "vhost88"]
backend = "socket"
path = "/tmp/vpp/vhost-user88.sock"
server = "off"

[netdev "netdev88"]
type = "vhost-user"
chardev = "vhost88"

[device "virtio88"]
driver = "virtio-net-pci"
netdev = "netdev88"
bus = "io11"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost89"]
backend = "socket"
path = "/tmp/vpp/vhost-user89.sock"
server = "off"

[netdev "netdev89"]
type = "vhost-user"
chardev = "vhost89"

[device "virtio89"]
driver = "virtio-net-pci"
netdev = "netdev89"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost90"]
backend = "socket"
path = "/tmp/vpp/vhost-user90.sock"
server = "off"

[netdev "netdev90"]
type = "vhost-user"
chardev = "vhost90"

[device "virtio90"]
driver = "virtio-net-pci"
netdev = "netdev90"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost91"]
backend = "socket"
path = "/tmp/vpp/vhost-user91.sock"
server = "off"

[netdev "netdev91"]
type = "vhost-user"
chardev = "vhost91"

[device "virtio91"]
driver = "virtio-net-pci"
netdev = "netdev91"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost92"]
backend = "socket"
path = "/tmp/vpp/vhost-user92.sock"
server = "off"

[netdev "netdev92"]
type = "vhost-user"
chardev = "vhost92"

[device "virtio92"]
driver = "virtio-net-pci"
netdev = "netdev92"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost93"]
backend = "socket"
path = "/tmp/vpp/vhost-user93.sock"
server = "off"

[netdev "netdev93"]
type = "vhost-user"
chardev = "vhost93"

[device "virtio93"]
driver = "virtio-net-pci"
netdev = "netdev93"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost94"]
backend = "socket"
path = "/tmp/vpp/vhost-user94.sock"
server = "off"

[netdev "netdev94"]
type = "vhost-user"
chardev = "vhost94"

[device "virtio94"]
driver = "virtio-net-pci"
netdev = "netdev94"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost95"]
backend = "socket"
path = "/tmp/vpp/vhost-user95.sock"
server = "off"

[netdev "netdev95"]
type = "vhost-user"
chardev = "vhost95"

[device "virtio95"]
driver = "virtio-net-pci"
netdev = "netdev95"
bus = "io11"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io12"]
driver = "ioh3420"
addr = "1.4"
chassis = "12"
bus = "pxb"

[chardev "vhost96"]
backend = "socket"
path = "/tmp/vpp/vhost-user96.sock"
server = "off"

[netdev "netdev96"]
type = "vhost-user"
chardev = "vhost96"

[device "virtio96"]
driver = "virtio-net-pci"
netdev = "netdev96"
bus = "io12"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost97"]
backend = "socket"
path = "/tmp/vpp/vhost-user97.sock"
server = "off"

[netdev "netdev97"]
type = "vhost-user"
chardev = "vhost97"

[device "virtio97"]
driver = "virtio-net-pci"
netdev = "netdev97"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost98"]
backend = "socket"
path = "/tmp/vpp/vhost-user98.sock"
server = "off"

[netdev "netdev98"]
type = "vhost-user"
chardev = "vhost98"

[device "virtio98"]
driver = "virtio-net-pci"
netdev = "netdev98"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost99"]
backend = "socket"
path = "/tmp/vpp/vhost-user99.sock"
server = "off"

[netdev "netdev99"]
type = "vhost-user"
chardev = "vhost99"

[device "virtio99"]
driver = "virtio-net-pci"
netdev = "netdev99"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost100"]
backend = "socket"
path = "/tmp/vpp/vhost-user100.sock"
server = "off"

[netdev "netdev100"]
type = "vhost-user"
chardev = "vhost100"

[device "virtio100"]
driver = "virtio-net-pci"
netdev = "netdev100"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost101"]
backend = "socket"
path = "/tmp/vpp/vhost-user101.sock"
server = "off"

[netdev "netdev101"]
type = "vhost-user"
chardev = "vhost101"

[device "virtio101"]
driver = "virtio-net-pci"
netdev = "netdev101"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost102"]
backend = "socket"
path = "/tmp/vpp/vhost-user102.sock"
server = "off"

[netdev "netdev102"]
type = "vhost-user"
chardev = "vhost102"

[device "virtio102"]
driver = "virtio-net-pci"
netdev = "netdev102"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost103"]
backend = "socket"
path = "/tmp/vpp/vhost-user103.sock"
server = "off"

[netdev "netdev103"]
type = "vhost-user"
chardev = "vhost103"

[device "virtio103"]
driver = "virtio-net-pci"
netdev = "netdev103"
bus = "io12"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io13"]
driver = "ioh3420"
addr = "1.5"
chassis = "13"
bus = "pxb"

[chardev "vhost104"]
backend = "socket"
path = "/tmp/vpp/vhost-user104.sock"
server = "off"

[netdev "netdev104"]
type = "vhost-user"
chardev = "vhost104"

[device "virtio104"]
driver = "virtio-net-pci"
netdev = "netdev104"
bus = "io13"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost105"]
backend = "socket"
path = "/tmp/vpp/vhost-user105.sock"
server = "off"

[netdev "netdev105"]
type = "vhost-user"
chardev = "vhost105"

[device "virtio105"]
driver = "virtio-net-pci"
netdev = "netdev105"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost106"]
backend = "socket"
path = "/tmp/vpp/vhost-user106.sock"
server = "off"

[netdev "netdev106"]
type = "vhost-user"
chardev = "vhost106"

[device "virtio106"]
driver = "virtio-net-pci"
netdev = "netdev106"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost107"]
backend = "socket"
path = "/tmp/vpp/vhost-user107.sock"
server = "off"

[netdev "netdev107"]
type = "vhost-user"
chardev = "vhost107"

[device "virtio107"]
driver = "virtio-net-pci"
netdev = "netdev107"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost108"]
backend = "socket"
path = "/tmp/vpp/vhost-user108.sock"
server = "off"

[netdev "netdev108"]
type = "vhost-user"
chardev = "vhost108"

[device "virtio108"]
driver = "virtio-net-pci"
netdev = "netdev108"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost109"]
backend = "socket"
path = "/tmp/vpp/vhost-user109.sock"
server = "off"

[netdev "netdev109"]
type = "vhost-user"
chardev = "vhost109"

[device "virtio109"]
driver = "virtio-net-pci"
netdev = "netdev109"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost110"]
backend = "socket"
path = "/tmp/vpp/vhost-user110.sock"
server = "off"

[netdev "netdev110"]
type = "vhost-user"
chardev = "vhost110"

[device "virtio110"]
driver = "virtio-net-pci"
netdev = "netdev110"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost111"]
backend = "socket"
path = "/tmp/vpp/vhost-user111.sock"
server = "off"

[netdev "netdev111"]
type = "vhost-user"
chardev = "vhost111"

[device "virtio111"]
driver = "virtio-net-pci"
netdev = "netdev111"
bus = "io13"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io14"]
driver = "ioh3420"
addr = "1.6"
chassis = "14"
bus = "pxb"

[chardev "vhost112"]
backend = "socket"
path = "/tmp/vpp/vhost-user112.sock"
server = "off"

[netdev "netdev112"]
type = "vhost-user"
chardev = "vhost112"

[device "virtio112"]
driver = "virtio-net-pci"
netdev = "netdev112"
bus = "io14"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost113"]
backend = "socket"
path = "/tmp/vpp/vhost-user113.sock"
server = "off"

[netdev "netdev113"]
type = "vhost-user"
chardev = "vhost113"

[device "virtio113"]
driver = "virtio-net-pci"
netdev = "netdev113"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost114"]
backend = "socket"
path = "/tmp/vpp/vhost-user114.sock"
server = "off"

[netdev "netdev114"]
type = "vhost-user"
chardev = "vhost114"

[device "virtio114"]
driver = "virtio-net-pci"
netdev = "netdev114"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost115"]
backend = "socket"
path = "/tmp/vpp/vhost-user115.sock"
server = "off"

[netdev "netdev115"]
type = "vhost-user"
chardev = "vhost115"

[device "virtio115"]
driver = "virtio-net-pci"
netdev = "netdev115"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost116"]
backend = "socket"
path = "/tmp/vpp/vhost-user116.sock"
server = "off"

[netdev "netdev116"]
type = "vhost-user"
chardev = "vhost116"

[device "virtio116"]
driver = "virtio-net-pci"
netdev = "netdev116"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost117"]
backend = "socket"
path = "/tmp/vpp/vhost-user117.sock"
server = "off"

[netdev "netdev117"]
type = "vhost-user"
chardev = "vhost117"

[device "virtio117"]
driver = "virtio-net-pci"
netdev = "netdev117"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost118"]
backend = "socket"
path = "/tmp/vpp/vhost-user118.sock"
server = "off"

[netdev "netdev118"]
type = "vhost-user"
chardev = "vhost118"

[device "virtio118"]
driver = "virtio-net-pci"
netdev = "netdev118"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost119"]
backend = "socket"
path = "/tmp/vpp/vhost-user119.sock"
server = "off"

[netdev "netdev119"]
type = "vhost-user"
chardev = "vhost119"

[device "virtio119"]
driver = "virtio-net-pci"
netdev = "netdev119"
bus = "io14"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io15"]
driver = "ioh3420"
addr = "1.7"
chassis = "15"
bus = "pxb"

[chardev "vhost120"]
backend = "socket"
path = "/tmp/vpp/vhost-user120.sock"
server = "off"

[netdev "netdev120"]
type = "vhost-user"
chardev = "vhost120"

[device "virtio120"]
driver = "virtio-net-pci"
netdev = "netdev120"
bus = "io15"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

[chardev "vhost121"]
backend = "socket"
path = "/tmp/vpp/vhost-user121.sock"
server = "off"

[netdev "netdev121"]
type = "vhost-user"
chardev = "vhost121"

[device "virtio121"]
driver = "virtio-net-pci"
netdev = "netdev121"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.1"

[chardev "vhost122"]
backend = "socket"
path = "/tmp/vpp/vhost-user122.sock"
server = "off"

[netdev "netdev122"]
type = "vhost-user"
chardev = "vhost122"

[device "virtio122"]
driver = "virtio-net-pci"
netdev = "netdev122"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.2"

[chardev "vhost123"]
backend = "socket"
path = "/tmp/vpp/vhost-user123.sock"
server = "off"

[netdev "netdev123"]
type = "vhost-user"
chardev = "vhost123"

[device "virtio123"]
driver = "virtio-net-pci"
netdev = "netdev123"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.3"

[chardev "vhost124"]
backend = "socket"
path = "/tmp/vpp/vhost-user124.sock"
server = "off"

[netdev "netdev124"]
type = "vhost-user"
chardev = "vhost124"

[device "virtio124"]
driver = "virtio-net-pci"
netdev = "netdev124"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.4"

[chardev "vhost125"]
backend = "socket"
path = "/tmp/vpp/vhost-user125.sock"
server = "off"

[netdev "netdev125"]
type = "vhost-user"
chardev = "vhost125"

[device "virtio125"]
driver = "virtio-net-pci"
netdev = "netdev125"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.5"

[chardev "vhost126"]
backend = "socket"
path = "/tmp/vpp/vhost-user126.sock"
server = "off"

[netdev "netdev126"]
type = "vhost-user"
chardev = "vhost126"

[device "virtio126"]
driver = "virtio-net-pci"
netdev = "netdev126"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.6"

[chardev "vhost127"]
backend = "socket"
path = "/tmp/vpp/vhost-user127.sock"
server = "off"

[netdev "netdev127"]
type = "vhost-user"
chardev = "vhost127"

[device "virtio127"]
driver = "virtio-net-pci"
netdev = "netdev127"
bus = "io15"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.7"

[device "io16"]
driver = "ioh3420"
multifunction = "on"
addr = "2.0"
chassis = "16"
bus = "pxb"

[chardev "vhost128"]
backend = "socket"
path = "/tmp/vpp/vhost-user128.sock"
server = "off"

[netdev "netdev128"]
type = "vhost-user"
chardev = "vhost128"

[device "virtio128"]
driver = "virtio-net-pci"
netdev = "netdev128"
bus = "io16"
multifunction = "on"
disable-modern = "off"
disable-legacy = "on"
x-ignore-backend-features = "on"
addr = "0.0"

Re: [Qemu-devel] [PATCH 2/3] virtio: postpone the execution of event_notifier_cleanup function

[Michal Privoznik]

On 01/14/2018 11:06 AM, Gal Hammer wrote:
> Use the EventNotifier's cleanup callback function to execute the
> event_notifier_cleanup function after kvm unregistered the eventfd.
> 
> This change supports running the virtio_bus_set_host_notifier
> function inside a memory region transaction. Otherwise, a closed
> fd is sent to kvm, which results in a failure.
> 
> Signed-off-by: Gal Hammer <ghammer@redhat.com>
> ---
>  accel/kvm/kvm-all.c    |  4 ++++
>  hw/virtio/virtio-bus.c | 19 +++++++++++--------
>  2 files changed, 15 insertions(+), 8 deletions(-)
> 
> diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
> index f290f48..071f4f5 100644
> --- a/accel/kvm/kvm-all.c
> +++ b/accel/kvm/kvm-all.c
> @@ -812,6 +812,10 @@ static void kvm_mem_ioeventfd_del(MemoryListener *listener,
>      if (r < 0) {
>          abort();
>      }
> +
> +    if (e->cleanup) {
> +        e->cleanup(e);
> +    }
>  }
>  
>  static void kvm_io_ioeventfd_add(MemoryListener *listener,
> diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
> index 3042232..8106346 100644
> --- a/hw/virtio/virtio-bus.c
> +++ b/hw/virtio/virtio-bus.c
> @@ -256,6 +256,15 @@ bool virtio_bus_ioeventfd_enabled(VirtioBusState *bus)
>      return k->ioeventfd_assign && k->ioeventfd_enabled(proxy);
>  }
>  
> +static void virtio_bus_cleanup_event_notifier(EventNotifier *notifier)
> +{
> +    /* Test and clear notifier after disabling event,
> +     * in case poll callback didn't have time to run.
> +     */
> +    virtio_queue_host_notifier_read(notifier);
> +    event_notifier_cleanup(notifier);
> +}
> +
>  /*
>   * This function switches ioeventfd on/off in the device.
>   * The caller must set or clear the handlers for the EventNotifier.
> @@ -283,19 +292,13 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
>          r = k->ioeventfd_assign(proxy, notifier, n, true);
>          if (r < 0) {
>              error_report("%s: unable to assign ioeventfd: %d", __func__, r);
> -            goto cleanup_event_notifier;
> +            virtio_bus_cleanup_event_notifier(notifier);
>          }
> -        return 0;
>      } else {
> +        notifier->cleanup = virtio_bus_cleanup_event_notifier;
>          k->ioeventfd_assign(proxy, notifier, n, false);
>      }
>  
> -cleanup_event_notifier:
> -    /* Test and clear notifier after disabling event,
> -     * in case poll callback didn't have time to run.
> -     */
> -    virtio_queue_host_notifier_read(notifier);
> -    event_notifier_cleanup(notifier);
>      return r;
>  }
>  
> 

This causes abort() to me:

Thread 7 "CPU 3/KVM" received signal SIGABRT, Aborted.
[Switching to Thread 0x7f78fa1f4700 (LWP 13737)]
__GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
51      }
__GI_raise 7 # bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
#2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
#3  0x000055f2a70a6fac in address_space_add_del_ioeventfds (as=0x55f2a7e86620 <address_space_io>, fds_new=0x7f78e40b9e80, fds_new_nb=2, fds_old=0x7f78e40b9f10, fds_old_nb=3) at /home/zippy/work/qemu/qemu.git/memory.c:832
#4  0x000055f2a70a744d in address_space_update_ioeventfds (as=0x55f2a7e86620 <address_space_io>) at /home/zippy/work/qemu/qemu.git/memory.c:895
#5  0x000055f2a70a7f04 in memory_region_transaction_commit () at /home/zippy/work/qemu/qemu.git/memory.c:1097
#6  0x000055f2a70ab0d1 in memory_region_del_eventfd (mr=0x55f2a9422250, addr=16, size=2, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/memory.c:2273
#7  0x000055f2a7411f18 in virtio_pci_ioeventfd_assign (d=0x55f2a9421970, notifier=0x7f79fc81f080, n=0, assign=false) at hw/virtio/virtio-pci.c:280
#8  0x000055f2a7418e6f in virtio_bus_set_host_notifier (bus=0x55f2a9429a68, n=0, assign=false) at hw/virtio/virtio-bus.c:299
#9  0x000055f2a710236a in virtio_scsi_dataplane_stop (vdev=0x55f2a9429ae0) at /home/zippy/work/qemu/qemu.git/hw/scsi/virtio-scsi-dataplane.c:215
#10 0x000055f2a7418bb3 in virtio_bus_stop_ioeventfd (bus=0x55f2a9429a68) at hw/virtio/virtio-bus.c:246
#11 0x000055f2a7411f63 in virtio_pci_stop_ioeventfd (proxy=0x55f2a9421970) at hw/virtio/virtio-pci.c:294
#12 0x000055f2a74144ab in virtio_pci_common_write (opaque=0x55f2a9421970, addr=20, val=0, size=1) at hw/virtio/virtio-pci.c:1262
#13 0x000055f2a70a6016 in memory_region_write_accessor (mr=0x55f2a9422340, addr=20, value=0x7f78fa1f1558, size=1, shift=0, mask=255, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:560
#14 0x000055f2a70a622e in access_with_adjusted_size (addr=20, value=0x7f78fa1f1558, size=1, access_size_min=1, access_size_max=4, access_fn=0x55f2a70a5f2c <memory_region_write_accessor>, mr=0x55f2a9422340, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:627
#15 0x000055f2a70a8eb0 in memory_region_dispatch_write (mr=0x55f2a9422340, addr=20, data=0, size=1, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:1503
#16 0x000055f2a7058b2b in flatview_write_continue (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, addr1=20, l=1, mr=0x55f2a9422340) at /home/zippy/work/qemu/qemu.git/exec.c:3038
#17 0x000055f2a7058c79 in flatview_write (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1) at /home/zippy/work/qemu/qemu.git/exec.c:3083
#18 0x000055f2a7059050 in flatview_rw (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3192
#19 0x000055f2a7059108 in address_space_rw (as=0x55f2a7e86680 <address_space_memory>, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3202
#20 0x000055f2a70bf42c in kvm_cpu_exec (cpu=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:1941
#21 0x000055f2a708d455 in qemu_kvm_cpu_thread_fn (arg=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/cpus.c:1196
#22 0x000055f2a75ce829 in qemu_thread_start (args=0x55f2a852e260) at util/qemu-thread-posix.c:504
#23 0x00007f7a0a375887 in start_thread (arg=0x7f78fa1f4700) at pthread_create.c:456
#24 0x00007f7a0a0b7d4f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97
__GI_raise 7 # up
#1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
89            raise (SIGABRT);
__GI_abort 7 # 
#2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
852             abort();
kvm_io_ioeventfd_del 7 # p *e
$1 = {rfd = -1, wfd = -1, cleanup = 0x0}


I think the problem is that event_notifier_cleanup() is called before
kvm_io_ioeventfd_del() and both are called over the same EventNotifier.
This is supported by another data I was able to collect in a different
run:

Thread 4 "CPU 0/KVM" hit Breakpoint 1, event_notifier_cleanup (e=0x7fc6ac26f080) at util/event_notifier-posix.c:83
83          if (e->rfd != e->wfd) {
event_notifier_cleanup 4 # p *e
$1 = {rfd = 48, wfd = 48, cleanup = 0x55c91c76ac87 <virtio_bus_cleanup_event_notifier>}
event_notifier_cleanup 4 # c
Continuing.

Thread 4 "CPU 0/KVM" received signal SIGABRT, Aborted.
__GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
51      }
__GI_raise 4 # up
#1  0x00007fc6b9a45c1a in __GI_abort () at abort.c:89
89            raise (SIGABRT);
__GI_abort 4 # 
#2  0x000055c91c40ee9d in kvm_io_ioeventfd_del (listener=0x55c91d06b200 <kvm_io_listener>, section=0x7fc5ab1fa0f0, match_data=true, data=0, e=0x7fc6ac26f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
852             abort();
kvm_io_ioeventfd_del 4 # p *e
$2 = {rfd = -1, wfd = -1, cleanup = 0x0}


So I drafted dirty workaround:

index 071f4f57c0..bf15344533 100644
--- i/accel/kvm/kvm-all.c
+++ w/accel/kvm/kvm-all.c
@@ -845,6 +845,9 @@ static void kvm_io_ioeventfd_del(MemoryListener *listener,
     int fd = event_notifier_get_fd(e);
     int r;
 
+    if (fd == -1)
+        return;
+
     r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
                               data, false, int128_get64(section->size),
                               match_data);

But this time I'm getting following error:

kvm_io_ioeventfd_add: error adding ioeventfd: File exists

__GI_raise 7 # up
#1  0x00007f3ccd579c1a in __GI_abort () at abort.c:89
89            raise (SIGABRT);
__GI_abort 7 # 
#2  0x000055c68ac84e08 in kvm_io_ioeventfd_add (listener=0x55c68b8e1200 <kvm_io_listener>, section=0x7f3bbd7f1090, match_data=true, data=0, e=0x7f3bbcf8d080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:835
835             abort();
kvm_io_ioeventfd_add 7 # p *e
$1 = {rfd = 48, wfd = 48, cleanup = 0x0}




Michal

Re: [Qemu-devel] [PATCH 2/3] virtio: postpone the execution of event_notifier_cleanup function

[Marc-André Lureau]

Hi

On Mon, Jan 22, 2018 at 12:53 PM, Michal Privoznik <mprivozn@redhat.com> wrote:
> On 01/14/2018 11:06 AM, Gal Hammer wrote:
>> Use the EventNotifier's cleanup callback function to execute the
>> event_notifier_cleanup function after kvm unregistered the eventfd.
>>
>> This change supports running the virtio_bus_set_host_notifier
>> function inside a memory region transaction. Otherwise, a closed
>> fd is sent to kvm, which results in a failure.
>>
>> Signed-off-by: Gal Hammer <ghammer@redhat.com>
>> ---
>>  accel/kvm/kvm-all.c    |  4 ++++
>>  hw/virtio/virtio-bus.c | 19 +++++++++++--------
>>  2 files changed, 15 insertions(+), 8 deletions(-)
>>
>> diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
>> index f290f48..071f4f5 100644
>> --- a/accel/kvm/kvm-all.c
>> +++ b/accel/kvm/kvm-all.c
>> @@ -812,6 +812,10 @@ static void kvm_mem_ioeventfd_del(MemoryListener *listener,
>>      if (r < 0) {
>>          abort();
>>      }
>> +
>> +    if (e->cleanup) {
>> +        e->cleanup(e);
>> +    }
>>  }
>>
>>  static void kvm_io_ioeventfd_add(MemoryListener *listener,
>> diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
>> index 3042232..8106346 100644
>> --- a/hw/virtio/virtio-bus.c
>> +++ b/hw/virtio/virtio-bus.c
>> @@ -256,6 +256,15 @@ bool virtio_bus_ioeventfd_enabled(VirtioBusState *bus)
>>      return k->ioeventfd_assign && k->ioeventfd_enabled(proxy);
>>  }
>>
>> +static void virtio_bus_cleanup_event_notifier(EventNotifier *notifier)
>> +{
>> +    /* Test and clear notifier after disabling event,
>> +     * in case poll callback didn't have time to run.
>> +     */
>> +    virtio_queue_host_notifier_read(notifier);
>> +    event_notifier_cleanup(notifier);
>> +}
>> +
>>  /*
>>   * This function switches ioeventfd on/off in the device.
>>   * The caller must set or clear the handlers for the EventNotifier.
>> @@ -283,19 +292,13 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
>>          r = k->ioeventfd_assign(proxy, notifier, n, true);
>>          if (r < 0) {
>>              error_report("%s: unable to assign ioeventfd: %d", __func__, r);
>> -            goto cleanup_event_notifier;
>> +            virtio_bus_cleanup_event_notifier(notifier);
>>          }
>> -        return 0;
>>      } else {
>> +        notifier->cleanup = virtio_bus_cleanup_event_notifier;
>>          k->ioeventfd_assign(proxy, notifier, n, false);
>>      }
>>
>> -cleanup_event_notifier:
>> -    /* Test and clear notifier after disabling event,
>> -     * in case poll callback didn't have time to run.
>> -     */
>> -    virtio_queue_host_notifier_read(notifier);
>> -    event_notifier_cleanup(notifier);
>>      return r;
>>  }
>>
>>
>
> This causes abort() to me:
>
> Thread 7 "CPU 3/KVM" received signal SIGABRT, Aborted.
> [Switching to Thread 0x7f78fa1f4700 (LWP 13737)]
> __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> 51      }
> __GI_raise 7 # bt
> #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> #1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
> #2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> #3  0x000055f2a70a6fac in address_space_add_del_ioeventfds (as=0x55f2a7e86620 <address_space_io>, fds_new=0x7f78e40b9e80, fds_new_nb=2, fds_old=0x7f78e40b9f10, fds_old_nb=3) at /home/zippy/work/qemu/qemu.git/memory.c:832
> #4  0x000055f2a70a744d in address_space_update_ioeventfds (as=0x55f2a7e86620 <address_space_io>) at /home/zippy/work/qemu/qemu.git/memory.c:895
> #5  0x000055f2a70a7f04 in memory_region_transaction_commit () at /home/zippy/work/qemu/qemu.git/memory.c:1097
> #6  0x000055f2a70ab0d1 in memory_region_del_eventfd (mr=0x55f2a9422250, addr=16, size=2, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/memory.c:2273
> #7  0x000055f2a7411f18 in virtio_pci_ioeventfd_assign (d=0x55f2a9421970, notifier=0x7f79fc81f080, n=0, assign=false) at hw/virtio/virtio-pci.c:280
> #8  0x000055f2a7418e6f in virtio_bus_set_host_notifier (bus=0x55f2a9429a68, n=0, assign=false) at hw/virtio/virtio-bus.c:299
> #9  0x000055f2a710236a in virtio_scsi_dataplane_stop (vdev=0x55f2a9429ae0) at /home/zippy/work/qemu/qemu.git/hw/scsi/virtio-scsi-dataplane.c:215
> #10 0x000055f2a7418bb3 in virtio_bus_stop_ioeventfd (bus=0x55f2a9429a68) at hw/virtio/virtio-bus.c:246
> #11 0x000055f2a7411f63 in virtio_pci_stop_ioeventfd (proxy=0x55f2a9421970) at hw/virtio/virtio-pci.c:294
> #12 0x000055f2a74144ab in virtio_pci_common_write (opaque=0x55f2a9421970, addr=20, val=0, size=1) at hw/virtio/virtio-pci.c:1262
> #13 0x000055f2a70a6016 in memory_region_write_accessor (mr=0x55f2a9422340, addr=20, value=0x7f78fa1f1558, size=1, shift=0, mask=255, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:560
> #14 0x000055f2a70a622e in access_with_adjusted_size (addr=20, value=0x7f78fa1f1558, size=1, access_size_min=1, access_size_max=4, access_fn=0x55f2a70a5f2c <memory_region_write_accessor>, mr=0x55f2a9422340, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:627
> #15 0x000055f2a70a8eb0 in memory_region_dispatch_write (mr=0x55f2a9422340, addr=20, data=0, size=1, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:1503
> #16 0x000055f2a7058b2b in flatview_write_continue (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, addr1=20, l=1, mr=0x55f2a9422340) at /home/zippy/work/qemu/qemu.git/exec.c:3038
> #17 0x000055f2a7058c79 in flatview_write (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1) at /home/zippy/work/qemu/qemu.git/exec.c:3083
> #18 0x000055f2a7059050 in flatview_rw (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3192
> #19 0x000055f2a7059108 in address_space_rw (as=0x55f2a7e86680 <address_space_memory>, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3202
> #20 0x000055f2a70bf42c in kvm_cpu_exec (cpu=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:1941
> #21 0x000055f2a708d455 in qemu_kvm_cpu_thread_fn (arg=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/cpus.c:1196
> #22 0x000055f2a75ce829 in qemu_thread_start (args=0x55f2a852e260) at util/qemu-thread-posix.c:504
> #23 0x00007f7a0a375887 in start_thread (arg=0x7f78fa1f4700) at pthread_create.c:456
> #24 0x00007f7a0a0b7d4f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97
> __GI_raise 7 # up
> #1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
> 89            raise (SIGABRT);
> __GI_abort 7 #
> #2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> 852             abort();
> kvm_io_ioeventfd_del 7 # p *e
> $1 = {rfd = -1, wfd = -1, cleanup = 0x0}
>
>
> I think the problem is that event_notifier_cleanup() is called before
> kvm_io_ioeventfd_del() and both are called over the same EventNotifier.
> This is supported by another data I was able to collect in a different
> run:
>
> Thread 4 "CPU 0/KVM" hit Breakpoint 1, event_notifier_cleanup (e=0x7fc6ac26f080) at util/event_notifier-posix.c:83
> 83          if (e->rfd != e->wfd) {
> event_notifier_cleanup 4 # p *e
> $1 = {rfd = 48, wfd = 48, cleanup = 0x55c91c76ac87 <virtio_bus_cleanup_event_notifier>}
> event_notifier_cleanup 4 # c
> Continuing.
>
> Thread 4 "CPU 0/KVM" received signal SIGABRT, Aborted.
> __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> 51      }
> __GI_raise 4 # up
> #1  0x00007fc6b9a45c1a in __GI_abort () at abort.c:89
> 89            raise (SIGABRT);
> __GI_abort 4 #
> #2  0x000055c91c40ee9d in kvm_io_ioeventfd_del (listener=0x55c91d06b200 <kvm_io_listener>, section=0x7fc5ab1fa0f0, match_data=true, data=0, e=0x7fc6ac26f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> 852             abort();
> kvm_io_ioeventfd_del 4 # p *e
> $2 = {rfd = -1, wfd = -1, cleanup = 0x0}
>
>
> So I drafted dirty workaround:
>
> index 071f4f57c0..bf15344533 100644
> --- i/accel/kvm/kvm-all.c
> +++ w/accel/kvm/kvm-all.c
> @@ -845,6 +845,9 @@ static void kvm_io_ioeventfd_del(MemoryListener *listener,
>      int fd = event_notifier_get_fd(e);
>      int r;
>
> +    if (fd == -1)
> +        return;
> +
>      r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
>                                data, false, int128_get64(section->size),
>                                match_data);
>
> But this time I'm getting following error:
>
> kvm_io_ioeventfd_add: error adding ioeventfd: File exists
>
> __GI_raise 7 # up
> #1  0x00007f3ccd579c1a in __GI_abort () at abort.c:89
> 89            raise (SIGABRT);
> __GI_abort 7 #
> #2  0x000055c68ac84e08 in kvm_io_ioeventfd_add (listener=0x55c68b8e1200 <kvm_io_listener>, section=0x7f3bbd7f1090, match_data=true, data=0, e=0x7f3bbcf8d080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:835
> 835             abort();
> kvm_io_ioeventfd_add 7 # p *e
> $1 = {rfd = 48, wfd = 48, cleanup = 0x0}
>

I was about to report the same error when booting an uefi guest, I
bisected down to the same first bad commit.

Thanks Michal

-- 
Marc-André Lureau

Re: [Qemu-devel] [PATCH 2/3] virtio: postpone the execution of event_notifier_cleanup function

[Michael S. Tsirkin]

On Mon, Jan 22, 2018 at 06:35:03PM +0100, Marc-André Lureau wrote:
> Hi
> 
> On Mon, Jan 22, 2018 at 12:53 PM, Michal Privoznik <mprivozn@redhat.com> wrote:
> > On 01/14/2018 11:06 AM, Gal Hammer wrote:
> >> Use the EventNotifier's cleanup callback function to execute the
> >> event_notifier_cleanup function after kvm unregistered the eventfd.
> >>
> >> This change supports running the virtio_bus_set_host_notifier
> >> function inside a memory region transaction. Otherwise, a closed
> >> fd is sent to kvm, which results in a failure.
> >>
> >> Signed-off-by: Gal Hammer <ghammer@redhat.com>
> >> ---
> >>  accel/kvm/kvm-all.c    |  4 ++++
> >>  hw/virtio/virtio-bus.c | 19 +++++++++++--------
> >>  2 files changed, 15 insertions(+), 8 deletions(-)
> >>
> >> diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
> >> index f290f48..071f4f5 100644
> >> --- a/accel/kvm/kvm-all.c
> >> +++ b/accel/kvm/kvm-all.c
> >> @@ -812,6 +812,10 @@ static void kvm_mem_ioeventfd_del(MemoryListener *listener,
> >>      if (r < 0) {
> >>          abort();
> >>      }
> >> +
> >> +    if (e->cleanup) {
> >> +        e->cleanup(e);
> >> +    }
> >>  }
> >>
> >>  static void kvm_io_ioeventfd_add(MemoryListener *listener,
> >> diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
> >> index 3042232..8106346 100644
> >> --- a/hw/virtio/virtio-bus.c
> >> +++ b/hw/virtio/virtio-bus.c
> >> @@ -256,6 +256,15 @@ bool virtio_bus_ioeventfd_enabled(VirtioBusState *bus)
> >>      return k->ioeventfd_assign && k->ioeventfd_enabled(proxy);
> >>  }
> >>
> >> +static void virtio_bus_cleanup_event_notifier(EventNotifier *notifier)
> >> +{
> >> +    /* Test and clear notifier after disabling event,
> >> +     * in case poll callback didn't have time to run.
> >> +     */
> >> +    virtio_queue_host_notifier_read(notifier);
> >> +    event_notifier_cleanup(notifier);
> >> +}
> >> +
> >>  /*
> >>   * This function switches ioeventfd on/off in the device.
> >>   * The caller must set or clear the handlers for the EventNotifier.
> >> @@ -283,19 +292,13 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
> >>          r = k->ioeventfd_assign(proxy, notifier, n, true);
> >>          if (r < 0) {
> >>              error_report("%s: unable to assign ioeventfd: %d", __func__, r);
> >> -            goto cleanup_event_notifier;
> >> +            virtio_bus_cleanup_event_notifier(notifier);
> >>          }
> >> -        return 0;
> >>      } else {
> >> +        notifier->cleanup = virtio_bus_cleanup_event_notifier;
> >>          k->ioeventfd_assign(proxy, notifier, n, false);
> >>      }
> >>
> >> -cleanup_event_notifier:
> >> -    /* Test and clear notifier after disabling event,
> >> -     * in case poll callback didn't have time to run.
> >> -     */
> >> -    virtio_queue_host_notifier_read(notifier);
> >> -    event_notifier_cleanup(notifier);
> >>      return r;
> >>  }
> >>
> >>
> >
> > This causes abort() to me:
> >
> > Thread 7 "CPU 3/KVM" received signal SIGABRT, Aborted.
> > [Switching to Thread 0x7f78fa1f4700 (LWP 13737)]
> > __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> > 51      }
> > __GI_raise 7 # bt
> > #0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> > #1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
> > #2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> > #3  0x000055f2a70a6fac in address_space_add_del_ioeventfds (as=0x55f2a7e86620 <address_space_io>, fds_new=0x7f78e40b9e80, fds_new_nb=2, fds_old=0x7f78e40b9f10, fds_old_nb=3) at /home/zippy/work/qemu/qemu.git/memory.c:832
> > #4  0x000055f2a70a744d in address_space_update_ioeventfds (as=0x55f2a7e86620 <address_space_io>) at /home/zippy/work/qemu/qemu.git/memory.c:895
> > #5  0x000055f2a70a7f04 in memory_region_transaction_commit () at /home/zippy/work/qemu/qemu.git/memory.c:1097
> > #6  0x000055f2a70ab0d1 in memory_region_del_eventfd (mr=0x55f2a9422250, addr=16, size=2, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/memory.c:2273
> > #7  0x000055f2a7411f18 in virtio_pci_ioeventfd_assign (d=0x55f2a9421970, notifier=0x7f79fc81f080, n=0, assign=false) at hw/virtio/virtio-pci.c:280
> > #8  0x000055f2a7418e6f in virtio_bus_set_host_notifier (bus=0x55f2a9429a68, n=0, assign=false) at hw/virtio/virtio-bus.c:299
> > #9  0x000055f2a710236a in virtio_scsi_dataplane_stop (vdev=0x55f2a9429ae0) at /home/zippy/work/qemu/qemu.git/hw/scsi/virtio-scsi-dataplane.c:215
> > #10 0x000055f2a7418bb3 in virtio_bus_stop_ioeventfd (bus=0x55f2a9429a68) at hw/virtio/virtio-bus.c:246
> > #11 0x000055f2a7411f63 in virtio_pci_stop_ioeventfd (proxy=0x55f2a9421970) at hw/virtio/virtio-pci.c:294
> > #12 0x000055f2a74144ab in virtio_pci_common_write (opaque=0x55f2a9421970, addr=20, val=0, size=1) at hw/virtio/virtio-pci.c:1262
> > #13 0x000055f2a70a6016 in memory_region_write_accessor (mr=0x55f2a9422340, addr=20, value=0x7f78fa1f1558, size=1, shift=0, mask=255, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:560
> > #14 0x000055f2a70a622e in access_with_adjusted_size (addr=20, value=0x7f78fa1f1558, size=1, access_size_min=1, access_size_max=4, access_fn=0x55f2a70a5f2c <memory_region_write_accessor>, mr=0x55f2a9422340, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:627
> > #15 0x000055f2a70a8eb0 in memory_region_dispatch_write (mr=0x55f2a9422340, addr=20, data=0, size=1, attrs=...) at /home/zippy/work/qemu/qemu.git/memory.c:1503
> > #16 0x000055f2a7058b2b in flatview_write_continue (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, addr1=20, l=1, mr=0x55f2a9422340) at /home/zippy/work/qemu/qemu.git/exec.c:3038
> > #17 0x000055f2a7058c79 in flatview_write (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1) at /home/zippy/work/qemu/qemu.git/exec.c:3083
> > #18 0x000055f2a7059050 in flatview_rw (fv=0x7f78e405eba0, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3192
> > #19 0x000055f2a7059108 in address_space_rw (as=0x55f2a7e86680 <address_space_memory>, addr=4269834260, attrs=..., buf=0x7f7a26c36028 "", len=1, is_write=true) at /home/zippy/work/qemu/qemu.git/exec.c:3202
> > #20 0x000055f2a70bf42c in kvm_cpu_exec (cpu=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:1941
> > #21 0x000055f2a708d455 in qemu_kvm_cpu_thread_fn (arg=0x55f2a850cda0) at /home/zippy/work/qemu/qemu.git/cpus.c:1196
> > #22 0x000055f2a75ce829 in qemu_thread_start (args=0x55f2a852e260) at util/qemu-thread-posix.c:504
> > #23 0x00007f7a0a375887 in start_thread (arg=0x7f78fa1f4700) at pthread_create.c:456
> > #24 0x00007f7a0a0b7d4f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97
> > __GI_raise 7 # up
> > #1  0x00007f7a09ff5c1a in __GI_abort () at abort.c:89
> > 89            raise (SIGABRT);
> > __GI_abort 7 #
> > #2  0x000055f2a70bce9d in kvm_io_ioeventfd_del (listener=0x55f2a7d19200 <kvm_io_listener>, section=0x7f78fa1f10f0, match_data=true, data=0, e=0x7f79fc81f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> > 852             abort();
> > kvm_io_ioeventfd_del 7 # p *e
> > $1 = {rfd = -1, wfd = -1, cleanup = 0x0}
> >
> >
> > I think the problem is that event_notifier_cleanup() is called before
> > kvm_io_ioeventfd_del() and both are called over the same EventNotifier.
> > This is supported by another data I was able to collect in a different
> > run:
> >
> > Thread 4 "CPU 0/KVM" hit Breakpoint 1, event_notifier_cleanup (e=0x7fc6ac26f080) at util/event_notifier-posix.c:83
> > 83          if (e->rfd != e->wfd) {
> > event_notifier_cleanup 4 # p *e
> > $1 = {rfd = 48, wfd = 48, cleanup = 0x55c91c76ac87 <virtio_bus_cleanup_event_notifier>}
> > event_notifier_cleanup 4 # c
> > Continuing.
> >
> > Thread 4 "CPU 0/KVM" received signal SIGABRT, Aborted.
> > __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> > 51      }
> > __GI_raise 4 # up
> > #1  0x00007fc6b9a45c1a in __GI_abort () at abort.c:89
> > 89            raise (SIGABRT);
> > __GI_abort 4 #
> > #2  0x000055c91c40ee9d in kvm_io_ioeventfd_del (listener=0x55c91d06b200 <kvm_io_listener>, section=0x7fc5ab1fa0f0, match_data=true, data=0, e=0x7fc6ac26f080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:852
> > 852             abort();
> > kvm_io_ioeventfd_del 4 # p *e
> > $2 = {rfd = -1, wfd = -1, cleanup = 0x0}
> >
> >
> > So I drafted dirty workaround:
> >
> > index 071f4f57c0..bf15344533 100644
> > --- i/accel/kvm/kvm-all.c
> > +++ w/accel/kvm/kvm-all.c
> > @@ -845,6 +845,9 @@ static void kvm_io_ioeventfd_del(MemoryListener *listener,
> >      int fd = event_notifier_get_fd(e);
> >      int r;
> >
> > +    if (fd == -1)
> > +        return;
> > +
> >      r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
> >                                data, false, int128_get64(section->size),
> >                                match_data);
> >
> > But this time I'm getting following error:
> >
> > kvm_io_ioeventfd_add: error adding ioeventfd: File exists
> >
> > __GI_raise 7 # up
> > #1  0x00007f3ccd579c1a in __GI_abort () at abort.c:89
> > 89            raise (SIGABRT);
> > __GI_abort 7 #
> > #2  0x000055c68ac84e08 in kvm_io_ioeventfd_add (listener=0x55c68b8e1200 <kvm_io_listener>, section=0x7f3bbd7f1090, match_data=true, data=0, e=0x7f3bbcf8d080) at /home/zippy/work/qemu/qemu.git/accel/kvm/kvm-all.c:835
> > 835             abort();
> > kvm_io_ioeventfd_add 7 # p *e
> > $1 = {rfd = 48, wfd = 48, cleanup = 0x0}
> >
> 
> I was about to report the same error when booting an uefi guest, I
> bisected down to the same first bad commit.
> 
> Thanks Michal

Gal - any input on this before I revert?

> -- 
> Marc-André Lureau