[Qemu-devel] [PATCH v5 0/3] virtio-scsi: restart DMA after iothread

[Qemu-devel] [PATCH v5 0/3] virtio-scsi: restart DMA after iothread

[Stefan Hajnoczi]

v5:
 * Plumbing vm change state handlers into DeviceClass/BusClass is a rather
   large bug fix.  Instead I've combined the previous priorities approach with
   the observation from Kevin and Paolo that we really want to order by qdev
   tree depth.

   The new qdev_add_vm_change_state_handler() API lets DeviceStates register
   callbacks that execute in qdev tree depth order.  This solves the
   virtio-scsi bug since the virtio-scsi device's callback must complete before
   its child scsi-disk's callback runs.

   Is this a good compromise for everyone?

Stefan Hajnoczi (3):
  vl: add qemu_add_vm_change_state_handler_prio()
  qdev: add qdev_add_vm_change_state_handler()
  virtio-scsi: restart DMA after iothread

 hw/core/Makefile.objs             |  1 +
 include/hw/qdev-core.h            |  5 +++
 include/sysemu/sysemu.h           |  2 +
 hw/core/vm-change-state-handler.c | 61 +++++++++++++++++++++++++++++++
 hw/scsi/scsi-bus.c                |  4 +-
 hw/virtio/virtio.c                |  4 +-
 vl.c                              | 59 ++++++++++++++++++++++++------
 7 files changed, 120 insertions(+), 16 deletions(-)
 create mode 100644 hw/core/vm-change-state-handler.c

-- 
2.21.0

[Qemu-devel] [PATCH v5 1/3] vl: add qemu_add_vm_change_state_handler_prio()

[Stefan Hajnoczi]

Add an API for registering vm change state handlers with a well-defined
ordering.  This is necessary when handlers depend on each other.

Small coding style fixes are included to make checkpatch.pl happy.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/sysemu/sysemu.h |  2 ++
 vl.c                    | 59 ++++++++++++++++++++++++++++++++---------
 2 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 61579ae71e..984c439ac9 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -29,6 +29,8 @@ typedef void VMChangeStateHandler(void *opaque, int running, RunState state);
 
 VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
                                                      void *opaque);
+VMChangeStateEntry *qemu_add_vm_change_state_handler_prio(
+        VMChangeStateHandler *cb, void *opaque, int priority);
 void qemu_del_vm_change_state_handler(VMChangeStateEntry *e);
 void vm_state_notify(int running, RunState state);
 
diff --git a/vl.c b/vl.c
index 99a56b5556..7fac2ae7ca 100644
--- a/vl.c
+++ b/vl.c
@@ -1471,28 +1471,57 @@ static int machine_help_func(QemuOpts *opts, MachineState *machine)
 struct vm_change_state_entry {
     VMChangeStateHandler *cb;
     void *opaque;
-    QLIST_ENTRY (vm_change_state_entry) entries;
+    QTAILQ_ENTRY(vm_change_state_entry) entries;
+    int priority;
 };
 
-static QLIST_HEAD(, vm_change_state_entry) vm_change_state_head;
+static QTAILQ_HEAD(, vm_change_state_entry) vm_change_state_head;
 
-VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
-                                                     void *opaque)
+/**
+ * qemu_add_vm_change_state_handler_prio:
+ * @cb: the callback to invoke
+ * @opaque: user data passed to the callback
+ * @priority: low priorities execute first when the vm runs and the reverse is
+ *            true when the vm stops
+ *
+ * Register a callback function that is invoked when the vm starts or stops
+ * running.
+ *
+ * Returns: an entry to be freed using qemu_del_vm_change_state_handler()
+ */
+VMChangeStateEntry *qemu_add_vm_change_state_handler_prio(
+        VMChangeStateHandler *cb, void *opaque, int priority)
 {
     VMChangeStateEntry *e;
+    VMChangeStateEntry *other;
 
-    e = g_malloc0(sizeof (*e));
-
+    e = g_malloc0(sizeof(*e));
     e->cb = cb;
     e->opaque = opaque;
-    QLIST_INSERT_HEAD(&vm_change_state_head, e, entries);
+    e->priority = priority;
+
+    /* Keep list sorted in ascending priority order */
+    QTAILQ_FOREACH(other, &vm_change_state_head, entries) {
+        if (priority < other->priority) {
+            QTAILQ_INSERT_BEFORE(other, e, entries);
+            return e;
+        }
+    }
+
+    QTAILQ_INSERT_TAIL(&vm_change_state_head, e, entries);
     return e;
 }
 
+VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
+                                                     void *opaque)
+{
+    return qemu_add_vm_change_state_handler_prio(cb, opaque, 0);
+}
+
 void qemu_del_vm_change_state_handler(VMChangeStateEntry *e)
 {
-    QLIST_REMOVE (e, entries);
-    g_free (e);
+    QTAILQ_REMOVE(&vm_change_state_head, e, entries);
+    g_free(e);
 }
 
 void vm_state_notify(int running, RunState state)
@@ -1501,8 +1530,14 @@ void vm_state_notify(int running, RunState state)
 
     trace_vm_state_notify(running, state, RunState_str(state));
 
-    QLIST_FOREACH_SAFE(e, &vm_change_state_head, entries, next) {
-        e->cb(e->opaque, running, state);
+    if (running) {
+        QTAILQ_FOREACH_SAFE(e, &vm_change_state_head, entries, next) {
+            e->cb(e->opaque, running, state);
+        }
+    } else {
+        QTAILQ_FOREACH_REVERSE_SAFE(e, &vm_change_state_head, entries, next) {
+            e->cb(e->opaque, running, state);
+        }
     }
 }
 
@@ -3025,7 +3060,7 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     }
 
-    QLIST_INIT (&vm_change_state_head);
+    QTAILQ_INIT(&vm_change_state_head);
     os_setup_early_signal_handling();
 
     cpu_option = NULL;
-- 
2.21.0

[Qemu-devel] [PATCH v5 2/3] qdev: add qdev_add_vm_change_state_handler()

[Stefan Hajnoczi]

Children sometimes depend on their parent's vm change state handler
having completed.  Add a vm change state handler API for devices that
guarantees tree depth ordering.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/core/Makefile.objs             |  1 +
 include/hw/qdev-core.h            |  5 +++
 hw/core/vm-change-state-handler.c | 61 +++++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 hw/core/vm-change-state-handler.c

diff --git a/hw/core/Makefile.objs b/hw/core/Makefile.objs
index a799c83815..dc0895ee9a 100644
--- a/hw/core/Makefile.objs
+++ b/hw/core/Makefile.objs
@@ -7,6 +7,7 @@ common-obj-$(CONFIG_SOFTMMU) += fw-path-provider.o
 common-obj-y += irq.o
 common-obj-y += hotplug.o
 common-obj-$(CONFIG_SOFTMMU) += nmi.o
+common-obj-$(CONFIG_SOFTMMU) += vm-change-state-handler.o
 
 common-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
 common-obj-$(CONFIG_XILINX_AXI) += stream.o
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index fa55dc10ae..e157fc4acd 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -6,6 +6,7 @@
 #include "qom/object.h"
 #include "hw/irq.h"
 #include "hw/hotplug.h"
+#include "sysemu/sysemu.h"
 
 enum {
     DEV_NVECTORS_UNSPECIFIED = -1,
@@ -450,4 +451,8 @@ static inline bool qbus_is_hotpluggable(BusState *bus)
 void device_listener_register(DeviceListener *listener);
 void device_listener_unregister(DeviceListener *listener);
 
+VMChangeStateEntry *qdev_add_vm_change_state_handler(DeviceState *dev,
+                                                     VMChangeStateHandler *cb,
+                                                     void *opaque);
+
 #endif
diff --git a/hw/core/vm-change-state-handler.c b/hw/core/vm-change-state-handler.c
new file mode 100644
index 0000000000..f814813bdd
--- /dev/null
+++ b/hw/core/vm-change-state-handler.c
@@ -0,0 +1,61 @@
+/*
+ *  qdev vm change state handlers
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+static int qdev_get_dev_tree_depth(DeviceState *dev)
+{
+    int depth;
+
+    for (depth = 0; dev; depth++) {
+        BusState *bus = dev->parent_bus;
+
+        if (!bus) {
+            break;
+        }
+
+        dev = bus->parent;
+    }
+
+    return depth;
+}
+
+/**
+ * qdev_add_vm_change_state_handler:
+ * @dev: the device that owns this handler
+ * @cb: the callback function to be invoked
+ * @opaque: user data passed to the callback function
+ *
+ * This function works like qemu_add_vm_change_state_handler() except callbacks
+ * are invoked in qdev tree depth order.  Ordering is desirable when callbacks
+ * of children depend on their parent's callback having completed first.
+ *
+ * For example, when qdev_add_vm_change_state_handler() is used, a host
+ * controller's callback is invoked before the children on its bus when the VM
+ * starts running.  The order is reversed when the VM stops running.
+ *
+ * Returns: an entry to be freed with qemu_del_vm_change_state_handler()
+ */
+VMChangeStateEntry *qdev_add_vm_change_state_handler(DeviceState *dev,
+                                                     VMChangeStateHandler *cb,
+                                                     void *opaque)
+{
+    int depth = qdev_get_dev_tree_depth(dev);
+
+    return qemu_add_vm_change_state_handler_prio(cb, opaque, depth);
+}
-- 
2.21.0

[Qemu-devel] [PATCH v5 3/3] virtio-scsi: restart DMA after iothread

[Stefan Hajnoczi]

When the 'cont' command resumes guest execution the vm change state
handlers are invoked.  Unfortunately there is no explicit ordering
between classic qemu_add_vm_change_state_handler() callbacks.  When two
layers of code both use vm change state handlers, we don't control which
handler runs first.

virtio-scsi with iothreads hits a deadlock when a failed SCSI command is
restarted and completes before the iothread is re-initialized.

This patch uses the new qdev_add_vm_change_state_handler() API to
guarantee that virtio-scsi's virtio change state handler executes before
the SCSI bus children.  This way DMA is restarted after the iothread has
re-initialized.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/scsi/scsi-bus.c | 4 ++--
 hw/virtio/virtio.c | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index 196136a307..fdc3a0e4e0 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -207,8 +207,8 @@ static void scsi_qdev_realize(DeviceState *qdev, Error **errp)
         error_propagate(errp, local_err);
         return;
     }
-    dev->vmsentry = qemu_add_vm_change_state_handler(scsi_dma_restart_cb,
-                                                     dev);
+    dev->vmsentry = qdev_add_vm_change_state_handler(DEVICE(dev),
+            scsi_dma_restart_cb, dev);
 }
 
 static void scsi_qdev_unrealize(DeviceState *qdev, Error **errp)
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index e1e90fcfd6..e42e6710d2 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -2354,8 +2354,8 @@ void virtio_init(VirtIODevice *vdev, const char *name,
     } else {
         vdev->config = NULL;
     }
-    vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
-                                                     vdev);
+    vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
+            virtio_vmstate_change, vdev);
     vdev->device_endian = virtio_default_endian();
     vdev->use_guest_notifier_mask = true;
 }
-- 
2.21.0

Re: [Qemu-devel] [PATCH v5 0/3] virtio-scsi: restart DMA after iothread

[Paolo Bonzini]

On 20/06/19 19:37, Stefan Hajnoczi wrote:
> v5:
>  * Plumbing vm change state handlers into DeviceClass/BusClass is a rather
>    large bug fix.  Instead I've combined the previous priorities approach with
>    the observation from Kevin and Paolo that we really want to order by qdev
>    tree depth.
> 
>    The new qdev_add_vm_change_state_handler() API lets DeviceStates register
>    callbacks that execute in qdev tree depth order.  This solves the
>    virtio-scsi bug since the virtio-scsi device's callback must complete before
>    its child scsi-disk's callback runs.
> 
>    Is this a good compromise for everyone?

Yes!  Perhaps a bit of a hack, but it works and both the API and the
implementation are very sane.  Converting other devices to use
qdev_add_vm_change_state_handler() is left as an exercise for the
reviewer, I guess? :)

Paolo

Re: [Qemu-devel] [PATCH v5 0/3] virtio-scsi: restart DMA after iothread

[Kevin Wolf]

Am 20.06.2019 um 19:37 hat Stefan Hajnoczi geschrieben:
> v5:
>  * Plumbing vm change state handlers into DeviceClass/BusClass is a rather
>    large bug fix.  Instead I've combined the previous priorities approach with
>    the observation from Kevin and Paolo that we really want to order by qdev
>    tree depth.
> 
>    The new qdev_add_vm_change_state_handler() API lets DeviceStates register
>    callbacks that execute in qdev tree depth order.  This solves the
>    virtio-scsi bug since the virtio-scsi device's callback must complete before
>    its child scsi-disk's callback runs.
> 
>    Is this a good compromise for everyone?

I'd still call it a hack, but I can also see that doing the real thing
would be a lot of work that might not be worth the effort.

Thanks, applied to the block branch.

Kevin

[Qemu-devel] [PATCH v5 1/3] vl: add qemu_add_vm_change_state_handler_prio()

[Stefan Hajnoczi]

Add an API for registering vm change state handlers with a well-defined
ordering.  This is necessary when handlers depend on each other.

Small coding style fixes are included to make checkpatch.pl happy.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/sysemu/sysemu.h |  2 ++
 vl.c                    | 59 ++++++++++++++++++++++++++++++++---------
 2 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index 61579ae71e..984c439ac9 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -29,6 +29,8 @@ typedef void VMChangeStateHandler(void *opaque, int running, RunState state);
 
 VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
                                                      void *opaque);
+VMChangeStateEntry *qemu_add_vm_change_state_handler_prio(
+        VMChangeStateHandler *cb, void *opaque, int priority);
 void qemu_del_vm_change_state_handler(VMChangeStateEntry *e);
 void vm_state_notify(int running, RunState state);
 
diff --git a/vl.c b/vl.c
index 99a56b5556..7fac2ae7ca 100644
--- a/vl.c
+++ b/vl.c
@@ -1471,28 +1471,57 @@ static int machine_help_func(QemuOpts *opts, MachineState *machine)
 struct vm_change_state_entry {
     VMChangeStateHandler *cb;
     void *opaque;
-    QLIST_ENTRY (vm_change_state_entry) entries;
+    QTAILQ_ENTRY(vm_change_state_entry) entries;
+    int priority;
 };
 
-static QLIST_HEAD(, vm_change_state_entry) vm_change_state_head;
+static QTAILQ_HEAD(, vm_change_state_entry) vm_change_state_head;
 
-VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
-                                                     void *opaque)
+/**
+ * qemu_add_vm_change_state_handler_prio:
+ * @cb: the callback to invoke
+ * @opaque: user data passed to the callback
+ * @priority: low priorities execute first when the vm runs and the reverse is
+ *            true when the vm stops
+ *
+ * Register a callback function that is invoked when the vm starts or stops
+ * running.
+ *
+ * Returns: an entry to be freed using qemu_del_vm_change_state_handler()
+ */
+VMChangeStateEntry *qemu_add_vm_change_state_handler_prio(
+        VMChangeStateHandler *cb, void *opaque, int priority)
 {
     VMChangeStateEntry *e;
+    VMChangeStateEntry *other;
 
-    e = g_malloc0(sizeof (*e));
-
+    e = g_malloc0(sizeof(*e));
     e->cb = cb;
     e->opaque = opaque;
-    QLIST_INSERT_HEAD(&vm_change_state_head, e, entries);
+    e->priority = priority;
+
+    /* Keep list sorted in ascending priority order */
+    QTAILQ_FOREACH(other, &vm_change_state_head, entries) {
+        if (priority < other->priority) {
+            QTAILQ_INSERT_BEFORE(other, e, entries);
+            return e;
+        }
+    }
+
+    QTAILQ_INSERT_TAIL(&vm_change_state_head, e, entries);
     return e;
 }
 
+VMChangeStateEntry *qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
+                                                     void *opaque)
+{
+    return qemu_add_vm_change_state_handler_prio(cb, opaque, 0);
+}
+
 void qemu_del_vm_change_state_handler(VMChangeStateEntry *e)
 {
-    QLIST_REMOVE (e, entries);
-    g_free (e);
+    QTAILQ_REMOVE(&vm_change_state_head, e, entries);
+    g_free(e);
 }
 
 void vm_state_notify(int running, RunState state)
@@ -1501,8 +1530,14 @@ void vm_state_notify(int running, RunState state)
 
     trace_vm_state_notify(running, state, RunState_str(state));
 
-    QLIST_FOREACH_SAFE(e, &vm_change_state_head, entries, next) {
-        e->cb(e->opaque, running, state);
+    if (running) {
+        QTAILQ_FOREACH_SAFE(e, &vm_change_state_head, entries, next) {
+            e->cb(e->opaque, running, state);
+        }
+    } else {
+        QTAILQ_FOREACH_REVERSE_SAFE(e, &vm_change_state_head, entries, next) {
+            e->cb(e->opaque, running, state);
+        }
     }
 }
 
@@ -3025,7 +3060,7 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     }
 
-    QLIST_INIT (&vm_change_state_head);
+    QTAILQ_INIT(&vm_change_state_head);
     os_setup_early_signal_handling();
 
     cpu_option = NULL;
-- 
2.21.0