All of lore.kernel.org
 help / color / mirror / Atom feed
From: Alex Williamson <alex.williamson@redhat.com>
To: qemu-devel@nongnu.org
Cc: Yan Zhao <yan.y.zhao@intel.com>,
	Kirti Wankhede <kwankhede@nvidia.com>,
	Cornelia Huck <cohuck@redhat.com>, Neo Jia <cjia@nvidia.com>
Subject: [PULL v2 07/32] vfio: Register SaveVMHandlers for VFIO device
Date: Wed, 28 Oct 2020 10:40:28 -0600	[thread overview]
Message-ID: <160390322826.12234.6164436453856785935.stgit@gimli.home> (raw)
In-Reply-To: <160390309510.12234.8858324597971641979.stgit@gimli.home>

From: Kirti Wankhede <kwankhede@nvidia.com>

Define flags to be used as delimiter in migration stream for VFIO devices.
Added .save_setup and .save_cleanup functions. Map & unmap migration
region from these functions at source during saving or pre-copy phase.

Set VFIO device state depending on VM's state. During live migration, VM is
running when .save_setup is called, _SAVING | _RUNNING state is set for VFIO
device. During save-restore, VM is paused, _SAVING state is set for VFIO device.

Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
Reviewed-by: Neo Jia <cjia@nvidia.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
Reviewed-by: Yan Zhao <yan.y.zhao@intel.com>
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
---
 hw/vfio/migration.c  |  102 ++++++++++++++++++++++++++++++++++++++++++++++++++
 hw/vfio/trace-events |    2 +
 2 files changed, 104 insertions(+)

diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
index 7ec85b6469c5..ca6fd896655b 100644
--- a/hw/vfio/migration.c
+++ b/hw/vfio/migration.c
@@ -8,12 +8,15 @@
  */
 
 #include "qemu/osdep.h"
+#include "qemu/main-loop.h"
+#include "qemu/cutils.h"
 #include <linux/vfio.h>
 
 #include "sysemu/runstate.h"
 #include "hw/vfio/vfio-common.h"
 #include "cpu.h"
 #include "migration/migration.h"
+#include "migration/vmstate.h"
 #include "migration/qemu-file.h"
 #include "migration/register.h"
 #include "migration/blocker.h"
@@ -25,6 +28,22 @@
 #include "trace.h"
 #include "hw/hw.h"
 
+/*
+ * Flags to be used as unique delimiters for VFIO devices in the migration
+ * stream. These flags are composed as:
+ * 0xffffffff => MSB 32-bit all 1s
+ * 0xef10     => Magic ID, represents emulated (virtual) function IO
+ * 0x0000     => 16-bits reserved for flags
+ *
+ * The beginning of state information is marked by _DEV_CONFIG_STATE,
+ * _DEV_SETUP_STATE, or _DEV_DATA_STATE, respectively. The end of a
+ * certain state information is marked by _END_OF_STATE.
+ */
+#define VFIO_MIG_FLAG_END_OF_STATE      (0xffffffffef100001ULL)
+#define VFIO_MIG_FLAG_DEV_CONFIG_STATE  (0xffffffffef100002ULL)
+#define VFIO_MIG_FLAG_DEV_SETUP_STATE   (0xffffffffef100003ULL)
+#define VFIO_MIG_FLAG_DEV_DATA_STATE    (0xffffffffef100004ULL)
+
 static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
                                   off_t off, bool iswrite)
 {
@@ -129,6 +148,75 @@ static int vfio_migration_set_state(VFIODevice *vbasedev, uint32_t mask,
     return 0;
 }
 
+static void vfio_migration_cleanup(VFIODevice *vbasedev)
+{
+    VFIOMigration *migration = vbasedev->migration;
+
+    if (migration->region.mmaps) {
+        vfio_region_unmap(&migration->region);
+    }
+}
+
+/* ---------------------------------------------------------------------- */
+
+static int vfio_save_setup(QEMUFile *f, void *opaque)
+{
+    VFIODevice *vbasedev = opaque;
+    VFIOMigration *migration = vbasedev->migration;
+    int ret;
+
+    trace_vfio_save_setup(vbasedev->name);
+
+    qemu_put_be64(f, VFIO_MIG_FLAG_DEV_SETUP_STATE);
+
+    if (migration->region.mmaps) {
+        /*
+         * Calling vfio_region_mmap() from migration thread. Memory API called
+         * from this function require locking the iothread when called from
+         * outside the main loop thread.
+         */
+        qemu_mutex_lock_iothread();
+        ret = vfio_region_mmap(&migration->region);
+        qemu_mutex_unlock_iothread();
+        if (ret) {
+            error_report("%s: Failed to mmap VFIO migration region: %s",
+                         vbasedev->name, strerror(-ret));
+            error_report("%s: Falling back to slow path", vbasedev->name);
+        }
+    }
+
+    ret = vfio_migration_set_state(vbasedev, VFIO_DEVICE_STATE_MASK,
+                                   VFIO_DEVICE_STATE_SAVING);
+    if (ret) {
+        error_report("%s: Failed to set state SAVING", vbasedev->name);
+        return ret;
+    }
+
+    qemu_put_be64(f, VFIO_MIG_FLAG_END_OF_STATE);
+
+    ret = qemu_file_get_error(f);
+    if (ret) {
+        return ret;
+    }
+
+    return 0;
+}
+
+static void vfio_save_cleanup(void *opaque)
+{
+    VFIODevice *vbasedev = opaque;
+
+    vfio_migration_cleanup(vbasedev);
+    trace_vfio_save_cleanup(vbasedev->name);
+}
+
+static SaveVMHandlers savevm_vfio_handlers = {
+    .save_setup = vfio_save_setup,
+    .save_cleanup = vfio_save_cleanup,
+};
+
+/* ---------------------------------------------------------------------- */
+
 static void vfio_vmstate_change(void *opaque, int running, RunState state)
 {
     VFIODevice *vbasedev = opaque;
@@ -215,6 +303,8 @@ static int vfio_migration_init(VFIODevice *vbasedev,
     int ret;
     Object *obj;
     VFIOMigration *migration;
+    char id[256] = "";
+    g_autofree char *path = NULL, *oid = NULL;
 
     if (!vbasedev->ops->vfio_get_object) {
         return -EINVAL;
@@ -244,6 +334,18 @@ static int vfio_migration_init(VFIODevice *vbasedev,
 
     migration = vbasedev->migration;
     migration->vbasedev = vbasedev;
+
+    oid = vmstate_if_get_id(VMSTATE_IF(DEVICE(obj)));
+    if (oid) {
+        path = g_strdup_printf("%s/vfio", oid);
+    } else {
+        path = g_strdup("vfio");
+    }
+    strpadcpy(id, sizeof(id), path, '\0');
+
+    register_savevm_live(id, VMSTATE_INSTANCE_ID_ANY, 1, &savevm_vfio_handlers,
+                         vbasedev);
+
     migration->vm_state = qemu_add_vm_change_state_handler(vfio_vmstate_change,
                                                            vbasedev);
     migration->migration_state.notify = vfio_migration_state_notifier;
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 78d7d83b5ef8..f148b5e828c1 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -151,3 +151,5 @@ vfio_migration_probe(const char *name, uint32_t index) " (%s) Region %d"
 vfio_migration_set_state(const char *name, uint32_t state) " (%s) state %d"
 vfio_vmstate_change(const char *name, int running, const char *reason, uint32_t dev_state) " (%s) running %d reason %s device state %d"
 vfio_migration_state_notifier(const char *name, const char *state) " (%s) state %s"
+vfio_save_setup(const char *name) " (%s)"
+vfio_save_cleanup(const char *name) " (%s)"



  parent reply	other threads:[~2020-10-28 16:53 UTC|newest]

Thread overview: 35+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-10-28 16:39 [PULL v2 00/32] VFIO updates 2020-10-28 (for QEMU 5.2 soft-freeze) Alex Williamson
2020-10-28 16:39 ` [PULL v2 01/32] vfio: Add function to unmap VFIO region Alex Williamson
2020-10-28 16:39 ` [PULL v2 02/32] vfio: Add vfio_get_object callback to VFIODeviceOps Alex Williamson
2020-10-28 16:39 ` [PULL v2 03/32] vfio: Add save and load functions for VFIO PCI devices Alex Williamson
2020-10-28 16:40 ` [PULL v2 04/32] vfio: Add migration region initialization and finalize function Alex Williamson
2020-10-28 16:40 ` [PULL v2 05/32] vfio: Add VM state change handler to know state of VM Alex Williamson
2020-10-28 16:40 ` [PULL v2 06/32] vfio: Add migration state change notifier Alex Williamson
2020-10-28 16:40 ` Alex Williamson [this message]
2020-10-28 16:40 ` [PULL v2 08/32] vfio: Add save state functions to SaveVMHandlers Alex Williamson
2020-10-28 16:40 ` [PULL v2 09/32] vfio: Add load " Alex Williamson
2020-10-28 16:40 ` [PULL v2 10/32] memory: Set DIRTY_MEMORY_MIGRATION when IOMMU is enabled Alex Williamson
2020-10-28 16:41 ` [PULL v2 11/32] vfio: Get migration capability flags for container Alex Williamson
2020-10-28 16:41 ` [PULL v2 12/32] vfio: Add function to start and stop dirty pages tracking Alex Williamson
2020-10-28 16:41 ` [PULL v2 13/32] vfio: Add vfio_listener_log_sync to mark dirty pages Alex Williamson
2020-10-28 16:41 ` [PULL v2 14/32] vfio: Dirty page tracking when vIOMMU is enabled Alex Williamson
2020-10-28 16:41 ` [PULL v2 15/32] vfio: Add ioctl to get dirty pages bitmap during dma unmap Alex Williamson
2020-10-28 16:41 ` [PULL v2 16/32] vfio: Make vfio-pci device migration capable Alex Williamson
2020-10-28 16:42 ` [PULL v2 17/32] qapi: Add VFIO devices migration stats in Migration stats Alex Williamson
2020-10-28 16:42 ` [PULL v2 18/32] update-linux-headers: Add vfio_zdev.h Alex Williamson
2020-10-28 16:42 ` [PULL v2 19/32] linux-headers: update against 5.10-rc1 Alex Williamson
2020-10-28 16:42 ` [PULL v2 20/32] s390x/pci: Move header files to include/hw/s390x Alex Williamson
2020-10-28 16:42 ` [PULL v2 21/32] vfio: Create shared routine for scanning info capabilities Alex Williamson
2020-10-28 16:42 ` [PULL v2 22/32] vfio: Find DMA available capability Alex Williamson
2020-10-28 16:42 ` [PULL v2 23/32] s390x/pci: Add routine to get the vfio dma available count Alex Williamson
2020-10-28 16:42 ` [PULL v2 24/32] s390x/pci: Honor DMA limits set by vfio Alex Williamson
2020-10-28 16:43 ` [PULL v2 25/32] s390x/pci: create a header dedicated to PCI CLP Alex Williamson
2020-10-28 16:43 ` [PULL v2 26/32] s390x/pci: use a PCI Group structure Alex Williamson
2020-10-28 16:43 ` [PULL v2 27/32] s390x/pci: clean up s390 PCI groups Alex Williamson
2020-10-28 16:43 ` [PULL v2 28/32] s390x/pci: use a PCI Function structure Alex Williamson
2020-10-28 16:43 ` [PULL v2 29/32] vfio: Add routine for finding VFIO_DEVICE_GET_INFO capabilities Alex Williamson
2020-10-28 16:43 ` [PULL v2 30/32] s390x/pci: get zPCI function info from host Alex Williamson
2020-10-28 16:44 ` [PULL v2 31/32] hw/vfio: Use lock guard macros Alex Williamson
2020-10-28 16:44 ` [PULL v2 32/32] vfio: fix incorrect print type Alex Williamson
2020-10-31 14:54 ` [PULL v2 00/32] VFIO updates 2020-10-28 (for QEMU 5.2 soft-freeze) Peter Maydell
2020-11-01 20:46   ` Alex Williamson

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=160390322826.12234.6164436453856785935.stgit@gimli.home \
    --to=alex.williamson@redhat.com \
    --cc=cjia@nvidia.com \
    --cc=cohuck@redhat.com \
    --cc=kwankhede@nvidia.com \
    --cc=qemu-devel@nongnu.org \
    --cc=yan.y.zhao@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.