From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
To: Kirti Wankhede <kwankhede@nvidia.com>
Cc: cohuck@redhat.com, cjia@nvidia.com, aik@ozlabs.ru,
Zhengxiao.zx@alibaba-inc.com, shuangtai.tst@alibaba-inc.com,
qemu-devel@nongnu.org, peterx@redhat.com, eauger@redhat.com,
yi.l.liu@intel.com, quintela@redhat.com, ziye.yang@intel.com,
armbru@redhat.com, mlevitsk@redhat.com, pasic@linux.ibm.com,
felipe@nutanix.com, zhi.a.wang@intel.com, kevin.tian@intel.com,
yan.y.zhao@intel.com, alex.williamson@redhat.com,
changpeng.liu@intel.com, eskultet@redhat.com, Ken.Xue@amd.com,
jonathan.davies@nutanix.com, pbonzini@redhat.com
Subject: Re: [PATCH v26 17/17] qapi: Add VFIO devices migration stats in Migration stats
Date: Tue, 29 Sep 2020 11:40:19 +0100 [thread overview]
Message-ID: <20200929104019.GE2826@work-vm> (raw)
In-Reply-To: <1600817059-26721-18-git-send-email-kwankhede@nvidia.com>
* Kirti Wankhede (kwankhede@nvidia.com) wrote:
> Added amount of bytes transferred to the target VM by all VFIO devices
>
> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
> ---
>
> Note: Comments from v25 for this patch are not addressed yet.
> https://www.mail-archive.com/qemu-devel@nongnu.org/msg715620.html
>
> Alex, need more pointer on documentation part raised Markus Armbruster.
I think I'm OK with this from the migration side, except for the minor
5.2's below, so
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Markus is right that we do have lots more information that falls out of
the migration stats, but I'm not sure what more you have to collect.
Dave
>
> hw/vfio/common.c | 20 ++++++++++++++++++++
> hw/vfio/migration.c | 10 ++++++++++
> include/qemu/vfio-helpers.h | 3 +++
> migration/migration.c | 14 ++++++++++++++
> monitor/hmp-cmds.c | 6 ++++++
> qapi/migration.json | 17 +++++++++++++++++
> 6 files changed, 70 insertions(+)
>
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 7eeaa368187a..286cdaac8674 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -39,6 +39,7 @@
> #include "trace.h"
> #include "qapi/error.h"
> #include "migration/migration.h"
> +#include "qemu/vfio-helpers.h"
>
> VFIOGroupList vfio_group_list =
> QLIST_HEAD_INITIALIZER(vfio_group_list);
> @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = {
> * Device state interfaces
> */
>
> +bool vfio_mig_active(void)
> +{
> + VFIOGroup *group;
> + VFIODevice *vbasedev;
> +
> + if (QLIST_EMPTY(&vfio_group_list)) {
> + return false;
> + }
> +
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + QLIST_FOREACH(vbasedev, &group->device_list, next) {
> + if (vbasedev->migration_blocker) {
> + return false;
> + }
> + }
> + }
> + return true;
> +}
> +
> static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container)
> {
> VFIOGroup *group;
> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
> index 822b68b4e015..c4226fa8b183 100644
> --- a/hw/vfio/migration.c
> +++ b/hw/vfio/migration.c
> @@ -28,6 +28,7 @@
> #include "pci.h"
> #include "trace.h"
> #include "hw/hw.h"
> +#include "qemu/vfio-helpers.h"
>
> /*
> * Flags used as delimiter:
> @@ -40,6 +41,8 @@
> #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
> #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
>
> +static int64_t bytes_transferred;
> +
> static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
> off_t off, bool iswrite)
> {
> @@ -289,6 +292,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size)
> *size = data_size;
> }
>
> + bytes_transferred += data_size;
> return ret;
> }
>
> @@ -770,6 +774,7 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data)
> }
>
> vfio_set_dirty_page_tracking(vbasedev, false);
> + bytes_transferred = 0;
> }
> }
>
> @@ -820,6 +825,11 @@ static int vfio_migration_init(VFIODevice *vbasedev,
>
> /* ---------------------------------------------------------------------- */
>
> +int64_t vfio_mig_bytes_transferred(void)
> +{
> + return bytes_transferred;
> +}
> +
> int vfio_migration_probe(VFIODevice *vbasedev, Error **errp)
> {
> VFIOContainer *container = vbasedev->group->container;
> diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h
> index 1f057c2b9e40..26a7df0767b1 100644
> --- a/include/qemu/vfio-helpers.h
> +++ b/include/qemu/vfio-helpers.h
> @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,
> int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,
> int irq_type, Error **errp);
>
> +bool vfio_mig_active(void);
> +int64_t vfio_mig_bytes_transferred(void);
> +
> #endif
> diff --git a/migration/migration.c b/migration/migration.c
> index 58a5452471f9..b204bb1f6cd9 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -56,6 +56,7 @@
> #include "net/announce.h"
> #include "qemu/queue.h"
> #include "multifd.h"
> +#include "qemu/vfio-helpers.h"
>
> #define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */
>
> @@ -996,6 +997,17 @@ static void populate_disk_info(MigrationInfo *info)
> }
> }
>
> +static void populate_vfio_info(MigrationInfo *info)
> +{
> +#ifdef CONFIG_LINUX
> + if (vfio_mig_active()) {
> + info->has_vfio = true;
> + info->vfio = g_malloc0(sizeof(*info->vfio));
> + info->vfio->transferred = vfio_mig_bytes_transferred();
> + }
> +#endif
> +}
> +
> static void fill_source_migration_info(MigrationInfo *info)
> {
> MigrationState *s = migrate_get_current();
> @@ -1020,6 +1032,7 @@ static void fill_source_migration_info(MigrationInfo *info)
> populate_time_info(info, s);
> populate_ram_info(info, s);
> populate_disk_info(info);
> + populate_vfio_info(info);
> break;
> case MIGRATION_STATUS_COLO:
> info->has_status = true;
> @@ -1028,6 +1041,7 @@ static void fill_source_migration_info(MigrationInfo *info)
> case MIGRATION_STATUS_COMPLETED:
> populate_time_info(info, s);
> populate_ram_info(info, s);
> + populate_vfio_info(info);
> break;
> case MIGRATION_STATUS_FAILED:
> info->has_status = true;
> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
> index 7711726fd222..40d60d6a6651 100644
> --- a/monitor/hmp-cmds.c
> +++ b/monitor/hmp-cmds.c
> @@ -355,6 +355,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict)
> }
> monitor_printf(mon, "]\n");
> }
> +
> + if (info->has_vfio) {
> + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n",
> + info->vfio->transferred >> 10);
> + }
> +
> qapi_free_MigrationInfo(info);
> }
>
> diff --git a/qapi/migration.json b/qapi/migration.json
> index 675f70bb6734..3535977123d3 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -147,6 +147,18 @@
> 'active', 'postcopy-active', 'postcopy-paused',
> 'postcopy-recover', 'completed', 'failed', 'colo',
> 'pre-switchover', 'device', 'wait-unplug' ] }
> +##
> +# @VfioStats:
> +#
> +# Detailed VFIO devices migration statistics
> +#
> +# @transferred: amount of bytes transferred to the target VM by VFIO devices
> +#
> +# Since: 5.1
> +#
> +##
> +{ 'struct': 'VfioStats',
> + 'data': {'transferred': 'int' } }
>
> ##
> # @MigrationInfo:
> @@ -208,11 +220,16 @@
> #
> # @socket-address: Only used for tcp, to know what the real port is (Since 4.0)
> #
> +# @vfio: @VfioStats containing detailed VFIO devices migration statistics,
> +# only returned if VFIO device is present, migration is supported by all
> +# VFIO devices and status is 'active' or 'completed' (since 5.1)
> +#
> # Since: 0.14.0
> ##
> { 'struct': 'MigrationInfo',
> 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats',
> '*disk': 'MigrationStats',
> + '*vfio': 'VfioStats',
> '*xbzrle-cache': 'XBZRLECacheStats',
> '*total-time': 'int',
> '*expected-downtime': 'int',
> --
> 2.7.0
>
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
next prev parent reply other threads:[~2020-09-29 10:45 UTC|newest]
Thread overview: 72+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-22 23:24 [PATCH QEMU v25 00/17] Add migration support for VFIO devices Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 01/17] vfio: Add function to unmap VFIO region Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 02/17] vfio: Add vfio_get_object callback to VFIODeviceOps Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 03/17] vfio: Add save and load functions for VFIO PCI devices Kirti Wankhede
2020-09-23 6:38 ` Zenghui Yu
2020-09-24 22:49 ` Alex Williamson
2020-10-21 9:30 ` Zenghui Yu
2020-10-21 19:03 ` Alex Williamson
2020-09-22 23:24 ` [PATCH v26 04/17] vfio: Add migration region initialization and finalize function Kirti Wankhede
2020-09-24 14:08 ` Cornelia Huck
2020-10-17 20:14 ` Kirti Wankhede
2020-09-25 20:20 ` Alex Williamson
2020-09-28 9:39 ` Cornelia Huck
2020-10-17 20:17 ` Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 05/17] vfio: Add VM state change handler to know state of VM Kirti Wankhede
2020-09-24 15:02 ` Cornelia Huck
2020-09-29 11:03 ` Dr. David Alan Gilbert
2020-10-17 20:24 ` Kirti Wankhede
2020-10-20 10:51 ` Cornelia Huck
2020-10-21 5:33 ` Kirti Wankhede
2020-10-22 7:51 ` Cornelia Huck
2020-10-22 15:42 ` Kirti Wankhede
2020-10-22 15:49 ` Cornelia Huck
2020-09-25 20:20 ` Alex Williamson
2020-10-17 20:30 ` Kirti Wankhede
2020-10-17 23:44 ` Alex Williamson
2020-10-18 17:43 ` Kirti Wankhede
2020-10-19 17:51 ` Alex Williamson
2020-10-20 10:23 ` Cornelia Huck
2020-09-22 23:24 ` [PATCH v26 06/17] vfio: Add migration state change notifier Kirti Wankhede
2020-09-25 20:20 ` Alex Williamson
2020-10-17 20:35 ` Kirti Wankhede
2020-10-19 17:57 ` Alex Williamson
2020-10-20 10:55 ` Cornelia Huck
2020-09-22 23:24 ` [PATCH v26 07/17] vfio: Register SaveVMHandlers for VFIO device Kirti Wankhede
2020-09-24 15:15 ` Philippe Mathieu-Daudé
2020-09-29 10:19 ` Dr. David Alan Gilbert
2020-10-17 20:36 ` Kirti Wankhede
2020-09-25 11:53 ` Cornelia Huck
2020-10-18 20:55 ` Kirti Wankhede
2020-10-20 15:51 ` Cornelia Huck
2020-09-25 20:20 ` Alex Williamson
2020-10-18 17:40 ` Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 08/17] vfio: Add save state functions to SaveVMHandlers Kirti Wankhede
2020-09-23 11:42 ` Wang, Zhi A
2020-10-21 14:30 ` Kirti Wankhede
2020-09-25 21:02 ` Alex Williamson
2020-10-18 18:00 ` Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 09/17] vfio: Add load " Kirti Wankhede
2020-10-01 10:07 ` Cornelia Huck
2020-10-18 20:47 ` Kirti Wankhede
2020-10-20 16:25 ` Cornelia Huck
2020-09-22 23:24 ` [PATCH v26 10/17] memory: Set DIRTY_MEMORY_MIGRATION when IOMMU is enabled Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 11/17] vfio: Get migration capability flags for container Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 12/17] vfio: Add function to start and stop dirty pages tracking Kirti Wankhede
2020-09-25 21:55 ` Alex Williamson
2020-10-18 20:52 ` Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 13/17] vfio: create mapped iova list when vIOMMU is enabled Kirti Wankhede
2020-09-25 22:23 ` Alex Williamson
2020-10-19 6:01 ` Kirti Wankhede
2020-10-19 17:24 ` Alex Williamson
2020-10-19 19:15 ` Kirti Wankhede
2020-10-19 20:07 ` Alex Williamson
2020-09-22 23:24 ` [PATCH v26 14/17] vfio: Add vfio_listener_log_sync to mark dirty pages Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 15/17] vfio: Add ioctl to get dirty pages bitmap during dma unmap Kirti Wankhede
2020-09-22 23:24 ` [PATCH v26 16/17] vfio: Make vfio-pci device migration capable Kirti Wankhede
2020-09-25 12:17 ` Cornelia Huck
2020-09-22 23:24 ` [PATCH v26 17/17] qapi: Add VFIO devices migration stats in Migration stats Kirti Wankhede
2020-09-24 15:14 ` Eric Blake
2020-09-25 22:55 ` Alex Williamson
2020-09-29 10:40 ` Dr. David Alan Gilbert [this message]
2020-09-23 7:06 ` [PATCH QEMU v25 00/17] Add migration support for VFIO devices Zenghui Yu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200929104019.GE2826@work-vm \
--to=dgilbert@redhat.com \
--cc=Ken.Xue@amd.com \
--cc=Zhengxiao.zx@alibaba-inc.com \
--cc=aik@ozlabs.ru \
--cc=alex.williamson@redhat.com \
--cc=armbru@redhat.com \
--cc=changpeng.liu@intel.com \
--cc=cjia@nvidia.com \
--cc=cohuck@redhat.com \
--cc=eauger@redhat.com \
--cc=eskultet@redhat.com \
--cc=felipe@nutanix.com \
--cc=jonathan.davies@nutanix.com \
--cc=kevin.tian@intel.com \
--cc=kwankhede@nvidia.com \
--cc=mlevitsk@redhat.com \
--cc=pasic@linux.ibm.com \
--cc=pbonzini@redhat.com \
--cc=peterx@redhat.com \
--cc=qemu-devel@nongnu.org \
--cc=quintela@redhat.com \
--cc=shuangtai.tst@alibaba-inc.com \
--cc=yan.y.zhao@intel.com \
--cc=yi.l.liu@intel.com \
--cc=zhi.a.wang@intel.com \
--cc=ziye.yang@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).