From: Kirti Wankhede <kwankhede@nvidia.com>
To: Alex Williamson <alex.williamson@redhat.com>
Cc: cohuck@redhat.com, cjia@nvidia.com, zhi.wang.linux@gmail.com,
aik@ozlabs.ru, Zhengxiao.zx@Alibaba-inc.com,
shuangtai.tst@alibaba-inc.com, qemu-devel@nongnu.org,
peterx@redhat.com, eauger@redhat.com, yi.l.liu@intel.com,
quintela@redhat.com, ziye.yang@intel.com, armbru@redhat.com,
mlevitsk@redhat.com, pasic@linux.ibm.com, felipe@nutanix.com,
zhi.a.wang@intel.com, mcrossley@nvidia.com, kevin.tian@intel.com,
yan.y.zhao@intel.com, dgilbert@redhat.com,
changpeng.liu@intel.com, eskultet@redhat.com, Ken.Xue@amd.com,
jonathan.davies@nutanix.com, pbonzini@redhat.com,
dnigam@nvidia.com
Subject: Re: [PATCH v27 17/17] qapi: Add VFIO devices migration stats in Migration stats
Date: Fri, 23 Oct 2020 15:51:13 +0530 [thread overview]
Message-ID: <4caf6f0b-b291-8e73-fa5f-5357c6f194ff@nvidia.com> (raw)
In-Reply-To: <20201022161842.30585c2e@w520.home>
On 10/23/2020 3:48 AM, Alex Williamson wrote:
> On Thu, 22 Oct 2020 16:42:07 +0530
> Kirti Wankhede <kwankhede@nvidia.com> wrote:
>
>> Added amount of bytes transferred to the VM at destination by all VFIO
>> devices
>>
>> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com>
>> Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
>> ---
>> hw/vfio/common.c | 20 ++++++++++++++++++++
>> hw/vfio/migration.c | 10 ++++++++++
>> include/qemu/vfio-helpers.h | 3 +++
>> migration/migration.c | 14 ++++++++++++++
>> monitor/hmp-cmds.c | 6 ++++++
>> qapi/migration.json | 17 +++++++++++++++++
>> 6 files changed, 70 insertions(+)
>>
>> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
>> index 9c879e5c0f62..8d0758eda9fa 100644
>> --- a/hw/vfio/common.c
>> +++ b/hw/vfio/common.c
>> @@ -39,6 +39,7 @@
>> #include "trace.h"
>> #include "qapi/error.h"
>> #include "migration/migration.h"
>> +#include "qemu/vfio-helpers.h"
>>
>> VFIOGroupList vfio_group_list =
>> QLIST_HEAD_INITIALIZER(vfio_group_list);
>> @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = {
>> * Device state interfaces
>> */
>>
>> +bool vfio_mig_active(void)
>> +{
>> + VFIOGroup *group;
>> + VFIODevice *vbasedev;
>> +
>> + if (QLIST_EMPTY(&vfio_group_list)) {
>> + return false;
>> + }
>> +
>> + QLIST_FOREACH(group, &vfio_group_list, next) {
>> + QLIST_FOREACH(vbasedev, &group->device_list, next) {
>> + if (vbasedev->migration_blocker) {
>> + return false;
>> + }
>> + }
>> + }
>> + return true;
>> +}
>> +
>> static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container)
>> {
>> VFIOGroup *group;
>> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c
>> index 77ee60a43ea5..b23e21c6de2b 100644
>> --- a/hw/vfio/migration.c
>> +++ b/hw/vfio/migration.c
>> @@ -28,6 +28,7 @@
>> #include "pci.h"
>> #include "trace.h"
>> #include "hw/hw.h"
>> +#include "qemu/vfio-helpers.h"
>>
>> /*
>> * Flags to be used as unique delimiters for VFIO devices in the migration
>> @@ -45,6 +46,8 @@
>> #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL)
>> #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL)
>>
>> +static int64_t bytes_transferred;
>> +
>> static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count,
>> off_t off, bool iswrite)
>> {
>> @@ -255,6 +258,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size)
>> *size = data_size;
>> }
>>
>> + bytes_transferred += data_size;
>> return ret;
>> }
>>
>> @@ -776,6 +780,7 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data)
>> case MIGRATION_STATUS_CANCELLING:
>> case MIGRATION_STATUS_CANCELLED:
>> case MIGRATION_STATUS_FAILED:
>> + bytes_transferred = 0;
>> ret = vfio_migration_set_state(vbasedev,
>> ~(VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RESUMING),
>> VFIO_DEVICE_STATE_RUNNING);
>> @@ -862,6 +867,11 @@ err:
>>
>> /* ---------------------------------------------------------------------- */
>>
>> +int64_t vfio_mig_bytes_transferred(void)
>> +{
>> + return bytes_transferred;
>> +}
>> +
>> int vfio_migration_probe(VFIODevice *vbasedev, Error **errp)
>> {
>> VFIOContainer *container = vbasedev->group->container;
>> diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h
>> index 4491c8e1a6e9..7f7a46e6ef2d 100644
>> --- a/include/qemu/vfio-helpers.h
>> +++ b/include/qemu/vfio-helpers.h
>> @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,
>> int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,
>> int irq_type, Error **errp);
>>
>> +bool vfio_mig_active(void);
>> +int64_t vfio_mig_bytes_transferred(void);
>> +
>> #endif
>
>
> I don't think vfio-helpers is the right place for this, this header is
> specifically for using util/vfio-helpers.c. Would
> include/hw/vfio/vfio-common.h work?
>
>
Yes, works with CONFIG_VFIO check. Changing it.
>> diff --git a/migration/migration.c b/migration/migration.c
>> index 0575ecb37953..8b2865d25ef4 100644
>> --- a/migration/migration.c
>> +++ b/migration/migration.c
>> @@ -56,6 +56,7 @@
>> #include "net/announce.h"
>> #include "qemu/queue.h"
>> #include "multifd.h"
>> +#include "qemu/vfio-helpers.h"
>>
>> #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */
>>
>> @@ -1002,6 +1003,17 @@ static void populate_disk_info(MigrationInfo *info)
>> }
>> }
>>
>> +static void populate_vfio_info(MigrationInfo *info)
>> +{
>> +#ifdef CONFIG_LINUX
>
> Use CONFIG_VFIO? I get a build failure on qemu-system-avr
>
> /usr/bin/ld: /tmp/tmp.3QbqxgbENl/build/../migration/migration.c:1012:
> undefined reference to `vfio_mig_bytes_transferred'. Thanks,
>
Ok Changing it.
> Alex
>
>> + if (vfio_mig_active()) {
>> + info->has_vfio = true;
>> + info->vfio = g_malloc0(sizeof(*info->vfio));
>> + info->vfio->transferred = vfio_mig_bytes_transferred();
>> + }
>> +#endif
>> +}
>> +
>> static void fill_source_migration_info(MigrationInfo *info)
>> {
>> MigrationState *s = migrate_get_current();
>> @@ -1026,6 +1038,7 @@ static void fill_source_migration_info(MigrationInfo *info)
>> populate_time_info(info, s);
>> populate_ram_info(info, s);
>> populate_disk_info(info);
>> + populate_vfio_info(info);
>> break;
>> case MIGRATION_STATUS_COLO:
>> info->has_status = true;
>> @@ -1034,6 +1047,7 @@ static void fill_source_migration_info(MigrationInfo *info)
>> case MIGRATION_STATUS_COMPLETED:
>> populate_time_info(info, s);
>> populate_ram_info(info, s);
>> + populate_vfio_info(info);
>> break;
>> case MIGRATION_STATUS_FAILED:
>> info->has_status = true;
>> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
>> index 9789f4277f50..56e9bad33d94 100644
>> --- a/monitor/hmp-cmds.c
>> +++ b/monitor/hmp-cmds.c
>> @@ -357,6 +357,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict)
>> }
>> monitor_printf(mon, "]\n");
>> }
>> +
>> + if (info->has_vfio) {
>> + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n",
>> + info->vfio->transferred >> 10);
>> + }
>> +
>> qapi_free_MigrationInfo(info);
>> }
>>
>> diff --git a/qapi/migration.json b/qapi/migration.json
>> index a5da513c9e05..3c7582052725 100644
>> --- a/qapi/migration.json
>> +++ b/qapi/migration.json
>> @@ -147,6 +147,18 @@
>> 'active', 'postcopy-active', 'postcopy-paused',
>> 'postcopy-recover', 'completed', 'failed', 'colo',
>> 'pre-switchover', 'device', 'wait-unplug' ] }
>> +##
>> +# @VfioStats:
>> +#
>> +# Detailed VFIO devices migration statistics
>> +#
>> +# @transferred: amount of bytes transferred to the target VM by VFIO devices
>> +#
>> +# Since: 5.2
>> +#
>> +##
>> +{ 'struct': 'VfioStats',
>> + 'data': {'transferred': 'int' } }
>>
>> ##
>> # @MigrationInfo:
>> @@ -208,11 +220,16 @@
>> #
>> # @socket-address: Only used for tcp, to know what the real port is (Since 4.0)
>> #
>> +# @vfio: @VfioStats containing detailed VFIO devices migration statistics,
>> +# only returned if VFIO device is present, migration is supported by all
>> +# VFIO devices and status is 'active' or 'completed' (since 5.2)
>> +#
>> # Since: 0.14.0
>> ##
>> { 'struct': 'MigrationInfo',
>> 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats',
>> '*disk': 'MigrationStats',
>> + '*vfio': 'VfioStats',
>> '*xbzrle-cache': 'XBZRLECacheStats',
>> '*total-time': 'int',
>> '*expected-downtime': 'int',
>
next prev parent reply other threads:[~2020-10-23 10:26 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-10-22 11:11 [PATCH v27 00/17] Add migration support for VFIO devices Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 01/17] vfio: Add function to unmap VFIO region Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 02/17] vfio: Add vfio_get_object callback to VFIODeviceOps Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 03/17] vfio: Add save and load functions for VFIO PCI devices Kirti Wankhede
2020-10-22 14:06 ` Alex Williamson
2020-10-22 15:52 ` Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 04/17] vfio: Add migration region initialization and finalize function Kirti Wankhede
2020-10-22 14:22 ` Alex Williamson
2020-10-22 16:16 ` Kirti Wankhede
2020-10-23 11:17 ` Cornelia Huck
2020-10-22 11:11 ` [PATCH v27 05/17] vfio: Add VM state change handler to know state of VM Kirti Wankhede
2020-10-22 16:35 ` Alex Williamson
2020-10-22 17:41 ` Kirti Wankhede
2020-10-22 18:29 ` Alex Williamson
2020-10-22 11:11 ` [PATCH v27 06/17] vfio: Add migration state change notifier Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 07/17] vfio: Register SaveVMHandlers for VFIO device Kirti Wankhede
2020-10-22 18:51 ` Alex Williamson
2020-10-23 7:12 ` Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 08/17] vfio: Add save state functions to SaveVMHandlers Kirti Wankhede
2020-10-22 11:11 ` [PATCH v27 09/17] vfio: Add load " Kirti Wankhede
2020-10-22 19:50 ` Alex Williamson
2020-10-23 9:59 ` Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 10/17] memory: Set DIRTY_MEMORY_MIGRATION when IOMMU is enabled Kirti Wankhede
2020-10-22 19:52 ` Alex Williamson
2020-10-22 11:12 ` [PATCH v27 11/17] vfio: Get migration capability flags for container Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 12/17] vfio: Add function to start and stop dirty pages tracking Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 13/17] vfio: Add vfio_listener_log_sync to mark dirty pages Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 14/17] vfio: Dirty page tracking when vIOMMU is enabled Kirti Wankhede
2020-10-22 20:37 ` Alex Williamson
2020-10-23 7:55 ` Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 15/17] vfio: Add ioctl to get dirty pages bitmap during dma unmap Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 16/17] vfio: Make vfio-pci device migration capable Kirti Wankhede
2020-10-22 11:12 ` [PATCH v27 17/17] qapi: Add VFIO devices migration stats in Migration stats Kirti Wankhede
2020-10-22 22:18 ` Alex Williamson
2020-10-23 10:21 ` Kirti Wankhede [this message]
2020-10-22 21:28 ` [PATCH v27 00/17] Add migration support for VFIO devices Alex Williamson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=4caf6f0b-b291-8e73-fa5f-5357c6f194ff@nvidia.com \
--to=kwankhede@nvidia.com \
--cc=Ken.Xue@amd.com \
--cc=Zhengxiao.zx@Alibaba-inc.com \
--cc=aik@ozlabs.ru \
--cc=alex.williamson@redhat.com \
--cc=armbru@redhat.com \
--cc=changpeng.liu@intel.com \
--cc=cjia@nvidia.com \
--cc=cohuck@redhat.com \
--cc=dgilbert@redhat.com \
--cc=dnigam@nvidia.com \
--cc=eauger@redhat.com \
--cc=eskultet@redhat.com \
--cc=felipe@nutanix.com \
--cc=jonathan.davies@nutanix.com \
--cc=kevin.tian@intel.com \
--cc=mcrossley@nvidia.com \
--cc=mlevitsk@redhat.com \
--cc=pasic@linux.ibm.com \
--cc=pbonzini@redhat.com \
--cc=peterx@redhat.com \
--cc=qemu-devel@nongnu.org \
--cc=quintela@redhat.com \
--cc=shuangtai.tst@alibaba-inc.com \
--cc=yan.y.zhao@intel.com \
--cc=yi.l.liu@intel.com \
--cc=zhi.a.wang@intel.com \
--cc=zhi.wang.linux@gmail.com \
--cc=ziye.yang@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.