* [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
@ 2022-05-17 6:35 ` huangy81
2022-05-18 15:05 ` Eric Blake
2022-05-17 6:35 ` [RFC 2/6] qapi/migration: Introduce vcpu-dirtylimit parameters huangy81
` (4 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Introduce "vcpu-dirtylimit-period" migration parameters,
which is used to makes dirtyrate calculation period
configurable.
To implement that, refactor vcpu_dirty_rate_stat_collect
so that period can be configured instead of hardcode.
Meanwhile, introduce migrate_dirtylimit function to help
check if dirtylimit enabled during live migration, set
it false by default.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/migration.c | 20 ++++++++++++++++++++
migration/migration.h | 1 +
monitor/hmp-cmds.c | 8 ++++++++
qapi/migration.json | 19 +++++++++++++++----
softmmu/dirtylimit.c | 11 ++++++++++-
5 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/migration/migration.c b/migration/migration.c
index 695f0f2..5e20b2a 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -116,6 +116,8 @@
#define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS 5
#define DEFAULT_MIGRATE_ANNOUNCE_STEP 100
+#define DEFAULT_MIGRATE_VCPU_DIRTYLIMIT_PERIOD 500 /* ms */
+
static NotifierList migration_state_notifiers =
NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
@@ -921,6 +923,9 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
s->parameters.block_bitmap_mapping);
}
+ params->has_vcpu_dirtylimit_period = true;
+ params->vcpu_dirtylimit_period = s->parameters.vcpu_dirtylimit_period;
+
return params;
}
@@ -1581,6 +1586,10 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
dest->has_block_bitmap_mapping = true;
dest->block_bitmap_mapping = params->block_bitmap_mapping;
}
+
+ if (params->has_vcpu_dirtylimit_period) {
+ dest->vcpu_dirtylimit_period = params->vcpu_dirtylimit_period;
+ }
}
static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
@@ -1703,6 +1712,9 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
QAPI_CLONE(BitmapMigrationNodeAliasList,
params->block_bitmap_mapping);
}
+ if (params->has_vcpu_dirtylimit_period) {
+ s->parameters.vcpu_dirtylimit_period = params->vcpu_dirtylimit_period;
+ }
}
void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
@@ -2403,6 +2415,11 @@ bool migrate_auto_converge(void)
return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
}
+bool migrate_dirtylimit(void)
+{
+ return false;
+}
+
bool migrate_zero_blocks(void)
{
MigrationState *s;
@@ -4221,6 +4238,9 @@ static Property migration_properties[] = {
DEFINE_PROP_SIZE("announce-step", MigrationState,
parameters.announce_step,
DEFAULT_MIGRATE_ANNOUNCE_STEP),
+ DEFINE_PROP_UINT64("vcpu-dirtylimit-period", MigrationState,
+ parameters.vcpu_dirtylimit_period,
+ DEFAULT_MIGRATE_VCPU_DIRTYLIMIT_PERIOD),
/* Migration capabilities */
DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
diff --git a/migration/migration.h b/migration/migration.h
index 2de861d..43541cf 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -369,6 +369,7 @@ bool migrate_ignore_shared(void);
bool migrate_validate_uuid(void);
bool migrate_auto_converge(void);
+bool migrate_dirtylimit(void);
bool migrate_use_multifd(void);
bool migrate_pause_before_switchover(void);
int migrate_multifd_channels(void);
diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
index 6349684..fa33a35 100644
--- a/monitor/hmp-cmds.c
+++ b/monitor/hmp-cmds.c
@@ -525,6 +525,10 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
}
}
}
+
+ monitor_printf(mon, "%s: %" PRIu64 " MB/s\n",
+ MigrationParameter_str(MIGRATION_PARAMETER_VCPU_DIRTYLIMIT_PERIOD),
+ params->vcpu_dirtylimit_period);
}
qapi_free_MigrationParameters(params);
@@ -1344,6 +1348,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
error_setg(&err, "The block-bitmap-mapping parameter can only be set "
"through QMP");
break;
+ case MIGRATION_PARAMETER_VCPU_DIRTYLIMIT_PERIOD:
+ p->has_vcpu_dirtylimit_period = true;
+ visit_type_size(v, param, &p->vcpu_dirtylimit_period, &err);
+ break;
default:
assert(0);
}
diff --git a/qapi/migration.json b/qapi/migration.json
index 6a21027..260fb66 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -760,6 +760,9 @@
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
+# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
+# Defaults to 500ms. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -779,8 +782,8 @@
'multifd-channels',
'xbzrle-cache-size', 'max-postcopy-bandwidth',
'max-cpu-throttle', 'multifd-compression',
- 'multifd-zlib-level' ,'multifd-zstd-level',
- 'block-bitmap-mapping' ] }
+ 'multifd-zlib-level', 'multifd-zstd-level',
+ 'block-bitmap-mapping', 'vcpu-dirtylimit-period'] }
##
# @MigrateSetParameters:
@@ -925,6 +928,9 @@
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
+# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
+# Defaults to 500ms. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -960,7 +966,8 @@
'*multifd-compression': 'MultiFDCompression',
'*multifd-zlib-level': 'uint8',
'*multifd-zstd-level': 'uint8',
- '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } }
+ '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
+ '*vcpu-dirtylimit-period': 'uint64'} }
##
# @migrate-set-parameters:
@@ -1125,6 +1132,9 @@
# block device name if there is one, and to their node name
# otherwise. (Since 5.2)
#
+# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
+# Defaults to 500ms. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -1158,7 +1168,8 @@
'*multifd-compression': 'MultiFDCompression',
'*multifd-zlib-level': 'uint8',
'*multifd-zstd-level': 'uint8',
- '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ] } }
+ '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
+ '*vcpu-dirtylimit-period': 'uint64'} }
##
# @query-migrate-parameters:
diff --git a/softmmu/dirtylimit.c b/softmmu/dirtylimit.c
index 365bd43..affe993 100644
--- a/softmmu/dirtylimit.c
+++ b/softmmu/dirtylimit.c
@@ -23,6 +23,8 @@
#include "exec/memory.h"
#include "hw/boards.h"
#include "sysemu/kvm.h"
+#include "migration/misc.h"
+#include "migration/migration.h"
#include "trace.h"
/*
@@ -75,11 +77,18 @@ static bool dirtylimit_quit;
static void vcpu_dirty_rate_stat_collect(void)
{
+ MigrationState *s = migrate_get_current();
VcpuStat stat;
int i = 0;
+ int64_t period = DIRTYLIMIT_CALC_TIME_MS;
+
+ if (migrate_dirtylimit() &&
+ migration_is_active(s)) {
+ period = s->parameters.vcpu_dirtylimit_period;
+ }
/* calculate vcpu dirtyrate */
- vcpu_calculate_dirtyrate(DIRTYLIMIT_CALC_TIME_MS,
+ vcpu_calculate_dirtyrate(period,
&stat,
GLOBAL_DIRTY_LIMIT,
false);
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters
2022-05-17 6:35 ` [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters huangy81
@ 2022-05-18 15:05 ` Eric Blake
2022-05-19 3:05 ` Hyman Huang
0 siblings, 1 reply; 11+ messages in thread
From: Eric Blake @ 2022-05-18 15:05 UTC (permalink / raw)
To: huangy81
Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini
On Tue, May 17, 2022 at 02:35:01PM +0800, huangy81@chinatelecom.cn wrote:
> From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>
> Introduce "vcpu-dirtylimit-period" migration parameters,
> which is used to makes dirtyrate calculation period
make
> configurable.
>
> To implement that, refactor vcpu_dirty_rate_stat_collect
> so that period can be configured instead of hardcode.
hardcoded
>
> Meanwhile, introduce migrate_dirtylimit function to help
> check if dirtylimit enabled during live migration, set
> it false by default.
>
> Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
> ---
Focusing just on UI...
> +++ b/qapi/migration.json
> @@ -760,6 +760,9 @@
> # block device name if there is one, and to their node name
> # otherwise. (Since 5.2)
> #
> +# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
> +# Defaults to 500ms. (Since 7.0)
The next release is 7.1. You'll need to fix this and all other references.
Do we want 'dirty-limit' instead of 'dirtylimit'? There was a recent
thread on how to translate QAPI to other languages that are a bit more
insistent on MixedCase, where properly separating English words makes
it easier to translate to the expected case.
> ##
> # @migrate-set-parameters:
> @@ -1125,6 +1132,9 @@
> # block device name if there is one, and to their node name
> # otherwise. (Since 5.2)
> #
> +# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
> +# Defaults to 500ms. (Since 7.0)
> +#
> # Features:
> # @unstable: Member @x-checkpoint-delay is experimental.
Is this feature ready for prime time, or do we want to initially name
it x-vcpu-dirty[-]limit-period to mark it experimental?
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3266
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters
2022-05-18 15:05 ` Eric Blake
@ 2022-05-19 3:05 ` Hyman Huang
0 siblings, 0 replies; 11+ messages in thread
From: Hyman Huang @ 2022-05-19 3:05 UTC (permalink / raw)
To: Eric Blake
Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini
在 2022/5/18 23:05, Eric Blake 写道:
> On Tue, May 17, 2022 at 02:35:01PM +0800, huangy81@chinatelecom.cn wrote:
>> From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>>
>> Introduce "vcpu-dirtylimit-period" migration parameters,
>> which is used to makes dirtyrate calculation period
>
> make
>
>> configurable.
>>
>> To implement that, refactor vcpu_dirty_rate_stat_collect
>> so that period can be configured instead of hardcode.
>
> hardcoded
>
>>
>> Meanwhile, introduce migrate_dirtylimit function to help
>> check if dirtylimit enabled during live migration, set
>> it false by default.
>>
>> Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>> ---
>
> Focusing just on UI...
>
>> +++ b/qapi/migration.json
>> @@ -760,6 +760,9 @@
>> # block device name if there is one, and to their node name
>> # otherwise. (Since 5.2)
>> #
>> +# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
>> +# Defaults to 500ms. (Since 7.0)
>
> The next release is 7.1. You'll need to fix this and all other references.
Ok, i'll fix that in the v1.
>
> Do we want 'dirty-limit' instead of 'dirtylimit'? There was a recent
> thread on how to translate QAPI to other languages that are a bit more
> insistent on MixedCase, where properly separating English words makes
> it easier to translate to the expected case.
>
Changing the parameter name sounds ok to me, i'm not insistent that。
>> ##
>> # @migrate-set-parameters:
>> @@ -1125,6 +1132,9 @@
>> # block device name if there is one, and to their node name
>> # otherwise. (Since 5.2)
>> #
>> +# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
>> +# Defaults to 500ms. (Since 7.0)
>> +#
>> # Features:
>> # @unstable: Member @x-checkpoint-delay is experimental.
>
> Is this feature ready for prime time, or do we want to initially name
> it x-vcpu-dirty[-]limit-period to mark it experimental?
Indeed, for this fresh new feature, finding factors affecting migration
need more practice, marking it experimental could be much safer, i'll do
that in v1.
>
--
Best regard
Hyman Huang(黄勇)
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC 2/6] qapi/migration: Introduce vcpu-dirtylimit parameters
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
2022-05-17 6:35 ` [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters huangy81
@ 2022-05-17 6:35 ` huangy81
2022-05-18 15:06 ` Eric Blake
2022-05-17 6:35 ` [RFC 3/6] migration: Implement dirtylimit convergence algo huangy81
` (3 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Introduce "vcpu-dirtylimit" migration parameter used
to limit dirty page rate during live migration.
"vcpu-dirtylimit" and "vcpu-dirtylimit-period" are
two dirtylimit-related migration parameters, which can
be set before and during live migration by qmp
migrate-set-parameters.
This two parameters are used to help implement the dirty
page rate limit algo of migration.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/migration.c | 13 +++++++++++++
monitor/hmp-cmds.c | 7 +++++++
qapi/migration.json | 18 +++++++++++++++---
3 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/migration/migration.c b/migration/migration.c
index 5e20b2a..9e4ce01 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -117,6 +117,7 @@
#define DEFAULT_MIGRATE_ANNOUNCE_STEP 100
#define DEFAULT_MIGRATE_VCPU_DIRTYLIMIT_PERIOD 500 /* ms */
+#define DEFAULT_MIGRATE_VCPU_DIRTYLIMIT 1 /* MB/s */
static NotifierList migration_state_notifiers =
NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
@@ -925,6 +926,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
params->has_vcpu_dirtylimit_period = true;
params->vcpu_dirtylimit_period = s->parameters.vcpu_dirtylimit_period;
+ params->has_vcpu_dirtylimit = true;
+ params->vcpu_dirtylimit = s->parameters.vcpu_dirtylimit;
return params;
}
@@ -1590,6 +1593,10 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
if (params->has_vcpu_dirtylimit_period) {
dest->vcpu_dirtylimit_period = params->vcpu_dirtylimit_period;
}
+
+ if (params->has_vcpu_dirtylimit) {
+ dest->vcpu_dirtylimit = params->vcpu_dirtylimit;
+ }
}
static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
@@ -1715,6 +1722,9 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
if (params->has_vcpu_dirtylimit_period) {
s->parameters.vcpu_dirtylimit_period = params->vcpu_dirtylimit_period;
}
+ if (params->has_vcpu_dirtylimit) {
+ s->parameters.vcpu_dirtylimit = params->vcpu_dirtylimit;
+ }
}
void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
@@ -4241,6 +4251,9 @@ static Property migration_properties[] = {
DEFINE_PROP_UINT64("vcpu-dirtylimit-period", MigrationState,
parameters.vcpu_dirtylimit_period,
DEFAULT_MIGRATE_VCPU_DIRTYLIMIT_PERIOD),
+ DEFINE_PROP_UINT64("vcpu-dirtylimit", MigrationState,
+ parameters.vcpu_dirtylimit,
+ DEFAULT_MIGRATE_VCPU_DIRTYLIMIT),
/* Migration capabilities */
DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
index fa33a35..31a0564 100644
--- a/monitor/hmp-cmds.c
+++ b/monitor/hmp-cmds.c
@@ -529,6 +529,9 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
monitor_printf(mon, "%s: %" PRIu64 " MB/s\n",
MigrationParameter_str(MIGRATION_PARAMETER_VCPU_DIRTYLIMIT_PERIOD),
params->vcpu_dirtylimit_period);
+ monitor_printf(mon, "%s: %" PRIu64 " MB/s\n",
+ MigrationParameter_str(MIGRATION_PARAMETER_VCPU_DIRTYLIMIT),
+ params->vcpu_dirtylimit);
}
qapi_free_MigrationParameters(params);
@@ -1352,6 +1355,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
p->has_vcpu_dirtylimit_period = true;
visit_type_size(v, param, &p->vcpu_dirtylimit_period, &err);
break;
+ case MIGRATION_PARAMETER_VCPU_DIRTYLIMIT:
+ p->has_vcpu_dirtylimit = true;
+ visit_type_size(v, param, &p->vcpu_dirtylimit, &err);
+ break;
default:
assert(0);
}
diff --git a/qapi/migration.json b/qapi/migration.json
index 260fb66..68c1fe0 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -763,6 +763,9 @@
# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
# Defaults to 500ms. (Since 7.0)
#
+# @vcpu-dirtylimit: Dirtyrate limit (MB/s) during live migration.
+# Defaults to 1. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -783,7 +786,8 @@
'xbzrle-cache-size', 'max-postcopy-bandwidth',
'max-cpu-throttle', 'multifd-compression',
'multifd-zlib-level', 'multifd-zstd-level',
- 'block-bitmap-mapping', 'vcpu-dirtylimit-period'] }
+ 'block-bitmap-mapping', 'vcpu-dirtylimit-period',
+ 'vcpu-dirtylimit'] }
##
# @MigrateSetParameters:
@@ -931,6 +935,9 @@
# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
# Defaults to 500ms. (Since 7.0)
#
+# @vcpu-dirtylimit: Dirtyrate limit (MB/s) during live migration.
+# Defaults to 1. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -967,7 +974,8 @@
'*multifd-zlib-level': 'uint8',
'*multifd-zstd-level': 'uint8',
'*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
- '*vcpu-dirtylimit-period': 'uint64'} }
+ '*vcpu-dirtylimit-period': 'uint64',
+ '*vcpu-dirtylimit': 'uint64'} }
##
# @migrate-set-parameters:
@@ -1135,6 +1143,9 @@
# @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
# Defaults to 500ms. (Since 7.0)
#
+# @vcpu-dirtylimit: Dirtyrate limit (MB/s) during live migration.
+# Defaults to 1. (Since 7.0)
+#
# Features:
# @unstable: Member @x-checkpoint-delay is experimental.
#
@@ -1169,7 +1180,8 @@
'*multifd-zlib-level': 'uint8',
'*multifd-zstd-level': 'uint8',
'*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ],
- '*vcpu-dirtylimit-period': 'uint64'} }
+ '*vcpu-dirtylimit-period': 'uint64',
+ '*vcpu-dirtylimit': 'uint64'} }
##
# @query-migrate-parameters:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [RFC 2/6] qapi/migration: Introduce vcpu-dirtylimit parameters
2022-05-17 6:35 ` [RFC 2/6] qapi/migration: Introduce vcpu-dirtylimit parameters huangy81
@ 2022-05-18 15:06 ` Eric Blake
0 siblings, 0 replies; 11+ messages in thread
From: Eric Blake @ 2022-05-18 15:06 UTC (permalink / raw)
To: huangy81
Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini
On Tue, May 17, 2022 at 02:35:02PM +0800, huangy81@chinatelecom.cn wrote:
> From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>
> Introduce "vcpu-dirtylimit" migration parameter used
> to limit dirty page rate during live migration.
>
> "vcpu-dirtylimit" and "vcpu-dirtylimit-period" are
> two dirtylimit-related migration parameters, which can
> be set before and during live migration by qmp
> migrate-set-parameters.
>
> This two parameters are used to help implement the dirty
> page rate limit algo of migration.
>
> Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
> ---
> +++ b/qapi/migration.json
> @@ -763,6 +763,9 @@
> # @vcpu-dirtylimit-period: Periodic time (ms) of dirtylimit during live migration.
> # Defaults to 500ms. (Since 7.0)
> #
> +# @vcpu-dirtylimit: Dirtyrate limit (MB/s) during live migration.
> +# Defaults to 1. (Since 7.0)
> +#
Same comments as in patch 1 (since 7.1, question on whether
dirty-limit makes more sense).
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3266
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC 3/6] migration: Implement dirtylimit convergence algo
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
2022-05-17 6:35 ` [RFC 1/6] qapi/migration: Introduce vcpu-dirtylimit-period parameters huangy81
2022-05-17 6:35 ` [RFC 2/6] qapi/migration: Introduce vcpu-dirtylimit parameters huangy81
@ 2022-05-17 6:35 ` huangy81
2022-05-17 6:35 ` [RFC 4/6] migration: Introduce dirtylimit capability huangy81
` (2 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Implement dirtylimit convergence algo for live migration,
which is kind of like auto-converge algo but using dirtylimit
instead of cpu throttle to make migration convergent.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/ram.c | 53 +++++++++++++++++++++++++++++++++++++-------------
migration/trace-events | 1 +
2 files changed, 41 insertions(+), 13 deletions(-)
diff --git a/migration/ram.c b/migration/ram.c
index 3532f64..5dd3e69 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -44,6 +44,7 @@
#include "qapi/error.h"
#include "qapi/qapi-types-migration.h"
#include "qapi/qapi-events-migration.h"
+#include "qapi/qapi-commands-migration.h"
#include "qapi/qmp/qerror.h"
#include "trace.h"
#include "exec/ram_addr.h"
@@ -56,6 +57,8 @@
#include "qemu/iov.h"
#include "multifd.h"
#include "sysemu/runstate.h"
+#include "sysemu/dirtylimit.h"
+#include "sysemu/kvm.h"
#include "hw/boards.h" /* for machine_dump_guest_core() */
@@ -1082,6 +1085,21 @@ static void migration_update_rates(RAMState *rs, int64_t end_time)
}
}
+/*
+ * Enable dirtylimit to throttle down the guest
+ */
+static void migration_dirtylimit_guest(void)
+{
+ if (!dirtylimit_in_service()) {
+ MigrationState *s = migrate_get_current();
+ int64_t quota_dirtyrate = s->parameters.vcpu_dirtylimit;
+
+ /* Set quota dirtyrate if dirty limit not in service */
+ qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
+ trace_migration_dirtylimit_guest(quota_dirtyrate);
+ }
+}
+
static void migration_trigger_throttle(RAMState *rs)
{
MigrationState *s = migrate_get_current();
@@ -1091,22 +1109,31 @@ static void migration_trigger_throttle(RAMState *rs)
uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
- /* During block migration the auto-converge logic incorrectly detects
- * that ram migration makes no progress. Avoid this by disabling the
- * throttling logic during the bulk phase of block migration. */
- if (migrate_auto_converge() && !blk_mig_bulk_active()) {
- /* The following detection logic can be refined later. For now:
- Check to see if the ratio between dirtied bytes and the approx.
- amount of bytes that just got transferred since the last time
- we were in this routine reaches the threshold. If that happens
- twice, start or increase throttling. */
-
- if ((bytes_dirty_period > bytes_dirty_threshold) &&
- (++rs->dirty_rate_high_cnt >= 2)) {
+ /*
+ * The following detection logic can be refined later. For now:
+ * Check to see if the ratio between dirtied bytes and the approx.
+ * amount of bytes that just got transferred since the last time
+ * we were in this routine reaches the threshold. If that happens
+ * twice, start or increase throttling.
+ */
+
+ if ((bytes_dirty_period > bytes_dirty_threshold) &&
+ (++rs->dirty_rate_high_cnt >= 2)) {
+ rs->dirty_rate_high_cnt = 0;
+ /*
+ * During block migration the auto-converge logic incorrectly detects
+ * that ram migration makes no progress. Avoid this by disabling the
+ * throttling logic during the bulk phase of block migration.
+ */
+
+ if (migrate_auto_converge() && !blk_mig_bulk_active()) {
trace_migration_throttle();
- rs->dirty_rate_high_cnt = 0;
mig_throttle_guest_down(bytes_dirty_period,
bytes_dirty_threshold);
+ } else if (migrate_dirtylimit() &&
+ kvm_dirty_ring_enabled() &&
+ migration_is_active(s)) {
+ migration_dirtylimit_guest();
}
}
}
diff --git a/migration/trace-events b/migration/trace-events
index 1aec580..2c341fc 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -89,6 +89,7 @@ migration_bitmap_sync_start(void) ""
migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64
migration_bitmap_clear_dirty(char *str, uint64_t start, uint64_t size, unsigned long page) "rb %s start 0x%"PRIx64" size 0x%"PRIx64" page 0x%lx"
migration_throttle(void) ""
+migration_dirtylimit_guest(int64_t dirtyrate) "guest dirty page rate limit %" PRIi64 " MB/s"
ram_discard_range(const char *rbname, uint64_t start, size_t len) "%s: start: %" PRIx64 " %zx"
ram_load_loop(const char *rbname, uint64_t addr, int flags, void *host) "%s: addr: 0x%" PRIx64 " flags: 0x%x host: %p"
ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [RFC 4/6] migration: Introduce dirtylimit capability
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
` (2 preceding siblings ...)
2022-05-17 6:35 ` [RFC 3/6] migration: Implement dirtylimit convergence algo huangy81
@ 2022-05-17 6:35 ` huangy81
2022-05-18 15:20 ` Eric Blake
2022-05-17 6:35 ` [RFC 5/6] migration: Add dirtylimit data into migration info huangy81
2022-05-17 6:35 ` [RFC 6/6] tests: Add migration dirtylimit capability test huangy81
5 siblings, 1 reply; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Introduce migration dirtylimit capability, which can
be turned on before live migration and limit dirty
page rate durty live migration.
Dirtylimit dirtylimit capability is kind of like
auto-converge but using dirtylimit instead of traditional
cpu-throttle to throttle guest down.
To enable this feature, turn on the dirtylimit capability
before live migration using migratioin-set-capabilities,
and set dirtylimit-related parameters "vcpu-dirtylimit",
"vcpu-dirtylimit-period" suitably to speed up convergence.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
migration/migration.c | 7 ++++++-
qapi/migration.json | 5 ++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/migration/migration.c b/migration/migration.c
index 9e4ce01..4a659b6 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -2427,7 +2427,11 @@ bool migrate_auto_converge(void)
bool migrate_dirtylimit(void)
{
- return false;
+ MigrationState *s;
+
+ s = migrate_get_current();
+
+ return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTYLIMIT];
}
bool migrate_zero_blocks(void)
@@ -4270,6 +4274,7 @@ static Property migration_properties[] = {
DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
DEFINE_PROP_MIG_CAP("x-background-snapshot",
MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
+ DEFINE_PROP_MIG_CAP("x-dirtylimit", MIGRATION_CAPABILITY_DIRTYLIMIT),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/qapi/migration.json b/qapi/migration.json
index 68c1fe0..30ad413 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -463,6 +463,9 @@
# procedure starts. The VM RAM is saved with running VM.
# (since 6.0)
#
+# @dirtylimit: Use dirtylimit to throttle down guest if enabled.
+# (since 7.0)
+#
# Features:
# @unstable: Members @x-colo and @x-ignore-shared are experimental.
#
@@ -476,7 +479,7 @@
'block', 'return-path', 'pause-before-switchover', 'multifd',
'dirty-bitmaps', 'postcopy-blocktime', 'late-block-activate',
{ 'name': 'x-ignore-shared', 'features': [ 'unstable' ] },
- 'validate-uuid', 'background-snapshot'] }
+ 'validate-uuid', 'background-snapshot', 'dirtylimit'] }
##
# @MigrationCapabilityStatus:
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [RFC 4/6] migration: Introduce dirtylimit capability
2022-05-17 6:35 ` [RFC 4/6] migration: Introduce dirtylimit capability huangy81
@ 2022-05-18 15:20 ` Eric Blake
0 siblings, 0 replies; 11+ messages in thread
From: Eric Blake @ 2022-05-18 15:20 UTC (permalink / raw)
To: huangy81
Cc: qemu-devel, Dr. David Alan Gilbert, Juan Quintela,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini
On Tue, May 17, 2022 at 02:35:04PM +0800, huangy81@chinatelecom.cn wrote:
> From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
>
> Introduce migration dirtylimit capability, which can
> be turned on before live migration and limit dirty
> page rate durty live migration.
s/durty/during/ ?
>
> Dirtylimit dirtylimit capability is kind of like
Doubled word
> auto-converge but using dirtylimit instead of traditional
> cpu-throttle to throttle guest down.
>
> To enable this feature, turn on the dirtylimit capability
> before live migration using migratioin-set-capabilities,
> and set dirtylimit-related parameters "vcpu-dirtylimit",
> "vcpu-dirtylimit-period" suitably to speed up convergence.
>
> Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
> ---
> +++ b/qapi/migration.json
> @@ -463,6 +463,9 @@
> # procedure starts. The VM RAM is saved with running VM.
> # (since 6.0)
> #
> +# @dirtylimit: Use dirtylimit to throttle down guest if enabled.
> +# (since 7.0)
7.1
same question about naming it 'dirty-limit'
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3266
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 11+ messages in thread
* [RFC 5/6] migration: Add dirtylimit data into migration info
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
` (3 preceding siblings ...)
2022-05-17 6:35 ` [RFC 4/6] migration: Introduce dirtylimit capability huangy81
@ 2022-05-17 6:35 ` huangy81
2022-05-17 6:35 ` [RFC 6/6] tests: Add migration dirtylimit capability test huangy81
5 siblings, 0 replies; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Add dirtylimit throttle data into migration info, through
which we can observe the process of dirtylimit during
live migration.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
include/sysemu/dirtylimit.h | 2 ++
migration/migration.c | 10 ++++++++++
qapi/migration.json | 11 ++++++++++-
softmmu/dirtylimit.c | 22 ++++++++++++++++++++++
4 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/include/sysemu/dirtylimit.h b/include/sysemu/dirtylimit.h
index 8d2c1f3..0b8dd76 100644
--- a/include/sysemu/dirtylimit.h
+++ b/include/sysemu/dirtylimit.h
@@ -34,4 +34,6 @@ void dirtylimit_set_vcpu(int cpu_index,
void dirtylimit_set_all(uint64_t quota,
bool enable);
void dirtylimit_vcpu_execute(CPUState *cpu);
+int64_t dirtylimit_throttle_us_per_full(void);
+int64_t dirtylimit_us_per_full(void);
#endif
diff --git a/migration/migration.c b/migration/migration.c
index 4a659b6..935179e 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -61,6 +61,7 @@
#include "sysemu/cpus.h"
#include "yank_functions.h"
#include "sysemu/qtest.h"
+#include "sysemu/dirtylimit.h"
#define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */
@@ -1065,6 +1066,15 @@ static void populate_ram_info(MigrationInfo *info, MigrationState *s)
info->ram->remaining = ram_bytes_remaining();
info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
}
+
+ if (migrate_dirtylimit() && dirtylimit_in_service()) {
+ info->has_dirtylimit_throttle_us_per_full = true;
+ info->dirtylimit_throttle_us_per_full =
+ dirtylimit_throttle_us_per_full();
+
+ info->has_dirtylimit_us_per_full = true;
+ info->dirtylimit_us_per_full = dirtylimit_us_per_full();
+ }
}
static void populate_disk_info(MigrationInfo *info)
diff --git a/qapi/migration.json b/qapi/migration.json
index 30ad413..cac4c8d 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -239,6 +239,13 @@
# Present and non-empty when migration is blocked.
# (since 6.0)
#
+# @dirtylimit-throttle-us-per-full: Throttle time (us) during the period of
+# dirty ring full.
+# (since 7.0)
+#
+# @dirtylimit-us-per-full: Estimated the periodic time (us) of dirty ring full.
+# (since 7.0)
+#
# Since: 0.14
##
{ 'struct': 'MigrationInfo',
@@ -256,7 +263,9 @@
'*postcopy-blocktime' : 'uint32',
'*postcopy-vcpu-blocktime': ['uint32'],
'*compression': 'CompressionStats',
- '*socket-address': ['SocketAddress'] } }
+ '*socket-address': ['SocketAddress'],
+ '*dirtylimit-throttle-us-per-full': 'int64',
+ '*dirtylimit-us-per-full': 'int64'} }
##
# @query-migrate:
diff --git a/softmmu/dirtylimit.c b/softmmu/dirtylimit.c
index affe993..33440c0 100644
--- a/softmmu/dirtylimit.c
+++ b/softmmu/dirtylimit.c
@@ -546,6 +546,28 @@ static struct DirtyLimitInfo *dirtylimit_query_vcpu(int cpu_index)
return info;
}
+/* Pick up first vcpu throttle time by default */
+int64_t dirtylimit_throttle_us_per_full(void)
+{
+ CPUState *cpu = first_cpu;
+ return cpu->throttle_us_per_full;
+}
+
+/*
+ * Estimate dirty ring full time under current dirty page rate.
+ * Return -1 if guest doesn't dirty memory.
+ */
+int64_t dirtylimit_us_per_full(void)
+{
+ uint64_t curr_rate = vcpu_dirty_rate_get(0);
+
+ if (!curr_rate) {
+ return -1;
+ }
+
+ return dirtylimit_dirty_ring_full_time(curr_rate);
+}
+
static struct DirtyLimitInfoList *dirtylimit_query_all(void)
{
int i, index;
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [RFC 6/6] tests: Add migration dirtylimit capability test
2022-05-17 6:35 [RFC 0/6] migration: introduce dirtylimit capability huangy81
` (4 preceding siblings ...)
2022-05-17 6:35 ` [RFC 5/6] migration: Add dirtylimit data into migration info huangy81
@ 2022-05-17 6:35 ` huangy81
5 siblings, 0 replies; 11+ messages in thread
From: huangy81 @ 2022-05-17 6:35 UTC (permalink / raw)
To: qemu-devel
Cc: Dr. David Alan Gilbert, Juan Quintela, Eric Blake,
Markus Armbruster, Thomas Huth, Laurent Vivier, Paolo Bonzini,
Hyman Huang
From: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
Add migration dirtylimit capability test if kernel support
dirty ring.
Migration dirtylimit capability introduce dirtylimit capability
, two parameters: vcpu-dirtylimit-period and vcpu-dirtylimit to
implement the live migration with dirtylimit.
The test case enable the capability and set the corresponding
parameters to test migration. When migration switch to
pre-switchover phase, like the auto-converge, checking if
migration satisfy the convergence condition.
Signed-off-by: Hyman Huang(黄勇) <huangy81@chinatelecom.cn>
---
tests/qtest/migration-test.c | 89 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 89 insertions(+)
diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c
index 40f2873..b0a53d7 100644
--- a/tests/qtest/migration-test.c
+++ b/tests/qtest/migration-test.c
@@ -1650,6 +1650,93 @@ static void test_vcpu_dirty_limit(void)
dirtylimit_stop_vm(vm);
}
+static void test_migrate_dirty_limit(void)
+{
+ g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
+ MigrateStart *args = migrate_start_new();
+ QTestState *from, *to;
+ int64_t remaining, throttle_us_per_full;
+ /*
+ * We want the test to be stable and as fast as possible.
+ * E.g., with 1Gb/s bandwith migration may pass without dirtylimit,
+ * so we need to decrease a bandwidth.
+ */
+ const int64_t dirtylimit_period = 1000, dirtylimit_value = 50;
+ const int64_t max_bandwidth = 400000000; /* ~400Mb/s */
+ const int64_t downtime_limit = 250; /* 250ms */
+ /*
+ * We migrate through unix-socket (> 500Mb/s).
+ * Thus, expected migration speed ~= bandwidth limit (< 500Mb/s).
+ * So, we can predict expected_threshold
+ */
+ const int64_t expected_threshold = max_bandwidth * downtime_limit / 1000;
+
+ /* Enable dirty ring logging */
+ args->use_dirty_ring = true;
+
+ if (test_migrate_start(&from, &to, uri, &args)) {
+ return;
+ }
+
+ migrate_set_capability(from, "dirtylimit", true);
+ migrate_set_parameter_int(from, "vcpu-dirtylimit-period",
+ dirtylimit_period);
+ migrate_set_parameter_int(from, "vcpu-dirtylimit", dirtylimit_value);
+
+ /*
+ * Set the initial parameters so that the migration could not converge
+ * without dirtylimit.
+ */
+ migrate_set_parameter_int(from, "downtime-limit", 1);
+ migrate_set_parameter_int(from, "max-bandwidth", 100000000); /* ~100Mb/s */
+
+ /* To check limit rate after precopy */
+ migrate_set_capability(from, "pause-before-switchover", true);
+
+ /* Wait for the first serial output from the source */
+ wait_for_serial("src_serial");
+
+ migrate_qmp(from, uri, "{}");
+
+ /* Wait for dirtylimit begins */
+ throttle_us_per_full = 0;
+ while (throttle_us_per_full == 0) {
+ throttle_us_per_full =
+ read_migrate_property_int(from, "dirtylimit-throttle-us-per-full");
+ usleep(100);
+ g_assert_false(got_stop);
+ }
+
+ /*
+ * The dirtylimit rate should equals the return value of
+ * query-vcpu-dirty-limit if dirtylimit cap set
+ */
+ g_assert_cmpint(dirtylimit_value, ==, get_limit_rate(from));
+
+ /* Now, when we tested if dirtylimit works, let it converge */
+ migrate_set_parameter_int(from, "downtime-limit", downtime_limit);
+ migrate_set_parameter_int(from, "max-bandwidth", max_bandwidth);
+
+ /*
+ * Wait for pre-switchover status to check if migration
+ * satisfy the convergence condition
+ */
+ wait_for_migration_status(from, "pre-switchover", NULL);
+
+ remaining = read_ram_property_int(from, "remaining");
+ g_assert_cmpint(remaining, <,
+ (expected_threshold + expected_threshold / 100));
+
+ migrate_continue(from, "pre-switchover");
+
+ qtest_qmp_eventwait(to, "RESUME");
+
+ wait_for_serial("dest_serial");
+ wait_for_migration_complete(from);
+
+ test_migrate_end(from, to, true);
+}
+
static bool kvm_dirty_ring_supported(void)
{
#if defined(__linux__) && defined(HOST_X86_64)
@@ -1741,6 +1828,8 @@ int main(int argc, char **argv)
test_precopy_unix_dirty_ring);
qtest_add_func("/migration/vcpu_dirty_limit",
test_vcpu_dirty_limit);
+ qtest_add_func("/migration/dirty_limit",
+ test_migrate_dirty_limit);
}
ret = g_test_run();
--
1.8.3.1
^ permalink raw reply related [flat|nested] 11+ messages in thread