From: Marcelo Tosatti <mtosatti@redhat.com>
To: kvm@vger.kernel.org
Cc: qemu-devel@nongnu.org,
Dr David Alan Gilbert <dgilbert@redhat.com>,
Paolo Bonzini <pbonzini@redhat.com>,
Juan Quintela <quintela@redhat.com>,
Radim Krcmar <rkrcmar@redhat.com>,
Eduardo Habkost <ehabkost@redhat.com>,
Marcelo Tosatti <mtosatti@redhat.com>
Subject: [qemu patch V3 2/2] kvmclock: reduce kvmclock difference on migration
Date: Mon, 21 Nov 2016 08:50:04 -0200 [thread overview]
Message-ID: <20161121105052.598267440@redhat.com> (raw)
In-Reply-To: 20161121105002.291554230@redhat.com
[-- Attachment #1: kvmclock-advance-clock.patch --]
[-- Type: text/plain, Size: 7866 bytes --]
Check for KVM_CAP_ADJUST_CLOCK capability KVM_CLOCK_TSC_STABLE, which
indicates that KVM_GET_CLOCK returns a value as seen by the guest at
that moment.
For new machine types, use this value rather than reading
from guest memory.
This reduces kvmclock difference on migration from 5s to 0.1s
(when max_downtime == 5s).
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
---
hw/i386/kvm/clock.c | 107 ++++++++++++++++++++++++++++++++++++++++++-------
include/hw/i386/pc.h | 5 ++
target-i386/kvm.c | 7 +++
target-i386/kvm_i386.h | 1
4 files changed, 106 insertions(+), 14 deletions(-)
v2:
- improve variable names (Juan)
- consolidate code on kvm_get_clock function (Paolo)
- return mach_use_reliable_get_clock from needed function (Paolo)
v3:
- simplify check for src_use_reliable_get_clock (Eduardo)
- move src_use_reliable_get_clock initialization to realize (Eduardo)
Index: qemu-mig-advance-clock/hw/i386/kvm/clock.c
===================================================================
--- qemu-mig-advance-clock.orig/hw/i386/kvm/clock.c 2016-11-17 15:07:11.220632761 -0200
+++ qemu-mig-advance-clock/hw/i386/kvm/clock.c 2016-11-17 15:11:51.372048640 -0200
@@ -36,6 +36,12 @@
uint64_t clock;
bool clock_valid;
+
+ /* whether machine supports reliable KVM_GET_CLOCK */
+ bool mach_use_reliable_get_clock;
+
+ /* whether source host supported reliable KVM_GET_CLOCK */
+ bool src_use_reliable_get_clock;
} KVMClockState;
struct pvclock_vcpu_time_info {
@@ -81,6 +87,19 @@
return nsec + time.system_time;
}
+static uint64_t kvm_get_clock(void)
+{
+ struct kvm_clock_data data;
+ int ret;
+
+ ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
+ if (ret < 0) {
+ fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
+ abort();
+ }
+ return data.clock;
+}
+
static void kvmclock_vm_state_change(void *opaque, int running,
RunState state)
{
@@ -91,15 +110,36 @@
if (running) {
struct kvm_clock_data data = {};
- uint64_t time_at_migration = kvmclock_current_nsec(s);
+ uint64_t pvclock_via_mem = 0;
- s->clock_valid = false;
+ /* local (running VM) restore */
+ if (s->clock_valid) {
+ /*
+ * if host does not support reliable KVM_GET_CLOCK,
+ * read kvmclock value from memory
+ */
+ if (!kvm_has_adjust_clock_stable()) {
+ pvclock_via_mem = kvmclock_current_nsec(s);
+ }
+ /* migration/savevm/init restore */
+ } else {
+ /*
+ * use s->clock in case machine uses reliable
+ * get clock and source host supported
+ * reliable get clock
+ */
+ if (!s->src_use_reliable_get_clock) {
+ pvclock_via_mem = kvmclock_current_nsec(s);
+ }
+ }
- /* We can't rely on the migrated clock value, just discard it */
- if (time_at_migration) {
- s->clock = time_at_migration;
+ /* We can't rely on the saved clock value, just discard it */
+ if (pvclock_via_mem) {
+ s->clock = pvclock_via_mem;
}
+ s->clock_valid = false;
+
data.clock = s->clock;
ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
if (ret < 0) {
@@ -120,8 +160,6 @@
}
}
} else {
- struct kvm_clock_data data;
- int ret;
if (s->clock_valid) {
return;
@@ -129,13 +167,7 @@
kvm_synchronize_all_tsc();
- ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
- if (ret < 0) {
- fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
- abort();
- }
- s->clock = data.clock;
-
+ s->clock = kvm_get_clock();
/*
* If the VM is stopped, declare the clock state valid to
* avoid re-reading it on next vmsave (which would return
@@ -149,25 +181,72 @@
{
KVMClockState *s = KVM_CLOCK(dev);
+ /*
+ * On machine types that support reliable KVM_GET_CLOCK,
+ * if host kernel does provide reliable KVM_GET_CLOCK,
+ * set src_use_reliable_get_clock=true so that destination
+ * avoids reading kvmclock from memory.
+ */
+ if (s->mach_use_reliable_get_clock && kvm_has_adjust_clock_stable()) {
+ s->src_use_reliable_get_clock = true;
+ }
+
qemu_add_vm_change_state_handler(kvmclock_vm_state_change, s);
}
+static bool kvmclock_src_use_reliable_get_clock(void *opaque)
+{
+ KVMClockState *s = opaque;
+
+ return s->mach_use_reliable_get_clock;
+}
+
+static const VMStateDescription kvmclock_reliable_get_clock = {
+ .name = "kvmclock/src_use_reliable_get_clock",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .needed = kvmclock_src_use_reliable_get_clock,
+ .fields = (VMStateField[]) {
+ VMSTATE_BOOL(src_use_reliable_get_clock, KVMClockState),
+ VMSTATE_END_OF_LIST()
+ }
+};
+
+static void kvmclock_pre_save(void *opaque)
+{
+ KVMClockState *s = opaque;
+
+ s->clock = kvm_get_clock();
+}
+
static const VMStateDescription kvmclock_vmsd = {
.name = "kvmclock",
.version_id = 1,
.minimum_version_id = 1,
+ .pre_save = kvmclock_pre_save,
.fields = (VMStateField[]) {
VMSTATE_UINT64(clock, KVMClockState),
VMSTATE_END_OF_LIST()
+ },
+ .subsections = (const VMStateDescription * []) {
+ &kvmclock_reliable_get_clock,
+ NULL
}
};
+static Property kvmclock_properties[] = {
+ DEFINE_PROP_BOOL("x-mach-use-reliable-get-clock", KVMClockState,
+ mach_use_reliable_get_clock, true),
+ DEFINE_PROP_END_OF_LIST(),
+};
+
static void kvmclock_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
dc->realize = kvmclock_realize;
dc->vmsd = &kvmclock_vmsd;
+ dc->props = kvmclock_properties;
}
static const TypeInfo kvmclock_info = {
Index: qemu-mig-advance-clock/include/hw/i386/pc.h
===================================================================
--- qemu-mig-advance-clock.orig/include/hw/i386/pc.h 2016-11-17 15:07:11.220632761 -0200
+++ qemu-mig-advance-clock/include/hw/i386/pc.h 2016-11-17 15:08:58.096791416 -0200
@@ -370,6 +370,11 @@
#define PC_COMPAT_2_7 \
HW_COMPAT_2_7 \
{\
+ .driver = "kvmclock",\
+ .property = "x-mach-use-reliable-get-clock",\
+ .value = "off",\
+ },\
+ {\
.driver = TYPE_X86_CPU,\
.property = "l3-cache",\
.value = "off",\
Index: qemu-mig-advance-clock/target-i386/kvm.c
===================================================================
--- qemu-mig-advance-clock.orig/target-i386/kvm.c 2016-11-17 15:07:11.221632762 -0200
+++ qemu-mig-advance-clock/target-i386/kvm.c 2016-11-17 15:07:15.867639659 -0200
@@ -117,6 +117,13 @@
return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
}
+bool kvm_has_adjust_clock_stable(void)
+{
+ int ret = kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK);
+
+ return (ret == KVM_CLOCK_TSC_STABLE);
+}
+
bool kvm_allows_irq0_override(void)
{
return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
Index: qemu-mig-advance-clock/target-i386/kvm_i386.h
===================================================================
--- qemu-mig-advance-clock.orig/target-i386/kvm_i386.h 2016-11-17 15:07:11.222632764 -0200
+++ qemu-mig-advance-clock/target-i386/kvm_i386.h 2016-11-17 15:07:15.867639659 -0200
@@ -17,6 +17,7 @@
bool kvm_allows_irq0_override(void);
bool kvm_has_smm(void);
+bool kvm_has_adjust_clock_stable(void);
void kvm_synchronize_all_tsc(void);
void kvm_arch_reset_vcpu(X86CPU *cs);
void kvm_arch_do_init_vcpu(X86CPU *cs);
WARNING: multiple messages have this Message-ID (diff)
From: Marcelo Tosatti <mtosatti@redhat.com>
To: kvm@vger.kernel.org
Cc: qemu-devel@nongnu.org,
Dr David Alan Gilbert <dgilbert@redhat.com>,
Paolo Bonzini <pbonzini@redhat.com>,
Juan Quintela <quintela@redhat.com>,
Radim Krcmar <rkrcmar@redhat.com>,
Eduardo Habkost <ehabkost@redhat.com>,
Marcelo Tosatti <mtosatti@redhat.com>
Subject: [Qemu-devel] [qemu patch V3 2/2] kvmclock: reduce kvmclock difference on migration
Date: Mon, 21 Nov 2016 08:50:04 -0200 [thread overview]
Message-ID: <20161121105052.598267440@redhat.com> (raw)
In-Reply-To: 20161121105002.291554230@redhat.com
[-- Attachment #1: kvmclock-advance-clock.patch --]
[-- Type: text/plain, Size: 7864 bytes --]
Check for KVM_CAP_ADJUST_CLOCK capability KVM_CLOCK_TSC_STABLE, which
indicates that KVM_GET_CLOCK returns a value as seen by the guest at
that moment.
For new machine types, use this value rather than reading
from guest memory.
This reduces kvmclock difference on migration from 5s to 0.1s
(when max_downtime == 5s).
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
---
hw/i386/kvm/clock.c | 107 ++++++++++++++++++++++++++++++++++++++++++-------
include/hw/i386/pc.h | 5 ++
target-i386/kvm.c | 7 +++
target-i386/kvm_i386.h | 1
4 files changed, 106 insertions(+), 14 deletions(-)
v2:
- improve variable names (Juan)
- consolidate code on kvm_get_clock function (Paolo)
- return mach_use_reliable_get_clock from needed function (Paolo)
v3:
- simplify check for src_use_reliable_get_clock (Eduardo)
- move src_use_reliable_get_clock initialization to realize (Eduardo)
Index: qemu-mig-advance-clock/hw/i386/kvm/clock.c
===================================================================
--- qemu-mig-advance-clock.orig/hw/i386/kvm/clock.c 2016-11-17 15:07:11.220632761 -0200
+++ qemu-mig-advance-clock/hw/i386/kvm/clock.c 2016-11-17 15:11:51.372048640 -0200
@@ -36,6 +36,12 @@
uint64_t clock;
bool clock_valid;
+
+ /* whether machine supports reliable KVM_GET_CLOCK */
+ bool mach_use_reliable_get_clock;
+
+ /* whether source host supported reliable KVM_GET_CLOCK */
+ bool src_use_reliable_get_clock;
} KVMClockState;
struct pvclock_vcpu_time_info {
@@ -81,6 +87,19 @@
return nsec + time.system_time;
}
+static uint64_t kvm_get_clock(void)
+{
+ struct kvm_clock_data data;
+ int ret;
+
+ ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
+ if (ret < 0) {
+ fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
+ abort();
+ }
+ return data.clock;
+}
+
static void kvmclock_vm_state_change(void *opaque, int running,
RunState state)
{
@@ -91,15 +110,36 @@
if (running) {
struct kvm_clock_data data = {};
- uint64_t time_at_migration = kvmclock_current_nsec(s);
+ uint64_t pvclock_via_mem = 0;
- s->clock_valid = false;
+ /* local (running VM) restore */
+ if (s->clock_valid) {
+ /*
+ * if host does not support reliable KVM_GET_CLOCK,
+ * read kvmclock value from memory
+ */
+ if (!kvm_has_adjust_clock_stable()) {
+ pvclock_via_mem = kvmclock_current_nsec(s);
+ }
+ /* migration/savevm/init restore */
+ } else {
+ /*
+ * use s->clock in case machine uses reliable
+ * get clock and source host supported
+ * reliable get clock
+ */
+ if (!s->src_use_reliable_get_clock) {
+ pvclock_via_mem = kvmclock_current_nsec(s);
+ }
+ }
- /* We can't rely on the migrated clock value, just discard it */
- if (time_at_migration) {
- s->clock = time_at_migration;
+ /* We can't rely on the saved clock value, just discard it */
+ if (pvclock_via_mem) {
+ s->clock = pvclock_via_mem;
}
+ s->clock_valid = false;
+
data.clock = s->clock;
ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
if (ret < 0) {
@@ -120,8 +160,6 @@
}
}
} else {
- struct kvm_clock_data data;
- int ret;
if (s->clock_valid) {
return;
@@ -129,13 +167,7 @@
kvm_synchronize_all_tsc();
- ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
- if (ret < 0) {
- fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
- abort();
- }
- s->clock = data.clock;
-
+ s->clock = kvm_get_clock();
/*
* If the VM is stopped, declare the clock state valid to
* avoid re-reading it on next vmsave (which would return
@@ -149,25 +181,72 @@
{
KVMClockState *s = KVM_CLOCK(dev);
+ /*
+ * On machine types that support reliable KVM_GET_CLOCK,
+ * if host kernel does provide reliable KVM_GET_CLOCK,
+ * set src_use_reliable_get_clock=true so that destination
+ * avoids reading kvmclock from memory.
+ */
+ if (s->mach_use_reliable_get_clock && kvm_has_adjust_clock_stable()) {
+ s->src_use_reliable_get_clock = true;
+ }
+
qemu_add_vm_change_state_handler(kvmclock_vm_state_change, s);
}
+static bool kvmclock_src_use_reliable_get_clock(void *opaque)
+{
+ KVMClockState *s = opaque;
+
+ return s->mach_use_reliable_get_clock;
+}
+
+static const VMStateDescription kvmclock_reliable_get_clock = {
+ .name = "kvmclock/src_use_reliable_get_clock",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .needed = kvmclock_src_use_reliable_get_clock,
+ .fields = (VMStateField[]) {
+ VMSTATE_BOOL(src_use_reliable_get_clock, KVMClockState),
+ VMSTATE_END_OF_LIST()
+ }
+};
+
+static void kvmclock_pre_save(void *opaque)
+{
+ KVMClockState *s = opaque;
+
+ s->clock = kvm_get_clock();
+}
+
static const VMStateDescription kvmclock_vmsd = {
.name = "kvmclock",
.version_id = 1,
.minimum_version_id = 1,
+ .pre_save = kvmclock_pre_save,
.fields = (VMStateField[]) {
VMSTATE_UINT64(clock, KVMClockState),
VMSTATE_END_OF_LIST()
+ },
+ .subsections = (const VMStateDescription * []) {
+ &kvmclock_reliable_get_clock,
+ NULL
}
};
+static Property kvmclock_properties[] = {
+ DEFINE_PROP_BOOL("x-mach-use-reliable-get-clock", KVMClockState,
+ mach_use_reliable_get_clock, true),
+ DEFINE_PROP_END_OF_LIST(),
+};
+
static void kvmclock_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
dc->realize = kvmclock_realize;
dc->vmsd = &kvmclock_vmsd;
+ dc->props = kvmclock_properties;
}
static const TypeInfo kvmclock_info = {
Index: qemu-mig-advance-clock/include/hw/i386/pc.h
===================================================================
--- qemu-mig-advance-clock.orig/include/hw/i386/pc.h 2016-11-17 15:07:11.220632761 -0200
+++ qemu-mig-advance-clock/include/hw/i386/pc.h 2016-11-17 15:08:58.096791416 -0200
@@ -370,6 +370,11 @@
#define PC_COMPAT_2_7 \
HW_COMPAT_2_7 \
{\
+ .driver = "kvmclock",\
+ .property = "x-mach-use-reliable-get-clock",\
+ .value = "off",\
+ },\
+ {\
.driver = TYPE_X86_CPU,\
.property = "l3-cache",\
.value = "off",\
Index: qemu-mig-advance-clock/target-i386/kvm.c
===================================================================
--- qemu-mig-advance-clock.orig/target-i386/kvm.c 2016-11-17 15:07:11.221632762 -0200
+++ qemu-mig-advance-clock/target-i386/kvm.c 2016-11-17 15:07:15.867639659 -0200
@@ -117,6 +117,13 @@
return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
}
+bool kvm_has_adjust_clock_stable(void)
+{
+ int ret = kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK);
+
+ return (ret == KVM_CLOCK_TSC_STABLE);
+}
+
bool kvm_allows_irq0_override(void)
{
return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
Index: qemu-mig-advance-clock/target-i386/kvm_i386.h
===================================================================
--- qemu-mig-advance-clock.orig/target-i386/kvm_i386.h 2016-11-17 15:07:11.222632764 -0200
+++ qemu-mig-advance-clock/target-i386/kvm_i386.h 2016-11-17 15:07:15.867639659 -0200
@@ -17,6 +17,7 @@
bool kvm_allows_irq0_override(void);
bool kvm_has_smm(void);
+bool kvm_has_adjust_clock_stable(void);
void kvm_synchronize_all_tsc(void);
void kvm_arch_reset_vcpu(X86CPU *cs);
void kvm_arch_do_init_vcpu(X86CPU *cs);
next prev parent reply other threads:[~2016-11-21 10:52 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-11-21 10:50 [qemu patch V3 0/2] improve kvmclock difference on migration Marcelo Tosatti
2016-11-21 10:50 ` [Qemu-devel] " Marcelo Tosatti
2016-11-21 10:50 ` [qemu patch V3 1/2] kvm: sync linux headers Marcelo Tosatti
2016-11-21 10:50 ` [Qemu-devel] " Marcelo Tosatti
2016-11-21 10:50 ` Marcelo Tosatti [this message]
2016-11-21 10:50 ` [Qemu-devel] [qemu patch V3 2/2] kvmclock: reduce kvmclock difference on migration Marcelo Tosatti
2016-11-28 14:13 ` Eduardo Habkost
2016-11-28 14:13 ` [Qemu-devel] " Eduardo Habkost
2016-11-28 16:45 ` Marcelo Tosatti
2016-11-28 16:45 ` [Qemu-devel] " Marcelo Tosatti
2016-11-28 17:12 ` Eduardo Habkost
2016-11-28 17:12 ` [Qemu-devel] " Eduardo Habkost
2016-11-28 17:31 ` Paolo Bonzini
2016-11-28 17:31 ` [Qemu-devel] " Paolo Bonzini
2016-11-29 10:56 ` Marcelo Tosatti
2016-11-29 10:56 ` [Qemu-devel] " Marcelo Tosatti
2016-11-29 12:34 ` Eduardo Habkost
2016-11-29 12:34 ` [Qemu-devel] " Eduardo Habkost
2016-11-29 23:54 ` Marcelo Tosatti
2016-11-29 23:54 ` [Qemu-devel] " Marcelo Tosatti
2016-11-30 13:09 ` Eduardo Habkost
2016-11-30 13:09 ` [Qemu-devel] " Eduardo Habkost
2016-12-01 14:39 ` Marcelo Tosatti
2016-12-01 14:39 ` [Qemu-devel] " Marcelo Tosatti
2016-12-02 13:56 ` Eduardo Habkost
2016-12-02 13:56 ` [Qemu-devel] " Eduardo Habkost
2016-12-02 22:30 ` Marcelo Tosatti
2016-12-02 22:30 ` [Qemu-devel] " Marcelo Tosatti
2016-12-05 15:17 ` Eduardo Habkost
2016-12-05 15:17 ` [Qemu-devel] " Eduardo Habkost
2016-12-05 17:20 ` Marcelo Tosatti
2016-12-05 17:20 ` [Qemu-devel] " Marcelo Tosatti
2016-12-06 0:47 ` Eduardo Habkost
2016-12-06 0:47 ` [Qemu-devel] " Eduardo Habkost
2016-11-28 12:16 ` [qemu patch V3 0/2] improve " Marcelo Tosatti
2016-11-28 12:16 ` [Qemu-devel] " Marcelo Tosatti
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20161121105052.598267440@redhat.com \
--to=mtosatti@redhat.com \
--cc=dgilbert@redhat.com \
--cc=ehabkost@redhat.com \
--cc=kvm@vger.kernel.org \
--cc=pbonzini@redhat.com \
--cc=qemu-devel@nongnu.org \
--cc=quintela@redhat.com \
--cc=rkrcmar@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.