All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] tsc: use kvmclock for calibration
@ 2012-08-09 11:57 Gerd Hoffmann
  2012-08-09 12:53 ` Avi Kivity
  0 siblings, 1 reply; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-09 11:57 UTC (permalink / raw)
  To: seabios; +Cc: kvm, Gerd Hoffmann

Use kvmclock for tsc calibration when running on kvm.  Without this the
tsc frequency calibrated by seabios can be *way* off in case the virtual
machine is booted on a loaded host.  I've seen seabios calibrating 27
instead of ca. 2800 MHz, resulting in timeouts being to short by factor
100.  Which in turn leads to disk I/O errors due to timeouts, especially
as I/O requests tend to take a bit longer than usual on a loaded box ...

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 src/clock.c    |    9 +++++
 src/paravirt.c |   90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/paravirt.h |    1 +
 3 files changed, 100 insertions(+), 0 deletions(-)

diff --git a/src/clock.c b/src/clock.c
index 69e9f17..5883b1a 100644
--- a/src/clock.c
+++ b/src/clock.c
@@ -13,6 +13,7 @@
 #include "bregs.h" // struct bregs
 #include "biosvar.h" // GET_GLOBAL
 #include "usb-hid.h" // usb_check_event
+#include "paravirt.h" // kvm clock
 
 // RTC register flags
 #define RTC_A_UIP 0x80
@@ -80,6 +81,14 @@ calibrate_tsc(void)
         return;
     }
 
+    if (kvm_para_available()) {
+        u32 khz = kvm_tsc_khz();
+        if (khz != 0) {
+            SET_GLOBAL(cpu_khz, khz);
+            return;
+        }
+    }
+
     // Setup "timer2"
     u8 orig = inb(PORT_PS2_CTRLB);
     outb((orig & ~PPCB_SPKR) | PPCB_T2GATE, PORT_PS2_CTRLB);
diff --git a/src/paravirt.c b/src/paravirt.c
index 2a98d53..942ce11 100644
--- a/src/paravirt.c
+++ b/src/paravirt.c
@@ -12,6 +12,7 @@
 #include "ioport.h" // outw
 #include "paravirt.h" // qemu_cfg_port_probe
 #include "smbios.h" // struct smbios_structure_header
+#include "biosvar.h" // GET_GLOBAL
 
 int qemu_cfg_present;
 
@@ -346,3 +347,92 @@ void qemu_cfg_romfile_setup(void)
         dprintf(3, "Found fw_cfg file: %s (size=%d)\n", file->name, file->size);
     }
 }
+
+#define KVM_CPUID_SIGNATURE       0x40000000
+#define KVM_CPUID_FEATURES        0x40000001
+#define KVM_FEATURE_CLOCKSOURCE            0
+#define KVM_FEATURE_CLOCKSOURCE2           3
+#define MSR_KVM_SYSTEM_TIME             0x12
+#define MSR_KVM_SYSTEM_TIME_NEW   0x4b564d01
+
+struct pvclock_vcpu_time_info {
+	u32   version;
+	u32   pad0;
+	u64   tsc_timestamp;
+	u64   system_time;
+	u32   tsc_to_system_mul;
+	s8    tsc_shift;
+	u8    flags;
+	u8    pad[2];
+} PACKED;
+
+/*
+ * do_div() is NOT a C function. It wants to return
+ * two values (the quotient and the remainder), but
+ * since that doesn't work very well in C, what it
+ * does is:
+ *
+ * - modifies the 64-bit dividend _in_place_
+ * - returns the 32-bit remainder
+ *
+ * This ends up being the most efficient "calling
+ * convention" on x86.
+ */
+#define do_div(n, base)                                                 \
+    ({                                                                  \
+        unsigned long __upper, __low, __high, __mod, __base;            \
+        __base = (base);                                                \
+        asm("" : "=a" (__low), "=d" (__high) : "A" (n));                \
+        __upper = __high;                                               \
+        if (__high) {                                                   \
+            __upper = __high % (__base);                                \
+            __high = __high / (__base);                                 \
+        }                                                               \
+        asm("divl %2" : "=a" (__low), "=d" (__mod)                      \
+            : "rm" (__base), "0" (__low), "1" (__upper));               \
+        asm("" : "=A" (n) : "a" (__low), "d" (__high));                 \
+        __mod;                                                          \
+    })
+
+static u64 pvclock_tsc_khz(struct pvclock_vcpu_time_info *src)
+{
+    u64 pv_tsc_khz = 1000000ULL << 32;
+
+    do_div(pv_tsc_khz, src->tsc_to_system_mul);
+    if (src->tsc_shift < 0)
+        pv_tsc_khz <<= -src->tsc_shift;
+    else
+        pv_tsc_khz >>= src->tsc_shift;
+    return pv_tsc_khz;
+}
+
+u64 kvm_tsc_khz(void)
+{
+    u32 eax, ebx, ecx, edx, msr;
+    struct pvclock_vcpu_time_info time;
+    u32 addr = (u32)(&time);
+    u64 khz;
+
+    /* check presence and figure msr number */
+    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
+    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
+        msr = MSR_KVM_SYSTEM_TIME_NEW;
+    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
+        msr = MSR_KVM_SYSTEM_TIME;
+    } else {
+        return 0;
+    }
+
+    /* ask kvm hypervisor to fill struct */
+    memset(&time, 0, sizeof(time));
+    wrmsr(msr, addr | 1);
+    wrmsr(msr, 0);
+    if (time.version < 2 || time.tsc_to_system_mul == 0)
+        return 0;
+
+    /* go figure tsc frequency */
+    khz = pvclock_tsc_khz(&time);
+    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
+            msr, (u32)khz / 1000);
+    return khz;
+}
diff --git a/src/paravirt.h b/src/paravirt.h
index a284c41..eedfcc3 100644
--- a/src/paravirt.h
+++ b/src/paravirt.h
@@ -27,6 +27,7 @@ static inline int kvm_para_available(void)
 
     return 0;
 }
+extern u64 kvm_tsc_khz(void);
 
 #define QEMU_CFG_SIGNATURE              0x00
 #define QEMU_CFG_ID                     0x01
-- 
1.7.1


^ permalink raw reply related	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 11:57 [PATCH] tsc: use kvmclock for calibration Gerd Hoffmann
@ 2012-08-09 12:53 ` Avi Kivity
  2012-08-09 13:25   ` [SeaBIOS] " Fred .
                     ` (2 more replies)
  0 siblings, 3 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-09 12:53 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, kvm

On 08/09/2012 02:57 PM, Gerd Hoffmann wrote:
> Use kvmclock for tsc calibration when running on kvm.  Without this the
> tsc frequency calibrated by seabios can be *way* off in case the virtual
> machine is booted on a loaded host.  I've seen seabios calibrating 27
> instead of ca. 2800 MHz, resulting in timeouts being to short by factor
> 100.  Which in turn leads to disk I/O errors due to timeouts, especially
> as I/O requests tend to take a bit longer than usual on a loaded box ...

> +
> +struct pvclock_vcpu_time_info {
> +	u32   version;
> +	u32   pad0;
> +	u64   tsc_timestamp;
> +	u64   system_time;
> +	u32   tsc_to_system_mul;
> +	s8    tsc_shift;
> +	u8    flags;
> +	u8    pad[2];
> +} PACKED;
> +
> +
> +u64 kvm_tsc_khz(void)
> +{
> +    u32 eax, ebx, ecx, edx, msr;
> +    struct pvclock_vcpu_time_info time;
> +    u32 addr = (u32)(&time);
> +    u64 khz;
> +
> +    /* check presence and figure msr number */
> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
> +        msr = MSR_KVM_SYSTEM_TIME;
> +    } else {
> +        return 0;
> +    }
> +
> +    /* ask kvm hypervisor to fill struct */
> +    memset(&time, 0, sizeof(time));
> +    wrmsr(msr, addr | 1);

How can this work?  There is a 64-byte alignment requirement.

> +    wrmsr(msr, 0);
> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
> +        return 0;
> +
> +    /* go figure tsc frequency */
> +    khz = pvclock_tsc_khz(&time);
> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
> +            msr, (u32)khz / 1000);
> +    return khz;

That's a meaningless number.  You can be migrated to a cpu or a machine
with very different tsc.

You want accurate time on kvm, don't use the tsc.


-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [SeaBIOS] [PATCH] tsc: use kvmclock for calibration
  2012-08-09 12:53 ` Avi Kivity
@ 2012-08-09 13:25   ` Fred .
  2012-08-09 13:57   ` Gerd Hoffmann
  2012-08-09 18:59   ` Marcelo Tosatti
  2 siblings, 0 replies; 23+ messages in thread
From: Fred . @ 2012-08-09 13:25 UTC (permalink / raw)
  To: Avi Kivity; +Cc: Gerd Hoffmann, seabios, kvm

It should be kHz not khz.

-            msr, (u32)khz / 1000);
+            msr, (u32)kHz / 1000);

On Thu, Aug 9, 2012 at 2:53 PM, Avi Kivity <avi@redhat.com> wrote:
> On 08/09/2012 02:57 PM, Gerd Hoffmann wrote:
>> Use kvmclock for tsc calibration when running on kvm.  Without this the
>> tsc frequency calibrated by seabios can be *way* off in case the virtual
>> machine is booted on a loaded host.  I've seen seabios calibrating 27
>> instead of ca. 2800 MHz, resulting in timeouts being to short by factor
>> 100.  Which in turn leads to disk I/O errors due to timeouts, especially
>> as I/O requests tend to take a bit longer than usual on a loaded box ...
>
>> +
>> +struct pvclock_vcpu_time_info {
>> +     u32   version;
>> +     u32   pad0;
>> +     u64   tsc_timestamp;
>> +     u64   system_time;
>> +     u32   tsc_to_system_mul;
>> +     s8    tsc_shift;
>> +     u8    flags;
>> +     u8    pad[2];
>> +} PACKED;
>> +
>> +
>> +u64 kvm_tsc_khz(void)
>> +{
>> +    u32 eax, ebx, ecx, edx, msr;
>> +    struct pvclock_vcpu_time_info time;
>> +    u32 addr = (u32)(&time);
>> +    u64 khz;
>> +
>> +    /* check presence and figure msr number */
>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
>> +        msr = MSR_KVM_SYSTEM_TIME;
>> +    } else {
>> +        return 0;
>> +    }
>> +
>> +    /* ask kvm hypervisor to fill struct */
>> +    memset(&time, 0, sizeof(time));
>> +    wrmsr(msr, addr | 1);
>
> How can this work?  There is a 64-byte alignment requirement.
>
>> +    wrmsr(msr, 0);
>> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
>> +        return 0;
>> +
>> +    /* go figure tsc frequency */
>> +    khz = pvclock_tsc_khz(&time);
>> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
>> +            msr, (u32)khz / 1000);
>> +    return khz;
>
> That's a meaningless number.  You can be migrated to a cpu or a machine
> with very different tsc.
>
> You want accurate time on kvm, don't use the tsc.
>
>
> --
> error compiling committee.c: too many arguments to function
>
> _______________________________________________
> SeaBIOS mailing list
> SeaBIOS@seabios.org
> http://www.seabios.org/mailman/listinfo/seabios

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 12:53 ` Avi Kivity
  2012-08-09 13:25   ` [SeaBIOS] " Fred .
@ 2012-08-09 13:57   ` Gerd Hoffmann
  2012-08-09 14:01     ` Avi Kivity
  2012-08-09 18:59   ` Marcelo Tosatti
  2 siblings, 1 reply; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-09 13:57 UTC (permalink / raw)
  To: Avi Kivity; +Cc: seabios, kvm

  Hi,

>> +u64 kvm_tsc_khz(void)
>> +{
>> +    u32 eax, ebx, ecx, edx, msr;
>> +    struct pvclock_vcpu_time_info time;
>> +    u32 addr = (u32)(&time);
>> +    u64 khz;
>> +
>> +    /* check presence and figure msr number */
>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
>> +        msr = MSR_KVM_SYSTEM_TIME;
>> +    } else {
>> +        return 0;
>> +    }
>> +
>> +    /* ask kvm hypervisor to fill struct */
>> +    memset(&time, 0, sizeof(time));
>> +    wrmsr(msr, addr | 1);
> 
> How can this work?

It did in my testing, although maybe by pure luck ...

> There is a 64-byte alignment requirement.

64 bytes?  Sure?  The whole struct is only 32 bytes in size ...

Easily fixable though, just need to grab some memory with memalign
instead of using the stack.

>> +    wrmsr(msr, 0);
>> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
>> +        return 0;
>> +
>> +    /* go figure tsc frequency */
>> +    khz = pvclock_tsc_khz(&time);
>> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
>> +            msr, (u32)khz / 1000);
>> +    return khz;
> 
> That's a meaningless number.  You can be migrated to a cpu or a machine
> with very different tsc.

> You want accurate time on kvm, don't use the tsc.

seabios uses the tsc for timeout calculations only, so it doesn't need
to be 100% accurate.  The order of magnitude should be correct though.
The Linux kernel uses the value for delay loops too, so using it for the
given purpose can't be *that* horrible after all ...

It is certainly an improvement over the current code which tries to
calibrate the tsc and gets totally broken results in case the busy host
happens to schedule the guest in the middle of calibration.

So what do you suggest?  The options I see are:

  (1) Use this patch (with alignment issue fixed of course).
  (2) Do a full kvmclock implementation.  Feels a bit like overkill.
  (3) SeaBIOS can fallback to the PIT for timing on machines which
      have no TSC.  We could do that too in case we detect kvm ...

cheers,
  Gerd

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 13:57   ` Gerd Hoffmann
@ 2012-08-09 14:01     ` Avi Kivity
  2012-08-09 14:05       ` Avi Kivity
                         ` (2 more replies)
  0 siblings, 3 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-09 14:01 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, kvm

On 08/09/2012 04:57 PM, Gerd Hoffmann wrote:
>   Hi,
> 
>>> +u64 kvm_tsc_khz(void)
>>> +{
>>> +    u32 eax, ebx, ecx, edx, msr;
>>> +    struct pvclock_vcpu_time_info time;
>>> +    u32 addr = (u32)(&time);
>>> +    u64 khz;
>>> +
>>> +    /* check presence and figure msr number */
>>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
>>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
>>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
>>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
>>> +        msr = MSR_KVM_SYSTEM_TIME;
>>> +    } else {
>>> +        return 0;
>>> +    }
>>> +
>>> +    /* ask kvm hypervisor to fill struct */
>>> +    memset(&time, 0, sizeof(time));
>>> +    wrmsr(msr, addr | 1);
>> 
>> How can this work?
> 
> It did in my testing, although maybe by pure luck ...
> 
>> There is a 64-byte alignment requirement.
> 
> 64 bytes?  Sure?  The whole struct is only 32 bytes in size ...

er, the documentation says 4 bytes (so stack alignment works).  I
distinctly remember having a large alignment requirement so we don't
cross a page or slot boundary... something's wrong here.

> 
> Easily fixable though, just need to grab some memory with memalign
> instead of using the stack.

> 
>>> +    wrmsr(msr, 0);
>>> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
>>> +        return 0;
>>> +
>>> +    /* go figure tsc frequency */
>>> +    khz = pvclock_tsc_khz(&time);
>>> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
>>> +            msr, (u32)khz / 1000);
>>> +    return khz;
>> 
>> That's a meaningless number.  You can be migrated to a cpu or a machine
>> with very different tsc.
> 
>> You want accurate time on kvm, don't use the tsc.
> 
> seabios uses the tsc for timeout calculations only, so it doesn't need
> to be 100% accurate.  The order of magnitude should be correct though.
> The Linux kernel uses the value for delay loops too, so using it for the
> given purpose can't be *that* horrible after all ...
> 
> It is certainly an improvement over the current code which tries to
> calibrate the tsc and gets totally broken results in case the busy host
> happens to schedule the guest in the middle of calibration.
> 
> So what do you suggest?  The options I see are:
> 
>   (1) Use this patch (with alignment issue fixed of course).
>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>       have no TSC.  We could do that too in case we detect kvm ...

What sort of timeouts are these?  If seconds, maybe the rtc would be best.

-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:01     ` Avi Kivity
@ 2012-08-09 14:05       ` Avi Kivity
  2012-08-09 14:12         ` Gerd Hoffmann
  2012-08-09 14:18       ` Gerd Hoffmann
  2012-08-09 19:09       ` Marcelo Tosatti
  2 siblings, 1 reply; 23+ messages in thread
From: Avi Kivity @ 2012-08-09 14:05 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, kvm

On 08/09/2012 05:01 PM, Avi Kivity wrote:
> On 08/09/2012 04:57 PM, Gerd Hoffmann wrote:
>>   Hi,
>> 
>>>> +u64 kvm_tsc_khz(void)
>>>> +{
>>>> +    u32 eax, ebx, ecx, edx, msr;
>>>> +    struct pvclock_vcpu_time_info time;
>>>> +    u32 addr = (u32)(&time);
>>>> +    u64 khz;
>>>> +
>>>> +    /* check presence and figure msr number */
>>>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
>>>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
>>>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
>>>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
>>>> +        msr = MSR_KVM_SYSTEM_TIME;
>>>> +    } else {
>>>> +        return 0;
>>>> +    }
>>>> +
>>>> +    /* ask kvm hypervisor to fill struct */
>>>> +    memset(&time, 0, sizeof(time));
>>>> +    wrmsr(msr, addr | 1);
>>> 
>>> How can this work?
>> 
>> It did in my testing, although maybe by pure luck ...
>> 
>>> There is a 64-byte alignment requirement.
>> 
>> 64 bytes?  Sure?  The whole struct is only 32 bytes in size ...
> 
> er, the documentation says 4 bytes (so stack alignment works).  I
> distinctly remember having a large alignment requirement so we don't
> cross a page or slot boundary... something's wrong here.

	case MSR_KVM_SYSTEM_TIME: {
		kvmclock_reset(vcpu);

		vcpu->arch.time = data;
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		/* we verify if the enable bit is set... */
		if (!(data & 1))
			break;

		/* ...but clean it before doing the actual write */
		vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);

		vcpu->arch.time_page =
				gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);

		if (is_error_page(vcpu->arch.time_page))
			vcpu->arch.time_page = NULL;

		break;

So your tests worked by pure luck, but the bug is in kvm.  We need to
grab two pages here.


-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:05       ` Avi Kivity
@ 2012-08-09 14:12         ` Gerd Hoffmann
  2012-08-09 14:17           ` Avi Kivity
  0 siblings, 1 reply; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-09 14:12 UTC (permalink / raw)
  To: Avi Kivity; +Cc: seabios, kvm

  Hi,

>> er, the documentation says 4 bytes (so stack alignment works).  I
>> distinctly remember having a large alignment requirement so we don't
>> cross a page or slot boundary... something's wrong here.
> 
> 	case MSR_KVM_SYSTEM_TIME: {
[ ... ]

> So your tests worked by pure luck, but the bug is in kvm.  We need to
> grab two pages here.

Ok, so better use memalign(32,32) to make sure the struct doesn't cross
a page border ...

cheers,
  Gerd

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:12         ` Gerd Hoffmann
@ 2012-08-09 14:17           ` Avi Kivity
  0 siblings, 0 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-09 14:17 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, kvm

On 08/09/2012 05:12 PM, Gerd Hoffmann wrote:
>   Hi,
> 
>>> er, the documentation says 4 bytes (so stack alignment works).  I
>>> distinctly remember having a large alignment requirement so we don't
>>> cross a page or slot boundary... something's wrong here.
>> 
>> 	case MSR_KVM_SYSTEM_TIME: {
> [ ... ]
> 
>> So your tests worked by pure luck, but the bug is in kvm.  We need to
>> grab two pages here.
> 
> Ok, so better use memalign(32,32) to make sure the struct doesn't cross
> a page border ...

No, we need to fix kvm, no need to complicate the guest for that.


-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:01     ` Avi Kivity
  2012-08-09 14:05       ` Avi Kivity
@ 2012-08-09 14:18       ` Gerd Hoffmann
  2012-08-09 14:20         ` Avi Kivity
  2012-08-09 19:09       ` Marcelo Tosatti
  2 siblings, 1 reply; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-09 14:18 UTC (permalink / raw)
  To: Avi Kivity; +Cc: seabios, kvm

  Hi,

>> So what do you suggest?  The options I see are:
>>
>>   (1) Use this patch (with alignment issue fixed of course).
>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>>       have no TSC.  We could do that too in case we detect kvm ...
> 
> What sort of timeouts are these?  If seconds, maybe the rtc would be best.

All sorts of timeouts, from a few miliseconds to seconds.

The problematic ones are the longer timeouts, which wait for I/O stuff
like disk reads complete.  The stuff with smaller timeouts (like waiting
for AHCI link become ready) tend to finish instantly in kvm.

cheers,
  Gerd

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:18       ` Gerd Hoffmann
@ 2012-08-09 14:20         ` Avi Kivity
  2012-08-09 19:02           ` Marcelo Tosatti
  0 siblings, 1 reply; 23+ messages in thread
From: Avi Kivity @ 2012-08-09 14:20 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, kvm

On 08/09/2012 05:18 PM, Gerd Hoffmann wrote:
>   Hi,
> 
>>> So what do you suggest?  The options I see are:
>>>
>>>   (1) Use this patch (with alignment issue fixed of course).
>>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>>>       have no TSC.  We could do that too in case we detect kvm ...
>> 
>> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
> 
> All sorts of timeouts, from a few miliseconds to seconds.
> 
> The problematic ones are the longer timeouts, which wait for I/O stuff
> like disk reads complete.  The stuff with smaller timeouts (like waiting
> for AHCI link become ready) tend to finish instantly in kvm.

That's not guaranteed.  The AHCI adapter might be real hardware.  Or the
emulation may change.

What's wrong with having a full kvmclock implementation?  Instead of
issuing rdtsc call a function pointer.

-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 12:53 ` Avi Kivity
  2012-08-09 13:25   ` [SeaBIOS] " Fred .
  2012-08-09 13:57   ` Gerd Hoffmann
@ 2012-08-09 18:59   ` Marcelo Tosatti
  2012-08-12  9:01     ` Avi Kivity
  2 siblings, 1 reply; 23+ messages in thread
From: Marcelo Tosatti @ 2012-08-09 18:59 UTC (permalink / raw)
  To: Avi Kivity; +Cc: seabios, Gerd Hoffmann, kvm

On Thu, Aug 09, 2012 at 03:53:24PM +0300, Avi Kivity wrote:
> On 08/09/2012 02:57 PM, Gerd Hoffmann wrote:
> > Use kvmclock for tsc calibration when running on kvm.  Without this the
> > tsc frequency calibrated by seabios can be *way* off in case the virtual
> > machine is booted on a loaded host.  I've seen seabios calibrating 27
> > instead of ca. 2800 MHz, resulting in timeouts being to short by factor
> > 100.  Which in turn leads to disk I/O errors due to timeouts, especially
> > as I/O requests tend to take a bit longer than usual on a loaded box ...
> 
> > +
> > +struct pvclock_vcpu_time_info {
> > +	u32   version;
> > +	u32   pad0;
> > +	u64   tsc_timestamp;
> > +	u64   system_time;
> > +	u32   tsc_to_system_mul;
> > +	s8    tsc_shift;
> > +	u8    flags;
> > +	u8    pad[2];
> > +} PACKED;
> > +
> > +
> > +u64 kvm_tsc_khz(void)
> > +{
> > +    u32 eax, ebx, ecx, edx, msr;
> > +    struct pvclock_vcpu_time_info time;
> > +    u32 addr = (u32)(&time);
> > +    u64 khz;
> > +
> > +    /* check presence and figure msr number */
> > +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
> > +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
> > +        msr = MSR_KVM_SYSTEM_TIME_NEW;
> > +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
> > +        msr = MSR_KVM_SYSTEM_TIME;
> > +    } else {
> > +        return 0;
> > +    }
> > +
> > +    /* ask kvm hypervisor to fill struct */
> > +    memset(&time, 0, sizeof(time));
> > +    wrmsr(msr, addr | 1);
> 
> How can this work?  There is a 64-byte alignment requirement.
> 
> > +    wrmsr(msr, 0);
> > +    if (time.version < 2 || time.tsc_to_system_mul == 0)
> > +        return 0;
> > +
> > +    /* go figure tsc frequency */
> > +    khz = pvclock_tsc_khz(&time);
> > +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
> > +            msr, (u32)khz / 1000);
> > +    return khz;
> 
> That's a meaningless number.  You can be migrated to a cpu or a machine
> with very different tsc.

Thats why there exists hardware tsc frequency scaling and the software
equivalent for that on kvm.

> You want accurate time on kvm, don't use the tsc.
> 
> 
> -- 
> error compiling committee.c: too many arguments to function
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:20         ` Avi Kivity
@ 2012-08-09 19:02           ` Marcelo Tosatti
  2012-08-12 10:56             ` Avi Kivity
  0 siblings, 1 reply; 23+ messages in thread
From: Marcelo Tosatti @ 2012-08-09 19:02 UTC (permalink / raw)
  To: Avi Kivity; +Cc: seabios, Gerd Hoffmann, kvm

On Thu, Aug 09, 2012 at 05:20:11PM +0300, Avi Kivity wrote:
> On 08/09/2012 05:18 PM, Gerd Hoffmann wrote:
> >   Hi,
> > 
> >>> So what do you suggest?  The options I see are:
> >>>
> >>>   (1) Use this patch (with alignment issue fixed of course).
> >>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
> >>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
> >>>       have no TSC.  We could do that too in case we detect kvm ...
> >> 
> >> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
> > 
> > All sorts of timeouts, from a few miliseconds to seconds.
> > 
> > The problematic ones are the longer timeouts, which wait for I/O stuff
> > like disk reads complete.  The stuff with smaller timeouts (like waiting
> > for AHCI link become ready) tend to finish instantly in kvm.
> 
> That's not guaranteed.  The AHCI adapter might be real hardware.  Or the
> emulation may change.
> 
> What's wrong with having a full kvmclock implementation?  Instead of
> issuing rdtsc call a function pointer.

Its not necessary (someone is going to maintain the kvmclock frequency
retrieve, which patch is already here, versus maintainance of 
full kvmclock).

Frequency scaling (or the software equivalent: TSC trapping) are
required for other reasons anyway.

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 14:01     ` Avi Kivity
  2012-08-09 14:05       ` Avi Kivity
  2012-08-09 14:18       ` Gerd Hoffmann
@ 2012-08-09 19:09       ` Marcelo Tosatti
  2012-08-10  7:18         ` Gleb Natapov
  2012-08-10  8:10         ` Gerd Hoffmann
  2 siblings, 2 replies; 23+ messages in thread
From: Marcelo Tosatti @ 2012-08-09 19:09 UTC (permalink / raw)
  To: Avi Kivity; +Cc: Gerd Hoffmann, seabios, kvm

On Thu, Aug 09, 2012 at 05:01:34PM +0300, Avi Kivity wrote:
> On 08/09/2012 04:57 PM, Gerd Hoffmann wrote:
> >   Hi,
> > 
> >>> +u64 kvm_tsc_khz(void)
> >>> +{
> >>> +    u32 eax, ebx, ecx, edx, msr;
> >>> +    struct pvclock_vcpu_time_info time;
> >>> +    u32 addr = (u32)(&time);
> >>> +    u64 khz;
> >>> +
> >>> +    /* check presence and figure msr number */
> >>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
> >>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
> >>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
> >>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
> >>> +        msr = MSR_KVM_SYSTEM_TIME;
> >>> +    } else {
> >>> +        return 0;
> >>> +    }
> >>> +
> >>> +    /* ask kvm hypervisor to fill struct */
> >>> +    memset(&time, 0, sizeof(time));
> >>> +    wrmsr(msr, addr | 1);
> >> 
> >> How can this work?
> > 
> > It did in my testing, although maybe by pure luck ...
> > 
> >> There is a 64-byte alignment requirement.
> > 
> > 64 bytes?  Sure?  The whole struct is only 32 bytes in size ...
> 
> er, the documentation says 4 bytes (so stack alignment works).  I
> distinctly remember having a large alignment requirement so we don't
> cross a page or slot boundary... something's wrong here.
> 
> > 
> > Easily fixable though, just need to grab some memory with memalign
> > instead of using the stack.
> 
> > 
> >>> +    wrmsr(msr, 0);
> >>> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
> >>> +        return 0;
> >>> +
> >>> +    /* go figure tsc frequency */
> >>> +    khz = pvclock_tsc_khz(&time);
> >>> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
> >>> +            msr, (u32)khz / 1000);
> >>> +    return khz;
> >> 
> >> That's a meaningless number.  You can be migrated to a cpu or a machine
> >> with very different tsc.
> > 
> >> You want accurate time on kvm, don't use the tsc.
> > 
> > seabios uses the tsc for timeout calculations only, so it doesn't need
> > to be 100% accurate.  The order of magnitude should be correct though.
> > The Linux kernel uses the value for delay loops too, so using it for the
> > given purpose can't be *that* horrible after all ...
> > 
> > It is certainly an improvement over the current code which tries to
> > calibrate the tsc and gets totally broken results in case the busy host
> > happens to schedule the guest in the middle of calibration.
> > 
> > So what do you suggest?  The options I see are:
> > 
> >   (1) Use this patch (with alignment issue fixed of course).
> >   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
> >   (3) SeaBIOS can fallback to the PIT for timing on machines which
> >       have no TSC.  We could do that too in case we detect kvm ...
> 
> What sort of timeouts are these?  If seconds, maybe the rtc would be best.

I vote for 3 so nobody has to maintain kvmclock code in SeaBIOS and Gerd
can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.


^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 19:09       ` Marcelo Tosatti
@ 2012-08-10  7:18         ` Gleb Natapov
  2012-08-10  7:30           ` Gleb Natapov
  2012-08-10  8:10         ` Gerd Hoffmann
  1 sibling, 1 reply; 23+ messages in thread
From: Gleb Natapov @ 2012-08-10  7:18 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: seabios, Avi Kivity, kvm, Gerd Hoffmann

On Thu, Aug 09, 2012 at 04:09:13PM -0300, Marcelo Tosatti wrote:
> On Thu, Aug 09, 2012 at 05:01:34PM +0300, Avi Kivity wrote:
> > On 08/09/2012 04:57 PM, Gerd Hoffmann wrote:
> > >   Hi,
> > > 
> > >>> +u64 kvm_tsc_khz(void)
> > >>> +{
> > >>> +    u32 eax, ebx, ecx, edx, msr;
> > >>> +    struct pvclock_vcpu_time_info time;
> > >>> +    u32 addr = (u32)(&time);
> > >>> +    u64 khz;
> > >>> +
> > >>> +    /* check presence and figure msr number */
> > >>> +    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
> > >>> +    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
> > >>> +        msr = MSR_KVM_SYSTEM_TIME_NEW;
> > >>> +    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
> > >>> +        msr = MSR_KVM_SYSTEM_TIME;
> > >>> +    } else {
> > >>> +        return 0;
> > >>> +    }
> > >>> +
> > >>> +    /* ask kvm hypervisor to fill struct */
> > >>> +    memset(&time, 0, sizeof(time));
> > >>> +    wrmsr(msr, addr | 1);
> > >> 
> > >> How can this work?
> > > 
> > > It did in my testing, although maybe by pure luck ...
> > > 
> > >> There is a 64-byte alignment requirement.
> > > 
> > > 64 bytes?  Sure?  The whole struct is only 32 bytes in size ...
> > 
> > er, the documentation says 4 bytes (so stack alignment works).  I
> > distinctly remember having a large alignment requirement so we don't
> > cross a page or slot boundary... something's wrong here.
> > 
> > > 
> > > Easily fixable though, just need to grab some memory with memalign
> > > instead of using the stack.
> > 
> > > 
> > >>> +    wrmsr(msr, 0);
> > >>> +    if (time.version < 2 || time.tsc_to_system_mul == 0)
> > >>> +        return 0;
> > >>> +
> > >>> +    /* go figure tsc frequency */
> > >>> +    khz = pvclock_tsc_khz(&time);
> > >>> +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
> > >>> +            msr, (u32)khz / 1000);
> > >>> +    return khz;
> > >> 
> > >> That's a meaningless number.  You can be migrated to a cpu or a machine
> > >> with very different tsc.
> > > 
> > >> You want accurate time on kvm, don't use the tsc.
> > > 
> > > seabios uses the tsc for timeout calculations only, so it doesn't need
> > > to be 100% accurate.  The order of magnitude should be correct though.
> > > The Linux kernel uses the value for delay loops too, so using it for the
> > > given purpose can't be *that* horrible after all ...
> > > 
> > > It is certainly an improvement over the current code which tries to
> > > calibrate the tsc and gets totally broken results in case the busy host
> > > happens to schedule the guest in the middle of calibration.
> > > 
> > > So what do you suggest?  The options I see are:
> > > 
> > >   (1) Use this patch (with alignment issue fixed of course).
> > >   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
> > >   (3) SeaBIOS can fallback to the PIT for timing on machines which
> > >       have no TSC.  We could do that too in case we detect kvm ...
> > 
> > What sort of timeouts are these?  If seconds, maybe the rtc would be best.
> 
> I vote for 3 so nobody has to maintain kvmclock code in SeaBIOS and Gerd
That or pm timer.

> can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.
> 
What message exactly?

--
			Gleb.

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-10  7:18         ` Gleb Natapov
@ 2012-08-10  7:30           ` Gleb Natapov
  0 siblings, 0 replies; 23+ messages in thread
From: Gleb Natapov @ 2012-08-10  7:30 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: Avi Kivity, Gerd Hoffmann, seabios, kvm

On Fri, Aug 10, 2012 at 10:18:00AM +0300, Gleb Natapov wrote:
> > can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.
> > 
> What message exactly?
> 
found it.

--
			Gleb.

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 19:09       ` Marcelo Tosatti
  2012-08-10  7:18         ` Gleb Natapov
@ 2012-08-10  8:10         ` Gerd Hoffmann
  2012-08-10 21:26           ` Marcelo Tosatti
  2012-08-12  9:00           ` Avi Kivity
  1 sibling, 2 replies; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-10  8:10 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: seabios, Avi Kivity, kvm

[-- Attachment #1: Type: text/plain, Size: 975 bytes --]

  Hi,

>>>   (1) Use this patch (with alignment issue fixed of course).
>>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>>>       have no TSC.  We could do that too in case we detect kvm ...
>>
>> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
> 
> I vote for 3 so nobody has to maintain kvmclock code in SeaBIOS and Gerd
> can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.

(2) turned out to be not too bad when taking a shortcut: Go through an
enable/disable cycle each time we read the clock, then just grab
system_time.  Not that efficient, but should be ok for seabios.  Usually
it checks the clock when sitting around idle, waiting for something to
happen.  And it simplifies the implementation alot as we can just skip
all the tsc frequency & delta calculations.

Draft patch attached.  Comments?

cheers,
  Gerd

[-- Attachment #2: 0001-kvmclock-clocksource.patch --]
[-- Type: text/plain, Size: 4934 bytes --]

>From e42d62e90ae4b8a00413a0665d4022069154a516 Mon Sep 17 00:00:00 2001
From: Gerd Hoffmann <kraxel@redhat.com>
Date: Thu, 9 Aug 2012 13:26:18 +0200
Subject: [PATCH] kvmclock clocksource

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 Makefile       |    4 +-
 src/clock.c    |   13 +++++++++++
 src/paravirt.c |   65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/paravirt.h |    3 ++
 4 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 72ee152..b692a96 100644
--- a/Makefile
+++ b/Makefile
@@ -13,11 +13,11 @@ SRCBOTH=misc.c stacks.c pmm.c output.c util.c block.c floppy.c ata.c mouse.c \
     pnpbios.c pirtable.c vgahooks.c ramdisk.c pcibios.c blockcmd.c \
     usb.c usb-uhci.c usb-ohci.c usb-ehci.c usb-hid.c usb-msc.c \
     virtio-ring.c virtio-pci.c virtio-blk.c virtio-scsi.c apm.c ahci.c \
-    usb-uas.c lsi-scsi.c esp-scsi.c
+    usb-uas.c lsi-scsi.c esp-scsi.c paravirt.c
 SRC16=$(SRCBOTH) system.c disk.c font.c
 SRC32FLAT=$(SRCBOTH) post.c shadow.c memmap.c coreboot.c boot.c \
     acpi.c smm.c mptable.c smbios.c pciinit.c optionroms.c mtrr.c \
-    lzmadecode.c bootsplash.c jpeg.c usb-hub.c paravirt.c \
+    lzmadecode.c bootsplash.c jpeg.c usb-hub.c \
     biostables.c xen.c bmp.c romfile.c
 SRC32SEG=util.c output.c pci.c pcibios.c apm.c stacks.c
 
diff --git a/src/clock.c b/src/clock.c
index 69e9f17..15921fa 100644
--- a/src/clock.c
+++ b/src/clock.c
@@ -13,6 +13,7 @@
 #include "bregs.h" // struct bregs
 #include "biosvar.h" // GET_GLOBAL
 #include "usb-hid.h" // usb_check_event
+#include "paravirt.h" // kvm clock
 
 // RTC register flags
 #define RTC_A_UIP 0x80
@@ -64,6 +65,7 @@
 
 u32 cpu_khz VAR16VISIBLE;
 u8 no_tsc VAR16VISIBLE;
+u8 use_kvmclock VAR16VISIBLE;
 
 static void
 calibrate_tsc(void)
@@ -80,6 +82,15 @@ calibrate_tsc(void)
         return;
     }
 
+    if (kvm_para_available()) {
+        u32 hz = kvmclock_init();
+        if (hz != 0) {
+            SET_GLOBAL(use_kvmclock, 1);
+            SET_GLOBAL(cpu_khz, hz / 1000);
+            return;
+        }
+    }
+
     // Setup "timer2"
     u8 orig = inb(PORT_PS2_CTRLB);
     outb((orig & ~PPCB_SPKR) | PPCB_T2GATE, PORT_PS2_CTRLB);
@@ -134,6 +145,8 @@ get_tsc(void)
 {
     if (unlikely(GET_GLOBAL(no_tsc)))
         return emulate_tsc();
+    if (unlikely(GET_GLOBAL(use_kvmclock)))
+        return kvmclock_get();
     return rdtscll();
 }
 
diff --git a/src/paravirt.c b/src/paravirt.c
index 2a98d53..07aa926 100644
--- a/src/paravirt.c
+++ b/src/paravirt.c
@@ -12,6 +12,7 @@
 #include "ioport.h" // outw
 #include "paravirt.h" // qemu_cfg_port_probe
 #include "smbios.h" // struct smbios_structure_header
+#include "biosvar.h" // GET_GLOBAL
 
 int qemu_cfg_present;
 
@@ -346,3 +347,67 @@ void qemu_cfg_romfile_setup(void)
         dprintf(3, "Found fw_cfg file: %s (size=%d)\n", file->name, file->size);
     }
 }
+
+#define KVM_CPUID_SIGNATURE       0x40000000
+#define KVM_CPUID_FEATURES        0x40000001
+#define KVM_FEATURE_CLOCKSOURCE            0
+#define KVM_FEATURE_CLOCKSOURCE2           3
+#define MSR_KVM_SYSTEM_TIME             0x12
+#define MSR_KVM_SYSTEM_TIME_NEW   0x4b564d01
+
+struct pvclock_vcpu_time_info {
+	u32   version;
+	u32   pad0;
+	u64   tsc_timestamp;
+	u64   system_time;
+	u32   tsc_to_system_mul;
+	s8    tsc_shift;
+	u8    flags;
+	u8    pad[2];
+} PACKED;
+
+/* kvmclock system time runs with nanoseconds */
+#define KVM_SYSTIME_HZ   1000000000
+
+u32 kvm_systime_msr VAR16VISIBLE;
+
+static void kvmclock_fetch(struct pvclock_vcpu_time_info *time)
+{
+    u32 addr = (u32)MAKE_FLATPTR(GET_SEG(SS), time);
+    u32 msr = GET_GLOBAL(kvm_systime_msr);
+
+    memset(time, 0, sizeof(*time));
+    wrmsr(msr, addr | 1);
+    wrmsr(msr, 0);
+}
+
+u64 kvmclock_get(void)
+{
+    struct pvclock_vcpu_time_info time;
+
+    kvmclock_fetch(&time);
+    return time.system_time;
+}
+
+u32 kvmclock_init(void)
+{
+    u32 eax, ebx, ecx, edx;
+    struct pvclock_vcpu_time_info time;
+
+    cpuid(KVM_CPUID_FEATURES, &eax, &ebx, &ecx, &edx);
+    if (eax & KVM_FEATURE_CLOCKSOURCE2) {
+        SET_GLOBAL(kvm_systime_msr, MSR_KVM_SYSTEM_TIME_NEW);
+    } else if (eax & KVM_FEATURE_CLOCKSOURCE) {
+        SET_GLOBAL(kvm_systime_msr, MSR_KVM_SYSTEM_TIME);
+    } else {
+        return 0;
+    }
+
+    kvmclock_fetch(&time);
+    if (time.version < 2 || time.tsc_to_system_mul == 0)
+        return 0;
+
+    dprintf(1, "Using kvmclock, msr 0x%x\n",
+            GET_GLOBAL(kvm_systime_msr));
+    return KVM_SYSTIME_HZ;
+}
diff --git a/src/paravirt.h b/src/paravirt.h
index a284c41..64ed3d8 100644
--- a/src/paravirt.h
+++ b/src/paravirt.h
@@ -28,6 +28,9 @@ static inline int kvm_para_available(void)
     return 0;
 }
 
+extern u64 kvmclock_get(void);
+extern u32 kvmclock_init(void);
+
 #define QEMU_CFG_SIGNATURE              0x00
 #define QEMU_CFG_ID                     0x01
 #define QEMU_CFG_UUID                   0x02
-- 
1.7.1


[-- Attachment #3: Type: text/plain, Size: 137 bytes --]

_______________________________________________
SeaBIOS mailing list
SeaBIOS@seabios.org
http://www.seabios.org/mailman/listinfo/seabios

^ permalink raw reply related	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-10  8:10         ` Gerd Hoffmann
@ 2012-08-10 21:26           ` Marcelo Tosatti
  2012-08-13 10:37             ` Gerd Hoffmann
  2012-08-12  9:00           ` Avi Kivity
  1 sibling, 1 reply; 23+ messages in thread
From: Marcelo Tosatti @ 2012-08-10 21:26 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: seabios, Avi Kivity, kvm

On Fri, Aug 10, 2012 at 10:10:27AM +0200, Gerd Hoffmann wrote:
>   Hi,
> 
> >>>   (1) Use this patch (with alignment issue fixed of course).
> >>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
> >>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
> >>>       have no TSC.  We could do that too in case we detect kvm ...
> >>
> >> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
> > 
> > I vote for 3 so nobody has to maintain kvmclock code in SeaBIOS and Gerd
> > can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.
> 
> (2) turned out to be not too bad when taking a shortcut: Go through an
> enable/disable cycle each time we read the clock, then just grab
> system_time.  Not that efficient, but should be ok for seabios.  Usually
> it checks the clock when sitting around idle, waiting for something to
> happen.  And it simplifies the implementation alot as we can just skip
> all the tsc frequency & delta calculations.
> 
> Draft patch attached.  Comments?

Given the history of problems with kvmclock, would rather see it not
being used for delays, if possible. Your shortcut gets rid of a class of
problems, but there might be others (...).

Isnt pmtimer ioport usable? 14MHz.

Error handling in kvmclock_init is awkward.

Thanks

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-10  8:10         ` Gerd Hoffmann
  2012-08-10 21:26           ` Marcelo Tosatti
@ 2012-08-12  9:00           ` Avi Kivity
  1 sibling, 0 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-12  9:00 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: Marcelo Tosatti, seabios, kvm

On 08/10/2012 11:10 AM, Gerd Hoffmann wrote:
>   Hi,
> 
>>>> >>>   (1) Use this patch (with alignment issue fixed of course).
>>>> >>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>>>> >>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>>>> >>>       have no TSC.  We could do that too in case we detect kvm ...
>>> >>
>>> >> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
>> > 
>> > I vote for 3 so nobody has to maintain kvmclock code in SeaBIOS and Gerd
>> > can fix the in-kernel PIT issues with GRUB (see Michaels message) while testing.
> (2) turned out to be not too bad when taking a shortcut: Go through an
> enable/disable cycle each time we read the clock, then just grab
> system_time.  Not that efficient, but should be ok for seabios.  Usually
> it checks the clock when sitting around idle, waiting for something to
> happen.  And it simplifies the implementation alot as we can just skip
> all the tsc frequency & delta calculations.
> 
> Draft patch attached.  Comments?
> 
> +
> +static void kvmclock_fetch(struct pvclock_vcpu_time_info *time)
> +{
> +    u32 addr = (u32)MAKE_FLATPTR(GET_SEG(SS), time);
> +    u32 msr = GET_GLOBAL(kvm_systime_msr);
> +
> +    memset(time, 0, sizeof(*time));
> +    wrmsr(msr, addr | 1);

I'd put the time calculations in here.  We don't specify what happens to
the data area after disabling kvmclock; it could be in the middle of an
update.

> +    wrmsr(msr, 0);
> +}
> +
> +u64 kvmclock_get(void)
> +{
> +    struct pvclock_vcpu_time_info time;
> +
> +    kvmclock_fetch(&time);
> +    return time.system_time;

That's just a random number.  You have to do the full calculation.


-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 18:59   ` Marcelo Tosatti
@ 2012-08-12  9:01     ` Avi Kivity
  0 siblings, 0 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-12  9:01 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: Gerd Hoffmann, seabios, kvm

On 08/09/2012 09:59 PM, Marcelo Tosatti wrote:
>> 
>> > +    wrmsr(msr, 0);
>> > +    if (time.version < 2 || time.tsc_to_system_mul == 0)
>> > +        return 0;
>> > +
>> > +    /* go figure tsc frequency */
>> > +    khz = pvclock_tsc_khz(&time);
>> > +    dprintf(1, "Using kvmclock, msr 0x%x, tsc %d MHz\n",
>> > +            msr, (u32)khz / 1000);
>> > +    return khz;
>> 
>> That's a meaningless number.  You can be migrated to a cpu or a machine
>> with very different tsc.
> 
> Thats why there exists hardware tsc frequency scaling and the software
> equivalent for that on kvm.
> 

The hardware is only available on a minority of processors in the field.
 The software equivalent isn't there.

-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-09 19:02           ` Marcelo Tosatti
@ 2012-08-12 10:56             ` Avi Kivity
  0 siblings, 0 replies; 23+ messages in thread
From: Avi Kivity @ 2012-08-12 10:56 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: Gerd Hoffmann, seabios, kvm

On 08/09/2012 10:02 PM, Marcelo Tosatti wrote:
> On Thu, Aug 09, 2012 at 05:20:11PM +0300, Avi Kivity wrote:
>> On 08/09/2012 05:18 PM, Gerd Hoffmann wrote:
>> >   Hi,
>> > 
>> >>> So what do you suggest?  The options I see are:
>> >>>
>> >>>   (1) Use this patch (with alignment issue fixed of course).
>> >>>   (2) Do a full kvmclock implementation.  Feels a bit like overkill.
>> >>>   (3) SeaBIOS can fallback to the PIT for timing on machines which
>> >>>       have no TSC.  We could do that too in case we detect kvm ...
>> >> 
>> >> What sort of timeouts are these?  If seconds, maybe the rtc would be best.
>> > 
>> > All sorts of timeouts, from a few miliseconds to seconds.
>> > 
>> > The problematic ones are the longer timeouts, which wait for I/O stuff
>> > like disk reads complete.  The stuff with smaller timeouts (like waiting
>> > for AHCI link become ready) tend to finish instantly in kvm.
>> 
>> That's not guaranteed.  The AHCI adapter might be real hardware.  Or the
>> emulation may change.
>> 
>> What's wrong with having a full kvmclock implementation?  Instead of
>> issuing rdtsc call a function pointer.
> 
> Its not necessary (someone is going to maintain the kvmclock frequency
> retrieve, which patch is already here, versus maintainance of 
> full kvmclock).

The frequency is meaninless.

> 
> Frequency scaling (or the software equivalent: TSC trapping) are
> required for other reasons anyway.

One thing we can do is enable TSC trapping, then disable it if the guest
activates kvmclock.  That gives us accurate time either way.


-- 
error compiling committee.c: too many arguments to function

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-10 21:26           ` Marcelo Tosatti
@ 2012-08-13 10:37             ` Gerd Hoffmann
  2012-08-13 10:46               ` Gleb Natapov
  0 siblings, 1 reply; 23+ messages in thread
From: Gerd Hoffmann @ 2012-08-13 10:37 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: Avi Kivity, seabios, kvm

  Hi,

> Isnt pmtimer ioport usable? 14MHz.

Can give it a try.  14 MHz looks wrong though, apci.h says:

/* PM Timer ticks per second (HZ) */
#define PM_TIMER_FREQUENCY  3579545

Is this fixed?  Or hardware specific?

cheers,
  Gerd


^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [PATCH] tsc: use kvmclock for calibration
  2012-08-13 10:37             ` Gerd Hoffmann
@ 2012-08-13 10:46               ` Gleb Natapov
  2012-08-13 12:55                 ` [SeaBIOS] " Fred .
  0 siblings, 1 reply; 23+ messages in thread
From: Gleb Natapov @ 2012-08-13 10:46 UTC (permalink / raw)
  To: Gerd Hoffmann; +Cc: Marcelo Tosatti, Avi Kivity, seabios, kvm

On Mon, Aug 13, 2012 at 12:37:11PM +0200, Gerd Hoffmann wrote:
>   Hi,
> 
> > Isnt pmtimer ioport usable? 14MHz.
> 
> Can give it a try.  14 MHz looks wrong though, apci.h says:
> 
> /* PM Timer ticks per second (HZ) */
> #define PM_TIMER_FREQUENCY  3579545
> 
> Is this fixed?  Or hardware specific?
> 
3.579545 MHz clock required by ACPI spec.

--
			Gleb.

^ permalink raw reply	[flat|nested] 23+ messages in thread

* Re: [SeaBIOS] [PATCH] tsc: use kvmclock for calibration
  2012-08-13 10:46               ` Gleb Natapov
@ 2012-08-13 12:55                 ` Fred .
  0 siblings, 0 replies; 23+ messages in thread
From: Fred . @ 2012-08-13 12:55 UTC (permalink / raw)
  To: Gleb Natapov; +Cc: Gerd Hoffmann, seabios, Avi Kivity, kvm

Add a comment about it in the source code.

-#define PM_TIMER_FREQUENCY  3579545
+#define PM_TIMER_FREQUENCY  3579545 // 3.579545 MHz clock required by
ACPI spec.

On Mon, Aug 13, 2012 at 12:46 PM, Gleb Natapov <gleb@redhat.com> wrote:
> On Mon, Aug 13, 2012 at 12:37:11PM +0200, Gerd Hoffmann wrote:
>>   Hi,
>>
>> > Isnt pmtimer ioport usable? 14MHz.
>>
>> Can give it a try.  14 MHz looks wrong though, apci.h says:
>>
>> /* PM Timer ticks per second (HZ) */
>> #define PM_TIMER_FREQUENCY  3579545
>>
>> Is this fixed?  Or hardware specific?
>>
> 3.579545 MHz clock required by ACPI spec.
>
> --
>                         Gleb.
>
> _______________________________________________
> SeaBIOS mailing list
> SeaBIOS@seabios.org
> http://www.seabios.org/mailman/listinfo/seabios

^ permalink raw reply	[flat|nested] 23+ messages in thread

end of thread, other threads:[~2012-08-13 12:55 UTC | newest]

Thread overview: 23+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2012-08-09 11:57 [PATCH] tsc: use kvmclock for calibration Gerd Hoffmann
2012-08-09 12:53 ` Avi Kivity
2012-08-09 13:25   ` [SeaBIOS] " Fred .
2012-08-09 13:57   ` Gerd Hoffmann
2012-08-09 14:01     ` Avi Kivity
2012-08-09 14:05       ` Avi Kivity
2012-08-09 14:12         ` Gerd Hoffmann
2012-08-09 14:17           ` Avi Kivity
2012-08-09 14:18       ` Gerd Hoffmann
2012-08-09 14:20         ` Avi Kivity
2012-08-09 19:02           ` Marcelo Tosatti
2012-08-12 10:56             ` Avi Kivity
2012-08-09 19:09       ` Marcelo Tosatti
2012-08-10  7:18         ` Gleb Natapov
2012-08-10  7:30           ` Gleb Natapov
2012-08-10  8:10         ` Gerd Hoffmann
2012-08-10 21:26           ` Marcelo Tosatti
2012-08-13 10:37             ` Gerd Hoffmann
2012-08-13 10:46               ` Gleb Natapov
2012-08-13 12:55                 ` [SeaBIOS] " Fred .
2012-08-12  9:00           ` Avi Kivity
2012-08-09 18:59   ` Marcelo Tosatti
2012-08-12  9:01     ` Avi Kivity

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.