[RFC PATCH v4 0/2] Hyper-V timers

[RFC PATCH v4 0/2] Hyper-V timers

[Vadim Rozenfeld]

This RFC series adds support for two Hyper-V timer services -
a per-partition reference time counter, and a  partition reference
time enlightenment.


Vadim Rozenfeld (2):
  add support for Hyper-V reference time counter
  add support for Hyper-V partition reference time enlightenment

 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/include/uapi/asm/hyperv.h | 13 ++++++++
 arch/x86/kvm/x86.c                 | 61 +++++++++++++++++++++++++++++++++++++-
 include/uapi/linux/kvm.h           |  1 +
 4 files changed, 75 insertions(+), 1 deletion(-)

-- 
1.8.1.4

[RFC PATCH v4 1/2] add support for Hyper-V reference time counter

[Vadim Rozenfeld]

Signed-off: Peter Lieven <pl@dlh.net>
Signed-off: Gleb Natapov
Signed-off: Vadim Rozenfeld <vrozenfe@redhat.com>

v1 -> v2
1. mark TSC page dirty as suggested by 
    Eric Northup <digitaleric@google.com> and Gleb
2. disable local irq when calling get_kernel_ns, 
    as it was done by Peter Lieven <pl@kamp.de>
3. move check for TSC page enable from second patch
    to this one.

v3 -> v4
    Get rid of ref counter offset. 

---
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/include/uapi/asm/hyperv.h | 13 +++++++++++++
 arch/x86/kvm/x86.c                 | 32 +++++++++++++++++++++++++++++++-
 include/uapi/linux/kvm.h           |  1 +
 4 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index ae5d783..33fef07 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -605,6 +605,7 @@ struct kvm_arch {
 	/* fields used by HYPER-V emulation */
 	u64 hv_guest_os_id;
 	u64 hv_hypercall;
+	u64 hv_tsc_page;
 
 	#ifdef CONFIG_KVM_MMU_AUDIT
 	int audit_point;
diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h
index b8f1c01..462efe7 100644
--- a/arch/x86/include/uapi/asm/hyperv.h
+++ b/arch/x86/include/uapi/asm/hyperv.h
@@ -28,6 +28,9 @@
 /* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
 #define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
 
+/* A partition's reference time stamp counter (TSC) page */
+#define HV_X64_MSR_REFERENCE_TSC		0x40000021
+
 /*
  * There is a single feature flag that signifies the presence of the MSR
  * that can be used to retrieve both the local APIC Timer frequency as
@@ -198,6 +201,9 @@
 #define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
 		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
 
+#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
+#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
+
 #define HV_PROCESSOR_POWER_STATE_C0		0
 #define HV_PROCESSOR_POWER_STATE_C1		1
 #define HV_PROCESSOR_POWER_STATE_C2		2
@@ -210,4 +216,11 @@
 #define HV_STATUS_INVALID_ALIGNMENT		4
 #define HV_STATUS_INSUFFICIENT_BUFFERS		19
 
+typedef struct _HV_REFERENCE_TSC_PAGE {
+	__u32 tsc_sequence;
+	__u32 res1;
+	__u64 tsc_scale;
+	__s64 tsc_offset;
+} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
+
 #endif
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 5d004da..496bdb1 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -836,11 +836,12 @@ EXPORT_SYMBOL_GPL(kvm_rdpmc);
  * kvm-specific. Those are put in the beginning of the list.
  */
 
-#define KVM_SAVE_MSRS_BEGIN	10
+#define KVM_SAVE_MSRS_BEGIN	12
 static u32 msrs_to_save[] = {
 	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
 	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
 	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
+	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
 	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
 	MSR_KVM_PV_EOI_EN,
 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
@@ -1826,6 +1827,8 @@ static bool kvm_hv_msr_partition_wide(u32 msr)
 	switch (msr) {
 	case HV_X64_MSR_GUEST_OS_ID:
 	case HV_X64_MSR_HYPERCALL:
+	case HV_X64_MSR_REFERENCE_TSC:
+	case HV_X64_MSR_TIME_REF_COUNT:
 		r = true;
 		break;
 	}
@@ -1867,6 +1870,25 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 		kvm->arch.hv_hypercall = data;
 		break;
 	}
+	case HV_X64_MSR_REFERENCE_TSC: {
+		u64 gfn;
+		unsigned long addr;
+		HV_REFERENCE_TSC_PAGE tsc_ref;
+		memset(&tsc_ref, 0, sizeof(tsc_ref));
+		if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE)) {
+			kvm->arch.hv_tsc_page = data;
+			break;
+		}
+		gfn = data >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
+		addr = gfn_to_hva(kvm, gfn);
+		if (kvm_is_error_hva(addr))
+			return 1;
+		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
+			return 1;
+		mark_page_dirty(kvm, gfn);
+		kvm->arch.hv_tsc_page = data;
+		break;
+	}
 	default:
 		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
 			    "data 0x%llx\n", msr, data);
@@ -2291,6 +2313,13 @@ static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 	case HV_X64_MSR_HYPERCALL:
 		data = kvm->arch.hv_hypercall;
 		break;
+	case HV_X64_MSR_TIME_REF_COUNT: {
+		data = div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
+		break;
+	}
+	case HV_X64_MSR_REFERENCE_TSC:
+		data = kvm->arch.hv_tsc_page;
+		break;
 	default:
 		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
 		return 1;
@@ -2604,6 +2633,7 @@ int kvm_dev_ioctl_check_extension(long ext)
 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
 	case KVM_CAP_ASSIGN_DEV_IRQ:
 	case KVM_CAP_PCI_2_3:
+	case KVM_CAP_HYPERV_TIME:
 #endif
 		r = 1;
 		break;
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 902f124..686c1ca 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -674,6 +674,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_ARM_EL1_32BIT 93
 #define KVM_CAP_SPAPR_MULTITCE 94
 #define KVM_CAP_EXT_EMUL_CPUID 95
+#define KVM_CAP_HYPERV_TIME 96
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
-- 
1.8.1.4

[RFC PATCH v4 2/2] add support for Hyper-V partition reference time enlightenment

[Vadim Rozenfeld]

The following patch allows to activate a partition reference
time enlightenment that is based on the host platform's support
for an Invariant Time Stamp Counter (iTSC).

v2 -> v3
    Handle TSC sequence, scale, and offest changing during migration.

v3 -> v4
    1. Wrap access to iTSC page with kvm_read_guest/kvm_write_guest
    suggested by Andrew Honig <ahonig@google.com> and Marcelo
    2. iTSC data calculation/access fixes suggested by Marcelo and Paolo

---
 arch/x86/kvm/x86.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 496bdb1..6e6debf 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1873,6 +1873,7 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 	case HV_X64_MSR_REFERENCE_TSC: {
 		u64 gfn;
 		unsigned long addr;
+		struct kvm_arch *ka = &kvm->arch;
 		HV_REFERENCE_TSC_PAGE tsc_ref;
 		memset(&tsc_ref, 0, sizeof(tsc_ref));
 		if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE)) {
@@ -1883,6 +1884,13 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 		addr = gfn_to_hva(kvm, gfn);
 		if (kvm_is_error_hva(addr))
 			return 1;
+		if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
+		    && ka->use_master_clock) {
+			tsc_ref.tsc_sequence = 1;
+			tsc_ref.tsc_scale = ((10000LL << 32) /
+					     vcpu->arch.virtual_tsc_khz) << 32;
+			tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
+		}
 		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
 			return 1;
 		mark_page_dirty(kvm, gfn);
@@ -3871,6 +3879,27 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		local_irq_enable();
 		kvm->arch.kvmclock_offset = delta;
 		kvm_gen_update_masterclock(kvm);
+		if (kvm->arch.hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) {
+			HV_REFERENCE_TSC_PAGE tsc_ref;
+			struct kvm_arch *ka = &kvm->arch;
+			r = kvm_read_guest(kvm, kvm->arch.hv_tsc_page,
+					   &tsc_ref, sizeof(tsc_ref));
+			if (r)
+				goto out;
+			if (tsc_ref.tsc_sequence
+			    && boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
+			    && ka->use_master_clock) {
+				tsc_ref.tsc_sequence++;
+				tsc_ref.tsc_scale = ((10000LL << 32) /
+					__get_cpu_var(cpu_tsc_khz)) << 32;
+                        	tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
+			} else
+				tsc_ref.tsc_sequence = 0;
+			r = kvm_write_guest(kvm, kvm->arch.hv_tsc_page,
+					    &tsc_ref, sizeof(tsc_ref));
+			if (r)
+				goto out;
+		}
 		break;
 	}
 	case KVM_GET_CLOCK: {
-- 
1.8.1.4

Re: [RFC PATCH v4 2/2] add support for Hyper-V partition reference time enlightenment

[Paolo Bonzini]

The cleanup between v3 and v4 is really nice; thanks.  What happened to
the glitch you reported in the v3 thread.

Il 14/01/2014 10:22, Vadim Rozenfeld ha scritto:
> +			tsc_ref.tsc_scale = ((10000LL << 32) /
> +					     vcpu->arch.virtual_tsc_khz) << 32;

As mentioned in the review of v3, I don't really like the tsc_scale to
come from two different sources in the two cases of MSR write and
KVM_SET_CLOCK.  This is the only qualm I still have with this patch.
Marcelo, do you have any ideas?

Paolo

> +			tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
> +		}
>  		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
>  			return 1;
>  		mark_page_dirty(kvm, gfn);
> @@ -3871,6 +3879,27 @@ long kvm_arch_vm_ioctl(struct file *filp,
>  		local_irq_enable();
>  		kvm->arch.kvmclock_offset = delta;
>  		kvm_gen_update_masterclock(kvm);
> +		if (kvm->arch.hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) {
> +			HV_REFERENCE_TSC_PAGE tsc_ref;
> +			struct kvm_arch *ka = &kvm->arch;
> +			r = kvm_read_guest(kvm, kvm->arch.hv_tsc_page,
> +					   &tsc_ref, sizeof(tsc_ref));
> +			if (r)
> +				goto out;
> +			if (tsc_ref.tsc_sequence
> +			    && boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
> +			    && ka->use_master_clock) {
> +				tsc_ref.tsc_sequence++;
> +				tsc_ref.tsc_scale = ((10000LL << 32) /
> +					__get_cpu_var(cpu_tsc_khz)) << 32;

Re: [RFC PATCH v4 1/2] add support for Hyper-V reference time counter

[Paolo Bonzini]

Il 14/01/2014 10:22, Vadim Rozenfeld ha scritto:
> Signed-off: Peter Lieven <pl@dlh.net>
> Signed-off: Gleb Natapov
> Signed-off: Vadim Rozenfeld <vrozenfe@redhat.com>
> 
> v1 -> v2
> 1. mark TSC page dirty as suggested by 
>     Eric Northup <digitaleric@google.com> and Gleb
> 2. disable local irq when calling get_kernel_ns, 
>     as it was done by Peter Lieven <pl@kamp.de>
> 3. move check for TSC page enable from second patch
>     to this one.
> 
> v3 -> v4
>     Get rid of ref counter offset. 

One comment below.

> ---
>  arch/x86/include/asm/kvm_host.h    |  1 +
>  arch/x86/include/uapi/asm/hyperv.h | 13 +++++++++++++
>  arch/x86/kvm/x86.c                 | 32 +++++++++++++++++++++++++++++++-
>  include/uapi/linux/kvm.h           |  1 +
>  4 files changed, 46 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index ae5d783..33fef07 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -605,6 +605,7 @@ struct kvm_arch {
>  	/* fields used by HYPER-V emulation */
>  	u64 hv_guest_os_id;
>  	u64 hv_hypercall;
> +	u64 hv_tsc_page;
>  
>  	#ifdef CONFIG_KVM_MMU_AUDIT
>  	int audit_point;
> diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h
> index b8f1c01..462efe7 100644
> --- a/arch/x86/include/uapi/asm/hyperv.h
> +++ b/arch/x86/include/uapi/asm/hyperv.h
> @@ -28,6 +28,9 @@
>  /* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
>  #define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
>  
> +/* A partition's reference time stamp counter (TSC) page */
> +#define HV_X64_MSR_REFERENCE_TSC		0x40000021
> +
>  /*
>   * There is a single feature flag that signifies the presence of the MSR
>   * that can be used to retrieve both the local APIC Timer frequency as
> @@ -198,6 +201,9 @@
>  #define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
>  		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
>  
> +#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
> +#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
> +
>  #define HV_PROCESSOR_POWER_STATE_C0		0
>  #define HV_PROCESSOR_POWER_STATE_C1		1
>  #define HV_PROCESSOR_POWER_STATE_C2		2
> @@ -210,4 +216,11 @@
>  #define HV_STATUS_INVALID_ALIGNMENT		4
>  #define HV_STATUS_INSUFFICIENT_BUFFERS		19
>  
> +typedef struct _HV_REFERENCE_TSC_PAGE {
> +	__u32 tsc_sequence;
> +	__u32 res1;
> +	__u64 tsc_scale;
> +	__s64 tsc_offset;
> +} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
> +
>  #endif
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 5d004da..496bdb1 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -836,11 +836,12 @@ EXPORT_SYMBOL_GPL(kvm_rdpmc);
>   * kvm-specific. Those are put in the beginning of the list.
>   */
>  
> -#define KVM_SAVE_MSRS_BEGIN	10
> +#define KVM_SAVE_MSRS_BEGIN	12
>  static u32 msrs_to_save[] = {
>  	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
>  	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
>  	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
> +	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
>  	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
>  	MSR_KVM_PV_EOI_EN,
>  	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
> @@ -1826,6 +1827,8 @@ static bool kvm_hv_msr_partition_wide(u32 msr)
>  	switch (msr) {
>  	case HV_X64_MSR_GUEST_OS_ID:
>  	case HV_X64_MSR_HYPERCALL:
> +	case HV_X64_MSR_REFERENCE_TSC:
> +	case HV_X64_MSR_TIME_REF_COUNT:
>  		r = true;
>  		break;
>  	}
> @@ -1867,6 +1870,25 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
>  		kvm->arch.hv_hypercall = data;
>  		break;
>  	}
> +	case HV_X64_MSR_REFERENCE_TSC: {
> +		u64 gfn;
> +		unsigned long addr;
> +		HV_REFERENCE_TSC_PAGE tsc_ref;
> +		memset(&tsc_ref, 0, sizeof(tsc_ref));
> +		if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE)) {
> +			kvm->arch.hv_tsc_page = data;
> +			break;
> +		}
> +		gfn = data >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
> +		addr = gfn_to_hva(kvm, gfn);
> +		if (kvm_is_error_hva(addr))
> +			return 1;
> +		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
> +			return 1;

You can use kvm_write_guest here, too.

Otherwise looks good.

Paolo

> +		mark_page_dirty(kvm, gfn);
> +		kvm->arch.hv_tsc_page = data;
> +		break;
> +	}
>  	default:
>  		vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
>  			    "data 0x%llx\n", msr, data);
> @@ -2291,6 +2313,13 @@ static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
>  	case HV_X64_MSR_HYPERCALL:
>  		data = kvm->arch.hv_hypercall;
>  		break;
> +	case HV_X64_MSR_TIME_REF_COUNT: {
> +		data = div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
> +		break;
> +	}
> +	case HV_X64_MSR_REFERENCE_TSC:
> +		data = kvm->arch.hv_tsc_page;
> +		break;
>  	default:
>  		vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
>  		return 1;
> @@ -2604,6 +2633,7 @@ int kvm_dev_ioctl_check_extension(long ext)
>  #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
>  	case KVM_CAP_ASSIGN_DEV_IRQ:
>  	case KVM_CAP_PCI_2_3:
> +	case KVM_CAP_HYPERV_TIME:
>  #endif
>  		r = 1;
>  		break;
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 902f124..686c1ca 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -674,6 +674,7 @@ struct kvm_ppc_smmu_info {
>  #define KVM_CAP_ARM_EL1_32BIT 93
>  #define KVM_CAP_SPAPR_MULTITCE 94
>  #define KVM_CAP_EXT_EMUL_CPUID 95
> +#define KVM_CAP_HYPERV_TIME 96
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> 

Re: [RFC PATCH v4 2/2] add support for Hyper-V partition reference time enlightenment

[Marcelo Tosatti]

On Tue, Jan 14, 2014 at 12:03:24PM +0100, Paolo Bonzini wrote:
> The cleanup between v3 and v4 is really nice; thanks.  What happened to
> the glitch you reported in the v3 thread.
> 
> Il 14/01/2014 10:22, Vadim Rozenfeld ha scritto:
> > +			tsc_ref.tsc_scale = ((10000LL << 32) /
> > +					     vcpu->arch.virtual_tsc_khz) << 32;
> 
> As mentioned in the review of v3, I don't really like the tsc_scale to
> come from two different sources in the two cases of MSR write and
> KVM_SET_CLOCK.  This is the only qualm I still have with this patch.
> Marcelo, do you have any ideas?

It should be at kvm_guest_time_update.

Re: [RFC PATCH v4 2/2] add support for Hyper-V partition reference time enlightenment

[Marcelo Tosatti]

On Tue, Jan 14, 2014 at 08:22:17PM +1100, Vadim Rozenfeld wrote:
> The following patch allows to activate a partition reference
> time enlightenment that is based on the host platform's support
> for an Invariant Time Stamp Counter (iTSC).
> 
> v2 -> v3
>     Handle TSC sequence, scale, and offest changing during migration.
> 
> v3 -> v4
>     1. Wrap access to iTSC page with kvm_read_guest/kvm_write_guest
>     suggested by Andrew Honig <ahonig@google.com> and Marcelo
>     2. iTSC data calculation/access fixes suggested by Marcelo and Paolo
> 
> ---
>  arch/x86/kvm/x86.c | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
> 
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 496bdb1..6e6debf 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -1873,6 +1873,7 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
>  	case HV_X64_MSR_REFERENCE_TSC: {
>  		u64 gfn;
>  		unsigned long addr;
> +		struct kvm_arch *ka = &kvm->arch;
>  		HV_REFERENCE_TSC_PAGE tsc_ref;
>  		memset(&tsc_ref, 0, sizeof(tsc_ref));
>  		if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE)) {
> @@ -1883,6 +1884,13 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
>  		addr = gfn_to_hva(kvm, gfn);
>  		if (kvm_is_error_hva(addr))
>  			return 1;
> +		if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
> +		    && ka->use_master_clock) {
> +			tsc_ref.tsc_sequence = 1;
> +			tsc_ref.tsc_scale = ((10000LL << 32) /
> +					     vcpu->arch.virtual_tsc_khz) << 32;
> +			tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
> +		}
>  		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
>  			return 1;
>  		mark_page_dirty(kvm, gfn);
> @@ -3871,6 +3879,27 @@ long kvm_arch_vm_ioctl(struct file *filp,
>  		local_irq_enable();
>  		kvm->arch.kvmclock_offset = delta;
>  		kvm_gen_update_masterclock(kvm);
> +		if (kvm->arch.hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) {
> +			HV_REFERENCE_TSC_PAGE tsc_ref;
> +			struct kvm_arch *ka = &kvm->arch;
> +			r = kvm_read_guest(kvm, kvm->arch.hv_tsc_page,
> +					   &tsc_ref, sizeof(tsc_ref));
> +			if (r)
> +				goto out;
> +			if (tsc_ref.tsc_sequence
> +			    && boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
> +			    && ka->use_master_clock) {
> +				tsc_ref.tsc_sequence++;
> +				tsc_ref.tsc_scale = ((10000LL << 32) /
> +					__get_cpu_var(cpu_tsc_khz)) << 32;
> +                        	tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
> +			} else
> +				tsc_ref.tsc_sequence = 0;
> +			r = kvm_write_guest(kvm, kvm->arch.hv_tsc_page,
> +					    &tsc_ref, sizeof(tsc_ref));
> +			if (r)
> +				goto out;
> +		}
>  		break;
>  	}

Vadim,

A short comment explaining the lifecycle of the exposed
clock would be nice.

Can you explain why are you setting tsc_offset to kvmclock_offset?

Re: [RFC PATCH v4 2/2] add support for Hyper-V partition reference time enlightenment

[Paolo Bonzini]

Il 14/01/2014 10:22, Vadim Rozenfeld ha scritto:
> @@ -1883,6 +1884,13 @@ static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
>  		addr = gfn_to_hva(kvm, gfn);
>  		if (kvm_is_error_hva(addr))
>  			return 1;
> +		if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
> +		    && ka->use_master_clock) {
> +			tsc_ref.tsc_sequence = 1;

I think that after the first migration the sequence number will always
be 2: set_msr_hyperv_pw will set it to 1, and KVM_SET_KVMCLOCK will
increment it.  Can you check this?

The solution would be to propagate the "struct msr_data *" argument from
kvm_set_msr_common to kvm_set_msr_hyper_pw.  Then, here in this "if" you
can test msr_info->host_initiated, and increment the tsc_sequence
instead of resetting.  Like

	if ((!msr_info->host_initiated || tsc_ref.tsc_sequence) &&
	    boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
	    ka->use_master_clock) {
		tsc_ref.tsc_sequence = msr_info->host_initiated
			? tsc_ref.tsc_sequence + 1
			: 1;

> +			tsc_ref.tsc_scale = ((10000LL << 32) /
> +					     vcpu->arch.virtual_tsc_khz) << 32;
> +			tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
> +		}
>  		if (__copy_to_user((void __user *)addr, &tsc_ref, sizeof(tsc_ref)))
>  			return 1;
>  		mark_page_dirty(kvm, gfn);
> @@ -3871,6 +3879,27 @@ long kvm_arch_vm_ioctl(struct file *filp,
>  		local_irq_enable();
>  		kvm->arch.kvmclock_offset = delta;
>  		kvm_gen_update_masterclock(kvm);
> +		if (kvm->arch.hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE) {
> +			HV_REFERENCE_TSC_PAGE tsc_ref;
> +			struct kvm_arch *ka = &kvm->arch;
> +			r = kvm_read_guest(kvm, kvm->arch.hv_tsc_page,
> +					   &tsc_ref, sizeof(tsc_ref));
> +			if (r)
> +				goto out;
> +			if (tsc_ref.tsc_sequence
> +			    && boot_cpu_has(X86_FEATURE_CONSTANT_TSC)
> +			    && ka->use_master_clock) {
> +				tsc_ref.tsc_sequence++;
> +				tsc_ref.tsc_scale = ((10000LL << 32) /
> +					__get_cpu_var(cpu_tsc_khz)) << 32;

Since on migration kvm_set_hyperv_pw will always be called, do you need
to write tsc_ref.scale at all here?

Paolo

> +                        	tsc_ref.tsc_offset = kvm->arch.kvmclock_offset;
> +			} else
> +				tsc_ref.tsc_sequence = 0;
> +			r = kvm_write_guest(kvm, kvm->arch.hv_tsc_page,
> +					    &tsc_ref, sizeof(tsc_ref));
> +			if (r)
> +				goto out;
> +		}
>  		break;
>  	}
>  	case KVM_GET_CLOCK: {
>