[PATCH 0/2] QEMU kvm: Adding paravirtual spinlock support for x86.

[PATCH 0/2] QEMU kvm: Adding paravirtual spinlock support for x86.

[Raghavendra K T]

The patch, extends KVM-hypervisor and Linux guest running on
KVM-hypervisor to support pv-ticket spinlocks.

PV ticket spinlock helps to solve Lock Holder Preemption problem discussed in
http://www.amd64.org/fileadmin/user_upload/pub/LHP-commented_slides.pdf.

When spinlock is contended,a guest vcpu relinqueshes cpu by halt().
Correspondingly, One hypercall is introduced in KVM hypervisor,that allows
a vcpu to kick the halted vcpu to continue with execution.

Note:
Below patch should be applied only after corresponding
linux-header changes taken into qemu via scripts/update-linux-headers.sh script.
---
Changes in V3:
 rename PVLOCK_KICK --> PV_UNHALT
 add MSR related changes to aid migration

Changes in V2:
 Drop the syncing kernel header changes. (Alex) 
 rename KICK_VCPU --> PVLOCK_KICK.

Raghavendra K T(2):
 add PV_UNHALT feature support.
 add support to get/set vcpu unhalt msr to aid migration

 target-i386/cpu.h |    1 +
 target-i386/kvm.c |   14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletions(-)

V5 Kernel changes:
 https://lkml.org/lkml/2012/3/23/50

V4 kernel changes:
 https://lkml.org/lkml/2012/1/14/66
 Qemu changes for V4:
 http://www.mail-archive.com/kvm@vger.kernel.org/msg66450.html

 V3 kernel Changes:
 https://lkml.org/lkml/2011/11/30/62
 V2 kernel changes :
 https://lkml.org/lkml/2011/10/23/207

 Previous discussions : (posted by Srivatsa V).
 https://lkml.org/lkml/2010/7/26/24
 https://lkml.org/lkml/2011/1/19/212

 Qemu patch for V3:
 http://lists.gnu.org/archive/html/qemu-devel/2011-12/msg00397.html

[Qemu-devel] [PATCH 0/2] QEMU kvm: Adding paravirtual spinlock support for x86.

[Raghavendra K T]

The patch, extends KVM-hypervisor and Linux guest running on
KVM-hypervisor to support pv-ticket spinlocks.

PV ticket spinlock helps to solve Lock Holder Preemption problem discussed in
http://www.amd64.org/fileadmin/user_upload/pub/LHP-commented_slides.pdf.

When spinlock is contended,a guest vcpu relinqueshes cpu by halt().
Correspondingly, One hypercall is introduced in KVM hypervisor,that allows
a vcpu to kick the halted vcpu to continue with execution.

Note:
Below patch should be applied only after corresponding
linux-header changes taken into qemu via scripts/update-linux-headers.sh script.
---
Changes in V3:
 rename PVLOCK_KICK --> PV_UNHALT
 add MSR related changes to aid migration

Changes in V2:
 Drop the syncing kernel header changes. (Alex) 
 rename KICK_VCPU --> PVLOCK_KICK.

Raghavendra K T(2):
 add PV_UNHALT feature support.
 add support to get/set vcpu unhalt msr to aid migration

 target-i386/cpu.h |    1 +
 target-i386/kvm.c |   14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletions(-)

V5 Kernel changes:
 https://lkml.org/lkml/2012/3/23/50

V4 kernel changes:
 https://lkml.org/lkml/2012/1/14/66
 Qemu changes for V4:
 http://www.mail-archive.com/kvm@vger.kernel.org/msg66450.html

 V3 kernel Changes:
 https://lkml.org/lkml/2011/11/30/62
 V2 kernel changes :
 https://lkml.org/lkml/2011/10/23/207

 Previous discussions : (posted by Srivatsa V).
 https://lkml.org/lkml/2010/7/26/24
 https://lkml.org/lkml/2011/1/19/212

 Qemu patch for V3:
 http://lists.gnu.org/archive/html/qemu-devel/2011-12/msg00397.html

[PATCH 1/2] QEMU kvm: Add PV_UNHALT feature support

[Raghavendra K T]

From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>

Extend the KVM Hypervisor to enable PVLOCK_KICK feature that allows
a vcpu to kick the halted vcpu to continue with execution in PV ticket
spinlock.

Signed-off-by: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index e74a9e4..dbebd3a 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -98,6 +98,7 @@ struct kvm_para_features {
     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
+    { KVM_CAP_PV_UNHALT, KVM_FEATURE_PV_UNHALT },
     { -1, -1 }
 };
 

[Qemu-devel] [PATCH 1/2] QEMU kvm: Add PV_UNHALT feature support

[Raghavendra K T]

From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>

Extend the KVM Hypervisor to enable PVLOCK_KICK feature that allows
a vcpu to kick the halted vcpu to continue with execution in PV ticket
spinlock.

Signed-off-by: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index e74a9e4..dbebd3a 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -98,6 +98,7 @@ struct kvm_para_features {
     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
+    { KVM_CAP_PV_UNHALT, KVM_FEATURE_PV_UNHALT },
     { -1, -1 }
 };
 

[PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>

MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
used during migration.

Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index a1ed3e7..10286a5 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -697,6 +697,7 @@ typedef struct CPUX86State {
     uint64_t system_time_msr;
     uint64_t wall_clock_msr;
     uint64_t async_pf_en_msr;
+    uint64_t pv_unhalt_msr;
 
     uint64_t tsc;
     uint64_t tsc_deadline;
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index dbebd3a..98b9088 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -62,6 +62,7 @@ static bool has_msr_star;
 static bool has_msr_hsave_pa;
 static bool has_msr_tsc_deadline;
 static bool has_msr_async_pf_en;
+static bool has_msr_pv_unhalt;
 static bool has_msr_misc_enable;
 static int lm_capable_kernel;
 
@@ -444,7 +445,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     }
 
     has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
-
+    has_msr_pv_unhalt = c->eax & (1 << KVM_FEATURE_PV_UNHALT);
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0; i <= limit; i++) {
@@ -1012,6 +1013,10 @@ static int kvm_put_msrs(CPUX86State *env, int level)
         if (hyperv_vapic_recommended()) {
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
         }
+        if (has_msr_pv_unhalt) {
+            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_UNHALT,
+                              env->pv_unhalt_msr);
+        }
     }
     if (env->mcg_cap) {
         int i;
@@ -1247,6 +1252,9 @@ static int kvm_get_msrs(CPUX86State *env)
     if (has_msr_async_pf_en) {
         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
     }
+    if (has_msr_pv_unhalt) {
+        msrs[n++].index = MSR_KVM_PV_UNHALT;
+    }
 
     if (env->mcg_cap) {
         msrs[n++].index = MSR_MCG_STATUS;
@@ -1326,6 +1334,9 @@ static int kvm_get_msrs(CPUX86State *env)
         case MSR_KVM_ASYNC_PF_EN:
             env->async_pf_en_msr = msrs[i].data;
             break;
+        case MSR_KVM_PV_UNHALT:
+            env->pv_unhalt_msr = msrs[i].data;
+            break;
         }
     }
 

[Qemu-devel] [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>

MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
used during migration.

Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
---
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index a1ed3e7..10286a5 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -697,6 +697,7 @@ typedef struct CPUX86State {
     uint64_t system_time_msr;
     uint64_t wall_clock_msr;
     uint64_t async_pf_en_msr;
+    uint64_t pv_unhalt_msr;
 
     uint64_t tsc;
     uint64_t tsc_deadline;
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index dbebd3a..98b9088 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -62,6 +62,7 @@ static bool has_msr_star;
 static bool has_msr_hsave_pa;
 static bool has_msr_tsc_deadline;
 static bool has_msr_async_pf_en;
+static bool has_msr_pv_unhalt;
 static bool has_msr_misc_enable;
 static int lm_capable_kernel;
 
@@ -444,7 +445,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     }
 
     has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
-
+    has_msr_pv_unhalt = c->eax & (1 << KVM_FEATURE_PV_UNHALT);
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
     for (i = 0; i <= limit; i++) {
@@ -1012,6 +1013,10 @@ static int kvm_put_msrs(CPUX86State *env, int level)
         if (hyperv_vapic_recommended()) {
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
         }
+        if (has_msr_pv_unhalt) {
+            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_UNHALT,
+                              env->pv_unhalt_msr);
+        }
     }
     if (env->mcg_cap) {
         int i;
@@ -1247,6 +1252,9 @@ static int kvm_get_msrs(CPUX86State *env)
     if (has_msr_async_pf_en) {
         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
     }
+    if (has_msr_pv_unhalt) {
+        msrs[n++].index = MSR_KVM_PV_UNHALT;
+    }
 
     if (env->mcg_cap) {
         msrs[n++].index = MSR_MCG_STATUS;
@@ -1326,6 +1334,9 @@ static int kvm_get_msrs(CPUX86State *env)
         case MSR_KVM_ASYNC_PF_EN:
             env->async_pf_en_msr = msrs[i].data;
             break;
+        case MSR_KVM_PV_UNHALT:
+            env->pv_unhalt_msr = msrs[i].data;
+            break;
         }
     }
 

Re: [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Jan Kiszka]

On 2012-03-23 09:23, Raghavendra K T wrote:
> From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
> 
> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
> used during migration.

Err, and where is it actually saved to/restored from the vmstate? You
are lacking an extension of the CPU vmstate, preferably via a substate.
See e.g. cpu/async_pf_msr.

> 
> Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
> ---
> diff --git a/target-i386/cpu.h b/target-i386/cpu.h
> index a1ed3e7..10286a5 100644
> --- a/target-i386/cpu.h
> +++ b/target-i386/cpu.h
> @@ -697,6 +697,7 @@ typedef struct CPUX86State {
>      uint64_t system_time_msr;
>      uint64_t wall_clock_msr;
>      uint64_t async_pf_en_msr;
> +    uint64_t pv_unhalt_msr;
>  
>      uint64_t tsc;
>      uint64_t tsc_deadline;
> diff --git a/target-i386/kvm.c b/target-i386/kvm.c
> index dbebd3a..98b9088 100644
> --- a/target-i386/kvm.c
> +++ b/target-i386/kvm.c
> @@ -62,6 +62,7 @@ static bool has_msr_star;
>  static bool has_msr_hsave_pa;
>  static bool has_msr_tsc_deadline;
>  static bool has_msr_async_pf_en;
> +static bool has_msr_pv_unhalt;
>  static bool has_msr_misc_enable;
>  static int lm_capable_kernel;
>  
> @@ -444,7 +445,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
>      }
>  
>      has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
> -
> +    has_msr_pv_unhalt = c->eax & (1 << KVM_FEATURE_PV_UNHALT);
>      cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
>  
>      for (i = 0; i <= limit; i++) {
> @@ -1012,6 +1013,10 @@ static int kvm_put_msrs(CPUX86State *env, int level)
>          if (hyperv_vapic_recommended()) {
>              kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
>          }
> +        if (has_msr_pv_unhalt) {
> +            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_UNHALT,
> +                              env->pv_unhalt_msr);
> +        }
>      }
>      if (env->mcg_cap) {
>          int i;
> @@ -1247,6 +1252,9 @@ static int kvm_get_msrs(CPUX86State *env)
>      if (has_msr_async_pf_en) {
>          msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
>      }
> +    if (has_msr_pv_unhalt) {
> +        msrs[n++].index = MSR_KVM_PV_UNHALT;
> +    }
>  
>      if (env->mcg_cap) {
>          msrs[n++].index = MSR_MCG_STATUS;
> @@ -1326,6 +1334,9 @@ static int kvm_get_msrs(CPUX86State *env)
>          case MSR_KVM_ASYNC_PF_EN:
>              env->async_pf_en_msr = msrs[i].data;
>              break;
> +        case MSR_KVM_PV_UNHALT:
> +            env->pv_unhalt_msr = msrs[i].data;
> +            break;
>          }
>      }
>  
> 

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: [Qemu-devel] [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Jan Kiszka]

On 2012-03-23 09:23, Raghavendra K T wrote:
> From: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
> 
> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
> used during migration.

Err, and where is it actually saved to/restored from the vmstate? You
are lacking an extension of the CPU vmstate, preferably via a substate.
See e.g. cpu/async_pf_msr.

> 
> Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
> ---
> diff --git a/target-i386/cpu.h b/target-i386/cpu.h
> index a1ed3e7..10286a5 100644
> --- a/target-i386/cpu.h
> +++ b/target-i386/cpu.h
> @@ -697,6 +697,7 @@ typedef struct CPUX86State {
>      uint64_t system_time_msr;
>      uint64_t wall_clock_msr;
>      uint64_t async_pf_en_msr;
> +    uint64_t pv_unhalt_msr;
>  
>      uint64_t tsc;
>      uint64_t tsc_deadline;
> diff --git a/target-i386/kvm.c b/target-i386/kvm.c
> index dbebd3a..98b9088 100644
> --- a/target-i386/kvm.c
> +++ b/target-i386/kvm.c
> @@ -62,6 +62,7 @@ static bool has_msr_star;
>  static bool has_msr_hsave_pa;
>  static bool has_msr_tsc_deadline;
>  static bool has_msr_async_pf_en;
> +static bool has_msr_pv_unhalt;
>  static bool has_msr_misc_enable;
>  static int lm_capable_kernel;
>  
> @@ -444,7 +445,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
>      }
>  
>      has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
> -
> +    has_msr_pv_unhalt = c->eax & (1 << KVM_FEATURE_PV_UNHALT);
>      cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
>  
>      for (i = 0; i <= limit; i++) {
> @@ -1012,6 +1013,10 @@ static int kvm_put_msrs(CPUX86State *env, int level)
>          if (hyperv_vapic_recommended()) {
>              kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE, 0);
>          }
> +        if (has_msr_pv_unhalt) {
> +            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_UNHALT,
> +                              env->pv_unhalt_msr);
> +        }
>      }
>      if (env->mcg_cap) {
>          int i;
> @@ -1247,6 +1252,9 @@ static int kvm_get_msrs(CPUX86State *env)
>      if (has_msr_async_pf_en) {
>          msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
>      }
> +    if (has_msr_pv_unhalt) {
> +        msrs[n++].index = MSR_KVM_PV_UNHALT;
> +    }
>  
>      if (env->mcg_cap) {
>          msrs[n++].index = MSR_MCG_STATUS;
> @@ -1326,6 +1334,9 @@ static int kvm_get_msrs(CPUX86State *env)
>          case MSR_KVM_ASYNC_PF_EN:
>              env->async_pf_en_msr = msrs[i].data;
>              break;
> +        case MSR_KVM_PV_UNHALT:
> +            env->pv_unhalt_msr = msrs[i].data;
> +            break;
>          }
>      }
>  
> 

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

On 03/23/2012 02:27 PM, Jan Kiszka wrote:
> On 2012-03-23 09:23, Raghavendra K T wrote:
>> From: Raghavendra K T<raghavendra.kt@linux.vnet.ibm.com>
>>
>> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
>> used during migration.
>
> Err, and where is it actually saved to/restored from the vmstate? You
> are lacking an extension of the CPU vmstate, preferably via a substate.
> See e.g. cpu/async_pf_msr.
>

Thanks Jan for review, I 'll check that.

Re: [Qemu-devel] [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

On 03/23/2012 02:27 PM, Jan Kiszka wrote:
> On 2012-03-23 09:23, Raghavendra K T wrote:
>> From: Raghavendra K T<raghavendra.kt@linux.vnet.ibm.com>
>>
>> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
>> used during migration.
>
> Err, and where is it actually saved to/restored from the vmstate? You
> are lacking an extension of the CPU vmstate, preferably via a substate.
> See e.g. cpu/async_pf_msr.
>

Thanks Jan for review, I 'll check that.

Re: [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

On 03/23/2012 02:27 PM, Jan Kiszka wrote:
> On 2012-03-23 09:23, Raghavendra K T wrote:
>> From: Raghavendra K T<raghavendra.kt@linux.vnet.ibm.com>
>>
>> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
>> used during migration.
>
> Err, and where is it actually saved to/restored from the vmstate? You
> are lacking an extension of the CPU vmstate, preferably via a substate.
> See e.g. cpu/async_pf_msr.
>

Please let me know whether adding below patch make it complete. Or did
I miss something else ?

---
diff --git a/target-i386/machine.c b/target-i386/machine.c
index a8be058..c51d8d1 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -279,6 +279,13 @@ static bool async_pf_msr_needed(void *opaque)
      return cpu->async_pf_en_msr != 0;
  }

+static bool pv_unhalt_msr_needed(void *opaque)
+{
+    CPUX86State *cpu = opaque;
+
+    return cpu->pv_unhalt_msr != 0;
+}
+
  static const VMStateDescription vmstate_async_pf_msr = {
      .name = "cpu/async_pf_msr",
      .version_id = 1,
@@ -290,6 +297,17 @@ static const VMStateDescription 
vmstate_async_pf_msr = {
      }
  };

+static const VMStateDescription vmstate_pv_unhalt_msr = {
+    .name = "cpu/pv_unhalt_msr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT64(pv_unhalt_msr, CPUX86State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
  static bool fpop_ip_dp_needed(void *opaque)
  {
      CPUX86State *env = opaque;
@@ -462,6 +480,9 @@ static const VMStateDescription vmstate_cpu = {
          }, {
              .vmsd = &vmstate_msr_ia32_misc_enable,
              .needed = misc_enable_needed,
+        }, {
+            .vmsd = &vmstate_pv_unhalt_msr,
+            .needed = pv_unhalt_msr_needed,
          } , {
              /* empty */
          }

Re: [Qemu-devel] [PATCH 2/2] QEMU kvm: Add support to get/set vcpu unhalt msr to aid migration

[Raghavendra K T]

On 03/23/2012 02:27 PM, Jan Kiszka wrote:
> On 2012-03-23 09:23, Raghavendra K T wrote:
>> From: Raghavendra K T<raghavendra.kt@linux.vnet.ibm.com>
>>
>> MSR_KVM_PV_UNHALT tells whether vcpu is unhalted, which needs to be
>> used during migration.
>
> Err, and where is it actually saved to/restored from the vmstate? You
> are lacking an extension of the CPU vmstate, preferably via a substate.
> See e.g. cpu/async_pf_msr.
>

Please let me know whether adding below patch make it complete. Or did
I miss something else ?

---
diff --git a/target-i386/machine.c b/target-i386/machine.c
index a8be058..c51d8d1 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -279,6 +279,13 @@ static bool async_pf_msr_needed(void *opaque)
      return cpu->async_pf_en_msr != 0;
  }

+static bool pv_unhalt_msr_needed(void *opaque)
+{
+    CPUX86State *cpu = opaque;
+
+    return cpu->pv_unhalt_msr != 0;
+}
+
  static const VMStateDescription vmstate_async_pf_msr = {
      .name = "cpu/async_pf_msr",
      .version_id = 1,
@@ -290,6 +297,17 @@ static const VMStateDescription 
vmstate_async_pf_msr = {
      }
  };

+static const VMStateDescription vmstate_pv_unhalt_msr = {
+    .name = "cpu/pv_unhalt_msr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField []) {
+        VMSTATE_UINT64(pv_unhalt_msr, CPUX86State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
  static bool fpop_ip_dp_needed(void *opaque)
  {
      CPUX86State *env = opaque;
@@ -462,6 +480,9 @@ static const VMStateDescription vmstate_cpu = {
          }, {
              .vmsd = &vmstate_msr_ia32_misc_enable,
              .needed = misc_enable_needed,
+        }, {
+            .vmsd = &vmstate_pv_unhalt_msr,
+            .needed = pv_unhalt_msr_needed,
          } , {
              /* empty */
          }