[PATCH RFC v2 00/19] i386: KVM: expand Hyper-V features early

[PATCH RFC v2 00/19] i386: KVM: expand Hyper-V features early

[Vitaly Kuznetsov]

Changes since RFCv1:
- Took a slightly differenet approach, instead of adding all missing
 Hyper-V feature leaves to feature_word_info[] remove the existing ones
 from there. The logic for handling them is very different and some of
 these leaves are not actually 'feature leaves'. See "i386: drop
 FEAT_HYPERV feature leaves" for details.
- x86_cpu_hyperv_realize introduced [Eduardo].
- Rebased to the latest master.

This pairs with
https://lore.kernel.org/kvm/20200929150944.1235688-1-vkuznets@redhat.com/T/#t
and as KVM part is not yet fully accepted, sending this as RFC.

Upper layer tools like libvirt want to figure out which Hyper-V features are
supported by the underlying stack (QEMU/KVM) but currently they are unable to
do so. We have a nice 'hv_passthrough' CPU flag supported by QEMU but it has
no effect on e.g. QMP's 

query-cpu-model-expansion type=full model={"name":"host","props":{"hv-passthrough":true}}

command as we parse Hyper-V features after creating KVM vCPUs and not at
feature expansion time. To support the use-case we first need to make 
KVM_GET_SUPPORTED_HV_CPUID ioctl a system-wide ioctl as the existing
vCPU version can't be used that early. This is what KVM part does. With
that done, we can make early Hyper-V feature expansion (this series).

Vitaly Kuznetsov (19):
  WIP: update linux/headers
  i386: fill in FEAT_HYPERV_EDX from edx instead of eax
  i386: drop x86_cpu_get_supported_feature_word() forward declaration
  i386: move hyperv_vendor_id initialization to x86_cpu_realizefn()
  i386: move hyperv_interface_id initialization to x86_cpu_realizefn()
  i386: move hyperv_version_id initialization to x86_cpu_realizefn()
  i386: move hyperv_limits initialization to x86_cpu_realizefn()
  i386: keep hyperv_vendor string up-to-date
  i386: invert hyperv_spinlock_attempts setting logic with
    hv_passthrough
  i386: always fill Hyper-V CPUID feature leaves from X86CPU data
  i386: introduce hv_cpuid_cache
  i386: drop FEAT_HYPERV feature leaves
  i386: split hyperv_handle_properties() into
    hyperv_expand_features()/hyperv_fill_cpuids()
  i386: move eVMCS enablement to hyperv_init_vcpu()
  i386: switch hyperv_expand_features() to using error_setg()
  i386: adjust the expected KVM_GET_SUPPORTED_HV_CPUID array size
  i386: prefer system KVM_GET_SUPPORTED_HV_CPUID ioctl over vCPU's one
  i386: use global kvm_state in hyperv_enabled() check
  i386: expand Hyper-V features during CPU feature expansion time

 linux-headers/asm-x86/kvm.h |  20 ++
 linux-headers/linux/kvm.h   |  27 +-
 target/i386/cpu.c           | 135 +++------
 target/i386/cpu.h           |  12 +-
 target/i386/kvm-stub.c      |   5 +
 target/i386/kvm.c           | 526 ++++++++++++++++++++----------------
 target/i386/kvm_i386.h      |   1 +
 7 files changed, 398 insertions(+), 328 deletions(-)

-- 
2.25.4

[PATCH RFC v2 01/19] WIP: update linux/headers

[Vitaly Kuznetsov]

KVM_CAP_SYS_HYPERV_CPUID definition is needed for this series.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 linux-headers/asm-x86/kvm.h | 20 ++++++++++++++++++++
 linux-headers/linux/kvm.h   | 27 ++++++++++++++++++++++++---
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index 0780f97c1850..89e5f3d1bba8 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -192,6 +192,26 @@ struct kvm_msr_list {
 	__u32 indices[0];
 };
 
+/* Maximum size of any access bitmap in bytes */
+#define KVM_MSR_FILTER_MAX_BITMAP_SIZE 0x600
+
+/* for KVM_X86_SET_MSR_FILTER */
+struct kvm_msr_filter_range {
+#define KVM_MSR_FILTER_READ  (1 << 0)
+#define KVM_MSR_FILTER_WRITE (1 << 1)
+	__u32 flags;
+	__u32 nmsrs; /* number of msrs in bitmap */
+	__u32 base;  /* MSR index the bitmap starts at */
+	__u8 *bitmap; /* a 1 bit allows the operations in flags, 0 denies */
+};
+
+#define KVM_MSR_FILTER_MAX_RANGES 16
+struct kvm_msr_filter {
+#define KVM_MSR_FILTER_DEFAULT_ALLOW (0 << 0)
+#define KVM_MSR_FILTER_DEFAULT_DENY  (1 << 0)
+	__u32 flags;
+	struct kvm_msr_filter_range ranges[KVM_MSR_FILTER_MAX_RANGES];
+};
 
 struct kvm_cpuid_entry {
 	__u32 function;
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 6683e2e1b0a0..b28b806530b7 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -248,6 +248,8 @@ struct kvm_hyperv_exit {
 #define KVM_EXIT_IOAPIC_EOI       26
 #define KVM_EXIT_HYPERV           27
 #define KVM_EXIT_ARM_NISV         28
+#define KVM_EXIT_X86_RDMSR        29
+#define KVM_EXIT_X86_WRMSR        30
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -413,6 +415,17 @@ struct kvm_run {
 			__u64 esr_iss;
 			__u64 fault_ipa;
 		} arm_nisv;
+		/* KVM_EXIT_X86_RDMSR / KVM_EXIT_X86_WRMSR */
+		struct {
+			__u8 error; /* user -> kernel */
+			__u8 pad[7];
+#define KVM_MSR_EXIT_REASON_INVAL	(1 << 0)
+#define KVM_MSR_EXIT_REASON_UNKNOWN	(1 << 1)
+#define KVM_MSR_EXIT_REASON_FILTER	(1 << 2)
+			__u32 reason; /* kernel -> user */
+			__u32 index; /* kernel -> user */
+			__u64 data; /* kernel <-> user */
+		} msr;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -790,9 +803,10 @@ struct kvm_ppc_resize_hpt {
 #define KVM_VM_PPC_HV 1
 #define KVM_VM_PPC_PR 2
 
-/* on MIPS, 0 forces trap & emulate, 1 forces VZ ASE */
-#define KVM_VM_MIPS_TE		0
+/* on MIPS, 0 indicates auto, 1 forces VZ ASE, 2 forces trap & emulate */
+#define KVM_VM_MIPS_AUTO	0
 #define KVM_VM_MIPS_VZ		1
+#define KVM_VM_MIPS_TE		2
 
 #define KVM_S390_SIE_PAGE_OFFSET 1
 
@@ -1035,6 +1049,10 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_LAST_CPU 184
 #define KVM_CAP_SMALLER_MAXPHYADDR 185
 #define KVM_CAP_S390_DIAG318 186
+#define KVM_CAP_STEAL_TIME 187
+#define KVM_CAP_X86_USER_SPACE_MSR 188
+#define KVM_CAP_X86_MSR_FILTER 189
+#define KVM_CAP_SYS_HYPERV_CPUID 190
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1493,7 +1511,7 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_MANUAL_DIRTY_LOG_PROTECT_2 */
 #define KVM_CLEAR_DIRTY_LOG          _IOWR(KVMIO, 0xc0, struct kvm_clear_dirty_log)
 
-/* Available with KVM_CAP_HYPERV_CPUID */
+/* Available with KVM_CAP_HYPERV_CPUID (vcpu) / KVM_CAP_SYS_HYPERV_CPUID (system) */
 #define KVM_GET_SUPPORTED_HV_CPUID _IOWR(KVMIO, 0xc1, struct kvm_cpuid2)
 
 /* Available with KVM_CAP_ARM_SVE */
@@ -1536,6 +1554,9 @@ struct kvm_pv_cmd {
 /* Available with KVM_CAP_S390_PROTECTED */
 #define KVM_S390_PV_COMMAND		_IOWR(KVMIO, 0xc5, struct kvm_pv_cmd)
 
+/* Available with KVM_CAP_X86_MSR_FILTER */
+#define KVM_X86_SET_MSR_FILTER	_IOW(KVMIO,  0xc6, struct kvm_msr_filter)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
-- 
2.25.4

[PATCH RFC v2 02/19] i386: fill in FEAT_HYPERV_EDX from edx instead of eax

[Vitaly Kuznetsov]

There was a typo which went unnoticed.

Fixes: e48ddcc6ce13 ("i386/kvm: implement 'hv-passthrough' mode")
Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
- Similar fix had been submitted at:                                                                                                                                                                                                          https://lore.kernel.org/qemu-devel/20190820103030.12515-1-zhenyuw@linux.intel.com/
---
 target/i386/kvm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 9efb07e7c839..aa5dc562b697 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1229,7 +1229,7 @@ static int hyperv_handle_properties(CPUState *cs,
         if (c) {
             env->features[FEAT_HYPERV_EAX] = c->eax;
             env->features[FEAT_HYPERV_EBX] = c->ebx;
-            env->features[FEAT_HYPERV_EDX] = c->eax;
+            env->features[FEAT_HYPERV_EDX] = c->edx;
         }
         c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
         if (c) {
-- 
2.25.4

[PATCH RFC v2 03/19] i386: drop x86_cpu_get_supported_feature_word() forward declaration

[Vitaly Kuznetsov]

We only use x86_cpu_get_supported_feature_word() after its implementation,
no forward declaration needed.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 3ffd877dd51f..ca713bef5eaf 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -4179,9 +4179,6 @@ void x86_cpu_change_kvm_default(const char *prop, const char *value)
     assert(pv->prop);
 }
 
-static uint64_t x86_cpu_get_supported_feature_word(FeatureWord w,
-                                                   bool migratable_only);
-
 static bool lmce_supported(void)
 {
     uint64_t mce_cap = 0;
-- 
2.25.4

[PATCH RFC v2 04/19] i386: move hyperv_vendor_id initialization to x86_cpu_realizefn()

[Vitaly Kuznetsov]

As a preparation to expanding Hyper-V CPU features early, move
hyperv_vendor_id initialization to x86_cpu_realizefn(). Introduce
x86_cpu_hyperv_realize() to not not pollute x86_cpu_realizefn()
itself.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c | 23 ++++++++++++++++++++++-
 target/i386/cpu.h |  3 ++-
 target/i386/kvm.c | 25 ++++++++++---------------
 3 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index ca713bef5eaf..71b96e383573 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6489,6 +6489,24 @@ static void x86_cpu_filter_features(X86CPU *cpu, bool verbose)
     }
 }
 
+static void x86_cpu_hyperv_realize(X86CPU *cpu)
+{
+    size_t len;
+
+    if (!cpu->hyperv_vendor) {
+        memcpy(cpu->hyperv_vendor_id, "Microsoft Hv", 12);
+    } else {
+        len = strlen(cpu->hyperv_vendor);
+
+        if (len > 12) {
+            warn_report("hv-vendor-id truncated to 12 characters");
+            len = 12;
+        }
+        memset(cpu->hyperv_vendor_id, 0, 12);
+        memcpy(cpu->hyperv_vendor_id, cpu->hyperv_vendor, len);
+    }
+}
+
 static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -6539,6 +6557,9 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
         return;
     }
 
+    /* Process Hyper-V enlightenments */
+    x86_cpu_hyperv_realize(cpu);
+
     x86_cpu_expand_features(cpu, &local_err);
     if (local_err) {
         goto out;
@@ -7209,7 +7230,7 @@ static Property x86_cpu_properties[] = {
     DEFINE_PROP_UINT32("min-xlevel2", X86CPU, env.cpuid_min_xlevel2, 0),
     DEFINE_PROP_UINT64("ucode-rev", X86CPU, ucode_rev, 0),
     DEFINE_PROP_BOOL("full-cpuid-auto-level", X86CPU, full_cpuid_auto_level, true),
-    DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor_id),
+    DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor),
     DEFINE_PROP_BOOL("cpuid-0xb", X86CPU, enable_cpuid_0xb, true),
     DEFINE_PROP_BOOL("lmce", X86CPU, enable_lmce, false),
     DEFINE_PROP_BOOL("l3-cache", X86CPU, enable_l3_cache, true),
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index f519d2bfd48d..b0bef3f1c793 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1654,11 +1654,12 @@ struct X86CPU {
     uint64_t ucode_rev;
 
     uint32_t hyperv_spinlock_attempts;
-    char *hyperv_vendor_id;
+    char *hyperv_vendor;
     bool hyperv_synic_kvm_only;
     uint64_t hyperv_features;
     bool hyperv_passthrough;
     OnOffAuto hyperv_no_nonarch_cs;
+    uint32_t hyperv_vendor_id[3];
 
     bool check_cpuid;
     bool enforce_cpuid;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index aa5dc562b697..c5c4eaefe862 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1225,6 +1225,13 @@ static int hyperv_handle_properties(CPUState *cs,
         memcpy(cpuid_ent, &cpuid->entries[0],
                cpuid->nent * sizeof(cpuid->entries[0]));
 
+        c = cpuid_find_entry(cpuid, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, 0);
+        if (c) {
+            cpu->hyperv_vendor_id[0] = c->ebx;
+            cpu->hyperv_vendor_id[1] = c->ecx;
+            cpu->hyperv_vendor_id[2] = c->edx;
+        }
+
         c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0);
         if (c) {
             env->features[FEAT_HYPERV_EAX] = c->eax;
@@ -1299,23 +1306,11 @@ static int hyperv_handle_properties(CPUState *cs,
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
-    if (!cpu->hyperv_vendor_id) {
-        memcpy(signature, "Microsoft Hv", 12);
-    } else {
-        size_t len = strlen(cpu->hyperv_vendor_id);
-
-        if (len > 12) {
-            error_report("hv-vendor-id truncated to 12 characters");
-            len = 12;
-        }
-        memset(signature, 0, 12);
-        memcpy(signature, cpu->hyperv_vendor_id, len);
-    }
     c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ?
         HV_CPUID_NESTED_FEATURES : HV_CPUID_IMPLEMENT_LIMITS;
-    c->ebx = signature[0];
-    c->ecx = signature[1];
-    c->edx = signature[2];
+    c->ebx = cpu->hyperv_vendor_id[0];
+    c->ecx = cpu->hyperv_vendor_id[1];
+    c->edx = cpu->hyperv_vendor_id[2];
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_INTERFACE;
-- 
2.25.4

[PATCH RFC v2 05/19] i386: move hyperv_interface_id initialization to x86_cpu_realizefn()

[Vitaly Kuznetsov]

As a preparation to expanding Hyper-V CPU features early, move
hyperv_interface_id initialization to x86_cpu_realizefn().

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c |  6 ++++++
 target/i386/cpu.h |  1 +
 target/i386/kvm.c | 18 ++++++++++++------
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 71b96e383573..9e77d9579b5c 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6505,6 +6505,12 @@ static void x86_cpu_hyperv_realize(X86CPU *cpu)
         memset(cpu->hyperv_vendor_id, 0, 12);
         memcpy(cpu->hyperv_vendor_id, cpu->hyperv_vendor, len);
     }
+
+    /* 'Hv#1' interface identification*/
+    cpu->hyperv_interface_id[0] = 0x31237648;
+    cpu->hyperv_interface_id[1] = 0;
+    cpu->hyperv_interface_id[2] = 0;
+    cpu->hyperv_interface_id[3] = 0;
 }
 
 static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index b0bef3f1c793..702949462ab9 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1660,6 +1660,7 @@ struct X86CPU {
     bool hyperv_passthrough;
     OnOffAuto hyperv_no_nonarch_cs;
     uint32_t hyperv_vendor_id[3];
+    uint32_t hyperv_interface_id[4];
 
     bool check_cpuid;
     bool enforce_cpuid;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index c5c4eaefe862..6dde7c97deb3 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1188,7 +1188,6 @@ static int hyperv_handle_properties(CPUState *cs,
     CPUX86State *env = &cpu->env;
     struct kvm_cpuid2 *cpuid;
     struct kvm_cpuid_entry2 *c;
-    uint32_t signature[3];
     uint32_t cpuid_i = 0;
     int r;
 
@@ -1232,6 +1231,14 @@ static int hyperv_handle_properties(CPUState *cs,
             cpu->hyperv_vendor_id[2] = c->edx;
         }
 
+        c = cpuid_find_entry(cpuid, HV_CPUID_INTERFACE, 0);
+        if (c) {
+            cpu->hyperv_interface_id[0] = c->eax;
+            cpu->hyperv_interface_id[1] = c->ebx;
+            cpu->hyperv_interface_id[2] = c->ecx;
+            cpu->hyperv_interface_id[3] = c->edx;
+        }
+
         c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0);
         if (c) {
             env->features[FEAT_HYPERV_EAX] = c->eax;
@@ -1314,11 +1321,10 @@ static int hyperv_handle_properties(CPUState *cs,
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_INTERFACE;
-    memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
-    c->eax = signature[0];
-    c->ebx = 0;
-    c->ecx = 0;
-    c->edx = 0;
+    c->eax = cpu->hyperv_interface_id[0];
+    c->ebx = cpu->hyperv_interface_id[1];
+    c->ecx = cpu->hyperv_interface_id[2];
+    c->edx = cpu->hyperv_interface_id[3];
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_VERSION;
-- 
2.25.4

[PATCH RFC v2 06/19] i386: move hyperv_version_id initialization to x86_cpu_realizefn()

[Vitaly Kuznetsov]

As a preparation to expanding Hyper-V CPU features early, move
hyperv_version_id initialization to x86_cpu_realizefn().

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c |  4 ++++
 target/i386/cpu.h |  1 +
 target/i386/kvm.c | 14 ++++++++++++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 9e77d9579b5c..99bcfcbb448a 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6511,6 +6511,10 @@ static void x86_cpu_hyperv_realize(X86CPU *cpu)
     cpu->hyperv_interface_id[1] = 0;
     cpu->hyperv_interface_id[2] = 0;
     cpu->hyperv_interface_id[3] = 0;
+
+    /* Hypervisor system identity */
+    cpu->hyperv_version_id[0] = 0x00001bbc;
+    cpu->hyperv_version_id[1] = 0x00060001;
 }
 
 static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 702949462ab9..077c19c5642b 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1661,6 +1661,7 @@ struct X86CPU {
     OnOffAuto hyperv_no_nonarch_cs;
     uint32_t hyperv_vendor_id[3];
     uint32_t hyperv_interface_id[4];
+    uint32_t hyperv_version_id[4];
 
     bool check_cpuid;
     bool enforce_cpuid;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 6dde7c97deb3..b335b3a3d1c7 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1239,6 +1239,14 @@ static int hyperv_handle_properties(CPUState *cs,
             cpu->hyperv_interface_id[3] = c->edx;
         }
 
+        c = cpuid_find_entry(cpuid, HV_CPUID_VERSION, 0);
+        if (c) {
+            cpu->hyperv_version_id[0] = c->eax;
+            cpu->hyperv_version_id[1] = c->ebx;
+            cpu->hyperv_version_id[2] = c->ecx;
+            cpu->hyperv_version_id[3] = c->edx;
+        }
+
         c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0);
         if (c) {
             env->features[FEAT_HYPERV_EAX] = c->eax;
@@ -1328,8 +1336,10 @@ static int hyperv_handle_properties(CPUState *cs,
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_VERSION;
-    c->eax = 0x00001bbc;
-    c->ebx = 0x00060001;
+    c->eax = cpu->hyperv_version_id[0];
+    c->ebx = cpu->hyperv_version_id[1];
+    c->ecx = cpu->hyperv_version_id[2];
+    c->edx = cpu->hyperv_version_id[3];
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_FEATURES;
-- 
2.25.4

[PATCH RFC v2 07/19] i386: move hyperv_limits initialization to x86_cpu_realizefn()

[Vitaly Kuznetsov]

As a preparation to expanding Hyper-V CPU features early, move
hyperv_limits initialization to x86_cpu_realizefn().

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c |  5 +++++
 target/i386/cpu.h |  1 +
 target/i386/kvm.c | 13 ++++++++++++-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 99bcfcbb448a..d657590ab55d 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6515,6 +6515,11 @@ static void x86_cpu_hyperv_realize(X86CPU *cpu)
     /* Hypervisor system identity */
     cpu->hyperv_version_id[0] = 0x00001bbc;
     cpu->hyperv_version_id[1] = 0x00060001;
+
+    /* Hypervisor implementation limits */
+    cpu->hyperv_limits[0] = 64;
+    cpu->hyperv_limits[1] = 0;
+    cpu->hyperv_limits[2] = 0;
 }
 
 static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 077c19c5642b..187e4a09e65d 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1662,6 +1662,7 @@ struct X86CPU {
     uint32_t hyperv_vendor_id[3];
     uint32_t hyperv_interface_id[4];
     uint32_t hyperv_version_id[4];
+    uint32_t hyperv_limits[3];
 
     bool check_cpuid;
     bool enforce_cpuid;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index b335b3a3d1c7..afddc9a4a260 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1253,6 +1253,15 @@ static int hyperv_handle_properties(CPUState *cs,
             env->features[FEAT_HYPERV_EBX] = c->ebx;
             env->features[FEAT_HYPERV_EDX] = c->edx;
         }
+
+        c = cpuid_find_entry(cpuid, HV_CPUID_IMPLEMENT_LIMITS, 0);
+        if (c) {
+            cpu->hv_max_vps = c->eax;
+            cpu->hyperv_limits[0] = c->ebx;
+            cpu->hyperv_limits[1] = c->ecx;
+            cpu->hyperv_limits[2] = c->edx;
+        }
+
         c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
         if (c) {
             env->features[FEAT_HV_RECOMM_EAX] = c->eax;
@@ -1355,7 +1364,9 @@ static int hyperv_handle_properties(CPUState *cs,
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_IMPLEMENT_LIMITS;
     c->eax = cpu->hv_max_vps;
-    c->ebx = 0x40;
+    c->ebx = cpu->hyperv_limits[0];
+    c->ecx = cpu->hyperv_limits[1];
+    c->edx = cpu->hyperv_limits[2];
 
     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) {
         __u32 function;
-- 
2.25.4

[PATCH RFC v2 08/19] i386: keep hyperv_vendor string up-to-date

[Vitaly Kuznetsov]

When cpu->hyperv_vendor is not set manually we default to "Microsoft Hv"
and in 'hv_passthrough' mode we get the information from the host. This
information is stored in cpu->hyperv_vendor_id[] array but we don't update
cpu->hyperv_vendor string so e.g. QMP's query-cpu-model-expansion output
is incorrect.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c | 20 ++++++++++----------
 target/i386/kvm.c |  4 ++++
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index d657590ab55d..8ec0af0a6d48 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6493,18 +6493,18 @@ static void x86_cpu_hyperv_realize(X86CPU *cpu)
 {
     size_t len;
 
+    /* Hyper-V vendor id */
     if (!cpu->hyperv_vendor) {
-        memcpy(cpu->hyperv_vendor_id, "Microsoft Hv", 12);
-    } else {
-        len = strlen(cpu->hyperv_vendor);
-
-        if (len > 12) {
-            warn_report("hv-vendor-id truncated to 12 characters");
-            len = 12;
-        }
-        memset(cpu->hyperv_vendor_id, 0, 12);
-        memcpy(cpu->hyperv_vendor_id, cpu->hyperv_vendor, len);
+        object_property_set_str(OBJECT(cpu), "hv-vendor-id", "Microsoft Hv",
+                                &error_abort);
+    }
+    len = strlen(cpu->hyperv_vendor);
+    if (len > 12) {
+        warn_report("hv-vendor-id truncated to 12 characters");
+        len = 12;
     }
+    memset(cpu->hyperv_vendor_id, 0, 12);
+    memcpy(cpu->hyperv_vendor_id, cpu->hyperv_vendor, len);
 
     /* 'Hv#1' interface identification*/
     cpu->hyperv_interface_id[0] = 0x31237648;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index afddc9a4a260..eb21f77be51b 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1229,6 +1229,10 @@ static int hyperv_handle_properties(CPUState *cs,
             cpu->hyperv_vendor_id[0] = c->ebx;
             cpu->hyperv_vendor_id[1] = c->ecx;
             cpu->hyperv_vendor_id[2] = c->edx;
+            cpu->hyperv_vendor = g_realloc(cpu->hyperv_vendor,
+                                           sizeof(cpu->hyperv_vendor_id) + 1);
+            memcpy(cpu->hyperv_vendor, cpu->hyperv_vendor_id,
+                   sizeof(cpu->hyperv_vendor_id));
         }
 
         c = cpuid_find_entry(cpuid, HV_CPUID_INTERFACE, 0);
-- 
2.25.4

[PATCH RFC v2 09/19] i386: invert hyperv_spinlock_attempts setting logic with hv_passthrough

[Vitaly Kuznetsov]

There is no need to have this special case: like all other Hyper-V
enlightenments we can just use kernel's supplied value in hv_passthrough
mode.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index eb21f77be51b..94e2195acb36 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1269,11 +1269,7 @@ static int hyperv_handle_properties(CPUState *cs,
         c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
         if (c) {
             env->features[FEAT_HV_RECOMM_EAX] = c->eax;
-
-            /* hv-spinlocks may have been overriden */
-            if (cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_NOTIFY) {
-                c->ebx = cpu->hyperv_spinlock_attempts;
-            }
+            cpu->hyperv_spinlock_attempts = c->ebx;
         }
         c = cpuid_find_entry(cpuid, HV_CPUID_NESTED_FEATURES, 0);
         if (c) {
-- 
2.25.4

[PATCH RFC v2 10/19] i386: always fill Hyper-V CPUID feature leaves from X86CPU data

[Vitaly Kuznetsov]

We have all the required data in X86CPU already and as we are about to
split hyperv_handle_properties() into hyperv_expand_features()/
hyperv_fill_cpuids() we can remove the blind copy. The functional change
is that QEMU won't pass CPUID leaves it doesn't currently know about
to the guest but arguably this is a good change.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 94e2195acb36..a9823d4af7cb 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1221,9 +1221,6 @@ static int hyperv_handle_properties(CPUState *cs,
     }
 
     if (cpu->hyperv_passthrough) {
-        memcpy(cpuid_ent, &cpuid->entries[0],
-               cpuid->nent * sizeof(cpuid->entries[0]));
-
         c = cpuid_find_entry(cpuid, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, 0);
         if (c) {
             cpu->hyperv_vendor_id[0] = c->ebx;
@@ -1322,12 +1319,6 @@ static int hyperv_handle_properties(CPUState *cs,
         goto free;
     }
 
-    if (cpu->hyperv_passthrough) {
-        /* We already copied all feature words from KVM as is */
-        r = cpuid->nent;
-        goto free;
-    }
-
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
     c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ?
-- 
2.25.4

[PATCH RFC v2 11/19] i386: introduce hv_cpuid_cache

[Vitaly Kuznetsov]

Just like with cpuid_cache, it makes no sense to call
KVM_GET_SUPPORTED_HV_CPUID more than once and instead of (ab)using
env->features[] and/or trying to keep all the code in one place, it is
better to introduce persistent hv_cpuid_cache and hv_cpuid_get_host()
accessor to it.

Note, hv_cpuid_get_fw() is converted to using hv_cpuid_get_host()
just to be removed later with Hyper-V specific feature words.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 213 +++++++++++++++++++++++-----------------------
 1 file changed, 106 insertions(+), 107 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index a9823d4af7cb..0e6eef6a52c2 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -124,6 +124,7 @@ static int has_exception_payload;
 static bool has_msr_mcg_ext_ctl;
 
 static struct kvm_cpuid2 *cpuid_cache;
+static struct kvm_cpuid2 *hv_cpuid_cache;
 static struct kvm_msr_list *kvm_feature_msrs;
 
 int kvm_has_pit_state2(void)
@@ -1079,9 +1080,36 @@ static struct kvm_cpuid2 *get_supported_hv_cpuid_legacy(CPUState *cs)
     return cpuid;
 }
 
-static int hv_cpuid_get_fw(struct kvm_cpuid2 *cpuid, int fw, uint32_t *r)
+static uint32_t hv_cpuid_get_host(CPUState *cs, uint32_t func, int reg)
 {
     struct kvm_cpuid_entry2 *entry;
+    struct kvm_cpuid2 *cpuid;
+
+    if (hv_cpuid_cache) {
+        cpuid = hv_cpuid_cache;
+    } else {
+        if (kvm_check_extension(kvm_state, KVM_CAP_HYPERV_CPUID) > 0) {
+            cpuid = get_supported_hv_cpuid(cs);
+        } else {
+            cpuid = get_supported_hv_cpuid_legacy(cs);
+        }
+        hv_cpuid_cache = cpuid;
+    }
+
+    if (!cpuid) {
+        return 0;
+    }
+
+    entry = cpuid_find_entry(cpuid, func, 0);
+    if (!entry) {
+        return 0;
+    }
+
+    return cpuid_entry_get_reg(entry, reg);
+}
+
+static uint32_t hv_cpuid_get_fw(CPUState *cs, int fw)
+{
     uint32_t func;
     int reg;
 
@@ -1099,30 +1127,13 @@ static int hv_cpuid_get_fw(struct kvm_cpuid2 *cpuid, int fw, uint32_t *r)
         func = HV_CPUID_ENLIGHTMENT_INFO;
         break;
     default:
-        return -EINVAL;
-    }
-
-    entry = cpuid_find_entry(cpuid, func, 0);
-    if (!entry) {
-        return -ENOENT;
-    }
-
-    switch (reg) {
-    case R_EAX:
-        *r = entry->eax;
-        break;
-    case R_EDX:
-        *r = entry->edx;
-        break;
-    default:
-        return -EINVAL;
+        return 0;
     }
 
-    return 0;
+    return hv_cpuid_get_host(cs, func, reg);
 }
 
-static int hv_cpuid_check_and_set(CPUState *cs, struct kvm_cpuid2 *cpuid,
-                                  int feature)
+static int hv_cpuid_check_and_set(CPUState *cs, int feature)
 {
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
@@ -1155,7 +1166,8 @@ static int hv_cpuid_check_and_set(CPUState *cs, struct kvm_cpuid2 *cpuid,
             continue;
         }
 
-        if (hv_cpuid_get_fw(cpuid, fw, &r) || (r & bits) != bits) {
+        r = hv_cpuid_get_fw(cs, fw);
+        if ((r & bits) != bits) {
             if (hyperv_feat_enabled(cpu, feature)) {
                 fprintf(stderr,
                         "Hyper-V %s is not supported by kernel\n",
@@ -1186,7 +1198,6 @@ static int hyperv_handle_properties(CPUState *cs,
 {
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
-    struct kvm_cpuid2 *cpuid;
     struct kvm_cpuid_entry2 *c;
     uint32_t cpuid_i = 0;
     int r;
@@ -1214,92 +1225,85 @@ static int hyperv_handle_properties(CPUState *cs,
         }
     }
 
-    if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_CPUID) > 0) {
-        cpuid = get_supported_hv_cpuid(cs);
-    } else {
-        cpuid = get_supported_hv_cpuid_legacy(cs);
-    }
-
     if (cpu->hyperv_passthrough) {
-        c = cpuid_find_entry(cpuid, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, 0);
-        if (c) {
-            cpu->hyperv_vendor_id[0] = c->ebx;
-            cpu->hyperv_vendor_id[1] = c->ecx;
-            cpu->hyperv_vendor_id[2] = c->edx;
-            cpu->hyperv_vendor = g_realloc(cpu->hyperv_vendor,
-                                           sizeof(cpu->hyperv_vendor_id) + 1);
-            memcpy(cpu->hyperv_vendor, cpu->hyperv_vendor_id,
-                   sizeof(cpu->hyperv_vendor_id));
-        }
-
-        c = cpuid_find_entry(cpuid, HV_CPUID_INTERFACE, 0);
-        if (c) {
-            cpu->hyperv_interface_id[0] = c->eax;
-            cpu->hyperv_interface_id[1] = c->ebx;
-            cpu->hyperv_interface_id[2] = c->ecx;
-            cpu->hyperv_interface_id[3] = c->edx;
-        }
-
-        c = cpuid_find_entry(cpuid, HV_CPUID_VERSION, 0);
-        if (c) {
-            cpu->hyperv_version_id[0] = c->eax;
-            cpu->hyperv_version_id[1] = c->ebx;
-            cpu->hyperv_version_id[2] = c->ecx;
-            cpu->hyperv_version_id[3] = c->edx;
-        }
-
-        c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0);
-        if (c) {
-            env->features[FEAT_HYPERV_EAX] = c->eax;
-            env->features[FEAT_HYPERV_EBX] = c->ebx;
-            env->features[FEAT_HYPERV_EDX] = c->edx;
-        }
-
-        c = cpuid_find_entry(cpuid, HV_CPUID_IMPLEMENT_LIMITS, 0);
-        if (c) {
-            cpu->hv_max_vps = c->eax;
-            cpu->hyperv_limits[0] = c->ebx;
-            cpu->hyperv_limits[1] = c->ecx;
-            cpu->hyperv_limits[2] = c->edx;
-        }
-
-        c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
-        if (c) {
-            env->features[FEAT_HV_RECOMM_EAX] = c->eax;
-            cpu->hyperv_spinlock_attempts = c->ebx;
-        }
-        c = cpuid_find_entry(cpuid, HV_CPUID_NESTED_FEATURES, 0);
-        if (c) {
-            env->features[FEAT_HV_NESTED_EAX] = c->eax;
-        }
+        cpu->hyperv_vendor_id[0] =
+            hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EBX);
+        cpu->hyperv_vendor_id[1] =
+            hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_ECX);
+        cpu->hyperv_vendor_id[2] =
+            hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EDX);
+        cpu->hyperv_vendor = g_realloc(cpu->hyperv_vendor,
+                                       sizeof(cpu->hyperv_vendor_id) + 1);
+        memcpy(cpu->hyperv_vendor, cpu->hyperv_vendor_id,
+               sizeof(cpu->hyperv_vendor_id));
+
+        cpu->hyperv_interface_id[0] =
+            hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EAX);
+        cpu->hyperv_interface_id[1] =
+            hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EBX);
+        cpu->hyperv_interface_id[2] =
+            hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_ECX);
+        cpu->hyperv_interface_id[3] =
+            hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EDX);
+
+        cpu->hyperv_version_id[0] =
+            hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EAX);
+        cpu->hyperv_version_id[1] =
+            hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EBX);
+        cpu->hyperv_version_id[2] =
+            hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_ECX);
+        cpu->hyperv_version_id[3] =
+            hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX);
+
+        env->features[FEAT_HYPERV_EAX] =
+            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EAX);
+        env->features[FEAT_HYPERV_EBX] =
+            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EBX);
+        env->features[FEAT_HYPERV_EDX] =
+            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EDX);
+
+        cpu->hv_max_vps = hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS,
+                                            R_EAX);
+        cpu->hyperv_limits[0] =
+            hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EBX);
+        cpu->hyperv_limits[1] =
+            hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_ECX);
+        cpu->hyperv_limits[2] =
+            hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EDX);
+
+        env->features[FEAT_HV_RECOMM_EAX] =
+            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
+        cpu->hyperv_spinlock_attempts =
+            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EBX);
+
+        env->features[FEAT_HV_NESTED_EAX] =
+            hv_cpuid_get_host(cs, HV_CPUID_NESTED_FEATURES, R_EAX);
     }
 
     if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
         env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING;
     } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
-        c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
-        if (c) {
-            env->features[FEAT_HV_RECOMM_EAX] |=
-                c->eax & HV_NO_NONARCH_CORESHARING;
-        }
+        env->features[FEAT_HV_RECOMM_EAX] |=
+            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
+            HV_NO_NONARCH_CORESHARING;
     }
 
     /* Features */
-    r = hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RELAXED);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VAPIC);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TIME);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_CRASH);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RESET);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VPINDEX);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RUNTIME);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_SYNIC);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_FREQUENCIES);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_REENLIGHTENMENT);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TLBFLUSH);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_EVMCS);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_IPI);
-    r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER_DIRECT);
+    r = hv_cpuid_check_and_set(cs, HYPERV_FEAT_RELAXED);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_VAPIC);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_TIME);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_CRASH);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_RESET);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_VPINDEX);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_RUNTIME);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_SYNIC);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_FREQUENCIES);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_REENLIGHTENMENT);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_TLBFLUSH);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_EVMCS);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_IPI);
+    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER_DIRECT);
 
     /* Additional dependencies not covered by kvm_hyperv_properties[] */
     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) &&
@@ -1315,8 +1319,7 @@ static int hyperv_handle_properties(CPUState *cs,
     env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
 
     if (r) {
-        r = -ENOSYS;
-        goto free;
+        return -ENOSYS;
     }
 
     c = &cpuid_ent[cpuid_i++];
@@ -1373,12 +1376,8 @@ static int hyperv_handle_properties(CPUState *cs,
         c->function = HV_CPUID_NESTED_FEATURES;
         c->eax = env->features[FEAT_HV_NESTED_EAX];
     }
-    r = cpuid_i;
-
-free:
-    g_free(cpuid);
 
-    return r;
+    return cpuid_i;
 }
 
 static Error *hv_passthrough_mig_blocker;
-- 
2.25.4

[PATCH RFC v2 12/19] i386: drop FEAT_HYPERV feature leaves

[Vitaly Kuznetsov]

Hyper-V feature leaves are weird. We have some of them in
feature_word_info[] array but we don't use feature_word_info
magic to enable them. Neither do we use feature_dependencies[]
mechanism to validate the configuration as it doesn't allign
well with Hyper-V's many-to-many dependency chains. Some of
the feature leaves hold not only feature bits, but also values.
E.g. FEAT_HV_NESTED_EAX contains both features and the supported
Enlightened VMCS range.

Hyper-V features are already represented in 'struct X86CPU' with
uint64_t hyperv_features so duplicating them in env->features adds
little (or zero) benefits. THe other half of Hyper-V emulation features
is also stored with values in hyperv_vendor_id[], hyperv_limits[],...
so env->features[] is already incomplete.

Remove Hyper-V feature leaves from env->features[] completely.
kvm_hyperv_properties[] is converted to using raw CPUID func/reg
pairs for features, this allows us to get rid of hv_cpuid_get_fw()
conversion.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c |  90 +----------------------
 target/i386/cpu.h |   6 +-
 target/i386/kvm.c | 181 ++++++++++++++++++++++------------------------
 3 files changed, 90 insertions(+), 187 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index 8ec0af0a6d48..a178db255641 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -828,94 +828,6 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
          */
         .no_autoenable_flags = ~0U,
     },
-    /*
-     * .feat_names are commented out for Hyper-V enlightenments because we
-     * don't want to have two different ways for enabling them on QEMU command
-     * line. Some features (e.g. "hyperv_time", "hyperv_vapic", ...) require
-     * enabling several feature bits simultaneously, exposing these bits
-     * individually may just confuse guests.
-     */
-    [FEAT_HYPERV_EAX] = {
-        .type = CPUID_FEATURE_WORD,
-        .feat_names = {
-            NULL /* hv_msr_vp_runtime_access */, NULL /* hv_msr_time_refcount_access */,
-            NULL /* hv_msr_synic_access */, NULL /* hv_msr_stimer_access */,
-            NULL /* hv_msr_apic_access */, NULL /* hv_msr_hypercall_access */,
-            NULL /* hv_vpindex_access */, NULL /* hv_msr_reset_access */,
-            NULL /* hv_msr_stats_access */, NULL /* hv_reftsc_access */,
-            NULL /* hv_msr_idle_access */, NULL /* hv_msr_frequency_access */,
-            NULL /* hv_msr_debug_access */, NULL /* hv_msr_reenlightenment_access */,
-            NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-        },
-        .cpuid = { .eax = 0x40000003, .reg = R_EAX, },
-    },
-    [FEAT_HYPERV_EBX] = {
-        .type = CPUID_FEATURE_WORD,
-        .feat_names = {
-            NULL /* hv_create_partitions */, NULL /* hv_access_partition_id */,
-            NULL /* hv_access_memory_pool */, NULL /* hv_adjust_message_buffers */,
-            NULL /* hv_post_messages */, NULL /* hv_signal_events */,
-            NULL /* hv_create_port */, NULL /* hv_connect_port */,
-            NULL /* hv_access_stats */, NULL, NULL, NULL /* hv_debugging */,
-            NULL /* hv_cpu_power_management */, NULL /* hv_configure_profiler */,
-            NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-        },
-        .cpuid = { .eax = 0x40000003, .reg = R_EBX, },
-    },
-    [FEAT_HYPERV_EDX] = {
-        .type = CPUID_FEATURE_WORD,
-        .feat_names = {
-            NULL /* hv_mwait */, NULL /* hv_guest_debugging */,
-            NULL /* hv_perf_monitor */, NULL /* hv_cpu_dynamic_part */,
-            NULL /* hv_hypercall_params_xmm */, NULL /* hv_guest_idle_state */,
-            NULL, NULL,
-            NULL, NULL, NULL /* hv_guest_crash_msr */, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-        },
-        .cpuid = { .eax = 0x40000003, .reg = R_EDX, },
-    },
-    [FEAT_HV_RECOMM_EAX] = {
-        .type = CPUID_FEATURE_WORD,
-        .feat_names = {
-            NULL /* hv_recommend_pv_as_switch */,
-            NULL /* hv_recommend_pv_tlbflush_local */,
-            NULL /* hv_recommend_pv_tlbflush_remote */,
-            NULL /* hv_recommend_msr_apic_access */,
-            NULL /* hv_recommend_msr_reset */,
-            NULL /* hv_recommend_relaxed_timing */,
-            NULL /* hv_recommend_dma_remapping */,
-            NULL /* hv_recommend_int_remapping */,
-            NULL /* hv_recommend_x2apic_msrs */,
-            NULL /* hv_recommend_autoeoi_deprecation */,
-            NULL /* hv_recommend_pv_ipi */,
-            NULL /* hv_recommend_ex_hypercalls */,
-            NULL /* hv_hypervisor_is_nested */,
-            NULL /* hv_recommend_int_mbec */,
-            NULL /* hv_recommend_evmcs */,
-            NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL, NULL,
-        },
-        .cpuid = { .eax = 0x40000004, .reg = R_EAX, },
-    },
-    [FEAT_HV_NESTED_EAX] = {
-        .type = CPUID_FEATURE_WORD,
-        .cpuid = { .eax = 0x4000000A, .reg = R_EAX, },
-    },
     [FEAT_SVM] = {
         .type = CPUID_FEATURE_WORD,
         .feat_names = {
@@ -6906,7 +6818,7 @@ static GuestPanicInformation *x86_cpu_get_crash_info(CPUState *cs)
     CPUX86State *env = &cpu->env;
     GuestPanicInformation *panic_info = NULL;
 
-    if (env->features[FEAT_HYPERV_EDX] & HV_GUEST_CRASH_MSR_AVAILABLE) {
+    if (hyperv_feat_enabled(cpu, HYPERV_FEAT_CRASH)) {
         panic_info = g_malloc0(sizeof(GuestPanicInformation));
 
         panic_info->type = GUEST_PANIC_INFORMATION_TYPE_HYPER_V;
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 187e4a09e65d..b07325745bcb 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -519,11 +519,6 @@ typedef enum FeatureWord {
     FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
     FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
     FEAT_KVM_HINTS,     /* CPUID[4000_0001].EDX */
-    FEAT_HYPERV_EAX,    /* CPUID[4000_0003].EAX */
-    FEAT_HYPERV_EBX,    /* CPUID[4000_0003].EBX */
-    FEAT_HYPERV_EDX,    /* CPUID[4000_0003].EDX */
-    FEAT_HV_RECOMM_EAX, /* CPUID[4000_0004].EAX */
-    FEAT_HV_NESTED_EAX, /* CPUID[4000_000A].EAX */
     FEAT_SVM,           /* CPUID[8000_000A].EDX */
     FEAT_XSAVE,         /* CPUID[EAX=0xd,ECX=1].EAX */
     FEAT_6_EAX,         /* CPUID[6].EAX */
@@ -1663,6 +1658,7 @@ struct X86CPU {
     uint32_t hyperv_interface_id[4];
     uint32_t hyperv_version_id[4];
     uint32_t hyperv_limits[3];
+    uint32_t hyperv_nested[4];
 
     bool check_cpuid;
     bool enforce_cpuid;
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 0e6eef6a52c2..86a5cfa1e12b 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -815,7 +815,8 @@ static bool tsc_is_stable_and_known(CPUX86State *env)
 static struct {
     const char *desc;
     struct {
-        uint32_t fw;
+        uint32_t func;
+        int reg;
         uint32_t bits;
     } flags[2];
     uint64_t dependencies;
@@ -823,25 +824,25 @@ static struct {
     [HYPERV_FEAT_RELAXED] = {
         .desc = "relaxed timing (hv-relaxed)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_HYPERCALL_AVAILABLE},
-            {.fw = FEAT_HV_RECOMM_EAX,
+            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
              .bits = HV_RELAXED_TIMING_RECOMMENDED}
         }
     },
     [HYPERV_FEAT_VAPIC] = {
         .desc = "virtual APIC (hv-vapic)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_HYPERCALL_AVAILABLE | HV_APIC_ACCESS_AVAILABLE},
-            {.fw = FEAT_HV_RECOMM_EAX,
+            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
              .bits = HV_APIC_ACCESS_RECOMMENDED}
         }
     },
     [HYPERV_FEAT_TIME] = {
         .desc = "clocksources (hv-time)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_HYPERCALL_AVAILABLE | HV_TIME_REF_COUNT_AVAILABLE |
              HV_REFERENCE_TSC_AVAILABLE}
         }
@@ -849,42 +850,42 @@ static struct {
     [HYPERV_FEAT_CRASH] = {
         .desc = "crash MSRs (hv-crash)",
         .flags = {
-            {.fw = FEAT_HYPERV_EDX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
              .bits = HV_GUEST_CRASH_MSR_AVAILABLE}
         }
     },
     [HYPERV_FEAT_RESET] = {
         .desc = "reset MSR (hv-reset)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_RESET_AVAILABLE}
         }
     },
     [HYPERV_FEAT_VPINDEX] = {
         .desc = "VP_INDEX MSR (hv-vpindex)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_VP_INDEX_AVAILABLE}
         }
     },
     [HYPERV_FEAT_RUNTIME] = {
         .desc = "VP_RUNTIME MSR (hv-runtime)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_VP_RUNTIME_AVAILABLE}
         }
     },
     [HYPERV_FEAT_SYNIC] = {
         .desc = "synthetic interrupt controller (hv-synic)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_SYNIC_AVAILABLE}
         }
     },
     [HYPERV_FEAT_STIMER] = {
         .desc = "synthetic timers (hv-stimer)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_SYNTIMERS_AVAILABLE}
         },
         .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME)
@@ -892,23 +893,23 @@ static struct {
     [HYPERV_FEAT_FREQUENCIES] = {
         .desc = "frequency MSRs (hv-frequencies)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_ACCESS_FREQUENCY_MSRS},
-            {.fw = FEAT_HYPERV_EDX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
              .bits = HV_FREQUENCY_MSRS_AVAILABLE}
         }
     },
     [HYPERV_FEAT_REENLIGHTENMENT] = {
         .desc = "reenlightenment MSRs (hv-reenlightenment)",
         .flags = {
-            {.fw = FEAT_HYPERV_EAX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
              .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL}
         }
     },
     [HYPERV_FEAT_TLBFLUSH] = {
         .desc = "paravirtualized TLB flush (hv-tlbflush)",
         .flags = {
-            {.fw = FEAT_HV_RECOMM_EAX,
+            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
              .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED |
              HV_EX_PROCESSOR_MASKS_RECOMMENDED}
         },
@@ -917,7 +918,7 @@ static struct {
     [HYPERV_FEAT_EVMCS] = {
         .desc = "enlightened VMCS (hv-evmcs)",
         .flags = {
-            {.fw = FEAT_HV_RECOMM_EAX,
+            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
              .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED}
         },
         .dependencies = BIT(HYPERV_FEAT_VAPIC)
@@ -925,7 +926,7 @@ static struct {
     [HYPERV_FEAT_IPI] = {
         .desc = "paravirtualized IPI (hv-ipi)",
         .flags = {
-            {.fw = FEAT_HV_RECOMM_EAX,
+            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
              .bits = HV_CLUSTER_IPI_RECOMMENDED |
              HV_EX_PROCESSOR_MASKS_RECOMMENDED}
         },
@@ -934,7 +935,7 @@ static struct {
     [HYPERV_FEAT_STIMER_DIRECT] = {
         .desc = "direct mode synthetic timers (hv-stimer-direct)",
         .flags = {
-            {.fw = FEAT_HYPERV_EDX,
+            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
              .bits = HV_STIMER_DIRECT_MODE_AVAILABLE}
         },
         .dependencies = BIT(HYPERV_FEAT_STIMER)
@@ -1108,38 +1109,34 @@ static uint32_t hv_cpuid_get_host(CPUState *cs, uint32_t func, int reg)
     return cpuid_entry_get_reg(entry, reg);
 }
 
-static uint32_t hv_cpuid_get_fw(CPUState *cs, int fw)
+static bool hyperv_feature_supported(CPUState *cs, int feature)
 {
-    uint32_t func;
-    int reg;
+    uint32_t func, bits;
+    int i, reg;
 
-    switch (fw) {
-    case FEAT_HYPERV_EAX:
-        reg = R_EAX;
-        func = HV_CPUID_FEATURES;
-        break;
-    case FEAT_HYPERV_EDX:
-        reg = R_EDX;
-        func = HV_CPUID_FEATURES;
-        break;
-    case FEAT_HV_RECOMM_EAX:
-        reg = R_EAX;
-        func = HV_CPUID_ENLIGHTMENT_INFO;
-        break;
-    default:
-        return 0;
+    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
+
+        func = kvm_hyperv_properties[feature].flags[i].func;
+        reg = kvm_hyperv_properties[feature].flags[i].reg;
+        bits = kvm_hyperv_properties[feature].flags[i].bits;
+
+        if (!func) {
+            continue;
+        }
+
+        if ((hv_cpuid_get_host(cs, func, reg) & bits) != bits) {
+            return false;
+        }
     }
 
-    return hv_cpuid_get_host(cs, func, reg);
+    return true;
 }
 
 static int hv_cpuid_check_and_set(CPUState *cs, int feature)
 {
     X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    uint32_t r, fw, bits;
     uint64_t deps;
-    int i, dep_feat;
+    int dep_feat;
 
     if (!hyperv_feat_enabled(cpu, feature) && !cpu->hyperv_passthrough) {
         return 0;
@@ -1158,27 +1155,15 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
         deps &= ~(1ull << dep_feat);
     }
 
-    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
-        fw = kvm_hyperv_properties[feature].flags[i].fw;
-        bits = kvm_hyperv_properties[feature].flags[i].bits;
-
-        if (!fw) {
-            continue;
-        }
-
-        r = hv_cpuid_get_fw(cs, fw);
-        if ((r & bits) != bits) {
-            if (hyperv_feat_enabled(cpu, feature)) {
-                fprintf(stderr,
-                        "Hyper-V %s is not supported by kernel\n",
-                        kvm_hyperv_properties[feature].desc);
-                return 1;
-            } else {
-                return 0;
-            }
+    if (!hyperv_feature_supported(cs, feature)) {
+        if (hyperv_feat_enabled(cpu, feature)) {
+            fprintf(stderr,
+                    "Hyper-V %s is not supported by kernel\n",
+                    kvm_hyperv_properties[feature].desc);
+            return 1;
+        } else {
+            return 0;
         }
-
-        env->features[fw] |= bits;
     }
 
     if (cpu->hyperv_passthrough) {
@@ -1188,6 +1173,32 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
     return 0;
 }
 
+static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    uint32_t r = 0;
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties); i++) {
+        if (!hyperv_feat_enabled(cpu, i)) {
+            continue;
+        }
+
+        for (j = 0; j < ARRAY_SIZE(kvm_hyperv_properties[i].flags); j++) {
+            if (kvm_hyperv_properties[i].flags[j].func != func) {
+                continue;
+            }
+            if (kvm_hyperv_properties[i].flags[j].reg != reg) {
+                continue;
+            }
+
+            r |= kvm_hyperv_properties[i].flags[j].bits;
+        }
+    }
+
+    return r;
+}
+
 /*
  * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in
  * case of success, errno < 0 in case of failure and 0 when no Hyper-V
@@ -1197,7 +1208,6 @@ static int hyperv_handle_properties(CPUState *cs,
                                     struct kvm_cpuid_entry2 *cpuid_ent)
 {
     X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
     struct kvm_cpuid_entry2 *c;
     uint32_t cpuid_i = 0;
     int r;
@@ -1219,9 +1229,7 @@ static int hyperv_handle_properties(CPUState *cs,
         }
 
         if (!r) {
-            env->features[FEAT_HV_RECOMM_EAX] |=
-                HV_ENLIGHTENED_VMCS_RECOMMENDED;
-            env->features[FEAT_HV_NESTED_EAX] = evmcs_version;
+            cpu->hyperv_nested[0] = evmcs_version;
         }
     }
 
@@ -1255,13 +1263,6 @@ static int hyperv_handle_properties(CPUState *cs,
         cpu->hyperv_version_id[3] =
             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX);
 
-        env->features[FEAT_HYPERV_EAX] =
-            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EAX);
-        env->features[FEAT_HYPERV_EBX] =
-            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EBX);
-        env->features[FEAT_HYPERV_EDX] =
-            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EDX);
-
         cpu->hv_max_vps = hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS,
                                             R_EAX);
         cpu->hyperv_limits[0] =
@@ -1271,21 +1272,8 @@ static int hyperv_handle_properties(CPUState *cs,
         cpu->hyperv_limits[2] =
             hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EDX);
 
-        env->features[FEAT_HV_RECOMM_EAX] =
-            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
         cpu->hyperv_spinlock_attempts =
             hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EBX);
-
-        env->features[FEAT_HV_NESTED_EAX] =
-            hv_cpuid_get_host(cs, HV_CPUID_NESTED_FEATURES, R_EAX);
-    }
-
-    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
-        env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING;
-    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
-        env->features[FEAT_HV_RECOMM_EAX] |=
-            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
-            HV_NO_NONARCH_CORESHARING;
     }
 
     /* Features */
@@ -1315,9 +1303,6 @@ static int hyperv_handle_properties(CPUState *cs,
         r |= 1;
     }
 
-    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
-    env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
-
     if (r) {
         return -ENOSYS;
     }
@@ -1346,15 +1331,25 @@ static int hyperv_handle_properties(CPUState *cs,
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_FEATURES;
-    c->eax = env->features[FEAT_HYPERV_EAX];
-    c->ebx = env->features[FEAT_HYPERV_EBX];
-    c->edx = env->features[FEAT_HYPERV_EDX];
+    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EAX);
+    c->ebx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EBX);
+    c->edx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EDX);
+
+    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
+    c->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
 
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_ENLIGHTMENT_INFO;
-    c->eax = env->features[FEAT_HV_RECOMM_EAX];
+    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
     c->ebx = cpu->hyperv_spinlock_attempts;
 
+    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
+        c->eax |= HV_NO_NONARCH_CORESHARING;
+    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
+        c->eax |= hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
+            HV_NO_NONARCH_CORESHARING;
+    }
+
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_IMPLEMENT_LIMITS;
     c->eax = cpu->hv_max_vps;
@@ -1374,7 +1369,7 @@ static int hyperv_handle_properties(CPUState *cs,
 
         c = &cpuid_ent[cpuid_i++];
         c->function = HV_CPUID_NESTED_FEATURES;
-        c->eax = env->features[FEAT_HV_NESTED_EAX];
+        c->eax = cpu->hyperv_nested[0];
     }
 
     return cpuid_i;
-- 
2.25.4

[PATCH RFC v2 13/19] i386: split hyperv_handle_properties() into hyperv_expand_features()/hyperv_fill_cpuids()

[Vitaly Kuznetsov]

The intention is to call hyperv_expand_features() early, before vCPUs
are created and use the acquired data later when we set guest visible
CPUID data.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 86a5cfa1e12b..0808355bfcfd 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1200,16 +1200,15 @@ static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
 }
 
 /*
- * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in
- * case of success, errno < 0 in case of failure and 0 when no Hyper-V
- * extentions are enabled.
+ * Expand Hyper-V CPU features. In partucular, check that all the requested
+ * features are supported by the host and the sanity of the configuration
+ * (that all the required dependencies are included). Also, this takes care
+ * of 'hv_passthrough' mode and fills the environment with all supported
+ * Hyper-V features.
  */
-static int hyperv_handle_properties(CPUState *cs,
-                                    struct kvm_cpuid_entry2 *cpuid_ent)
+static int hyperv_expand_features(CPUState *cs)
 {
     X86CPU *cpu = X86_CPU(cs);
-    struct kvm_cpuid_entry2 *c;
-    uint32_t cpuid_i = 0;
     int r;
 
     if (!hyperv_enabled(cpu))
@@ -1307,6 +1306,19 @@ static int hyperv_handle_properties(CPUState *cs,
         return -ENOSYS;
     }
 
+    return 0;
+}
+
+/*
+ * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent.
+ */
+static int hyperv_fill_cpuids(CPUState *cs,
+                              struct kvm_cpuid_entry2 *cpuid_ent)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    struct kvm_cpuid_entry2 *c;
+    uint32_t cpuid_i = 0;
+
     c = &cpuid_ent[cpuid_i++];
     c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
     c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ?
@@ -1514,11 +1526,13 @@ int kvm_arch_init_vcpu(CPUState *cs)
     env->apic_bus_freq = KVM_APIC_BUS_FREQUENCY;
 
     /* Paravirtualization CPUIDs */
-    r = hyperv_handle_properties(cs, cpuid_data.entries);
+    r = hyperv_expand_features(cs);
     if (r < 0) {
         return r;
-    } else if (r > 0) {
-        cpuid_i = r;
+    }
+
+    if (hyperv_enabled(cpu)) {
+        cpuid_i = hyperv_fill_cpuids(cs, cpuid_data.entries);
         kvm_base = KVM_CPUID_SIGNATURE_NEXT;
         has_msr_hv_hypercall = true;
     }
-- 
2.25.4

[PATCH RFC v2 14/19] i386: move eVMCS enablement to hyperv_init_vcpu()

[Vitaly Kuznetsov]

hyperv_expand_features() will be called before we create vCPU so
evmcs enablement should go away. hyperv_init_vcpu() looks like the
right place.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 60 +++++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 0808355bfcfd..181e034da701 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -976,6 +976,7 @@ static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
 {
     struct kvm_cpuid2 *cpuid;
     int max = 7; /* 0x40000000..0x40000005, 0x4000000A */
+    int i;
 
     /*
      * When the buffer is too small, KVM_GET_SUPPORTED_HV_CPUID fails with
@@ -985,6 +986,22 @@ static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
     while ((cpuid = try_get_hv_cpuid(cs, max)) == NULL) {
         max++;
     }
+
+    /*
+     * KVM_GET_SUPPORTED_HV_CPUID does not set EVMCS CPUID bit before
+     * KVM_CAP_HYPERV_ENLIGHTENED_VMCS is enabled but we want to get the
+     * information early, just check for the capability and set the bit
+     * manually.
+     */
+    if (kvm_check_extension(cs->kvm_state,
+                            KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) {
+        for (i = 0; i < cpuid->nent; i++) {
+            if (cpuid->entries[i].function == HV_CPUID_ENLIGHTMENT_INFO) {
+                cpuid->entries[i].eax |= HV_ENLIGHTENED_VMCS_RECOMMENDED;
+            }
+        }
+    }
+
     return cpuid;
 }
 
@@ -1214,24 +1231,6 @@ static int hyperv_expand_features(CPUState *cs)
     if (!hyperv_enabled(cpu))
         return 0;
 
-    if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ||
-        cpu->hyperv_passthrough) {
-        uint16_t evmcs_version;
-
-        r = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENLIGHTENED_VMCS, 0,
-                                (uintptr_t)&evmcs_version);
-
-        if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) && r) {
-            fprintf(stderr, "Hyper-V %s is not supported by kernel\n",
-                    kvm_hyperv_properties[HYPERV_FEAT_EVMCS].desc);
-            return -ENOSYS;
-        }
-
-        if (!r) {
-            cpu->hyperv_nested[0] = evmcs_version;
-        }
-    }
-
     if (cpu->hyperv_passthrough) {
         cpu->hyperv_vendor_id[0] =
             hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EBX);
@@ -1468,6 +1467,21 @@ static int hyperv_init_vcpu(X86CPU *cpu)
         }
     }
 
+    if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) {
+        uint16_t evmcs_version;
+
+        ret = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENLIGHTENED_VMCS, 0,
+                                  (uintptr_t)&evmcs_version);
+
+        if (ret < 0) {
+            fprintf(stderr, "Hyper-V %s is not supported by kernel\n",
+                    kvm_hyperv_properties[HYPERV_FEAT_EVMCS].desc);
+            return ret;
+        }
+
+        cpu->hyperv_nested[0] = evmcs_version;
+    }
+
     return 0;
 }
 
@@ -1532,6 +1546,11 @@ int kvm_arch_init_vcpu(CPUState *cs)
     }
 
     if (hyperv_enabled(cpu)) {
+        r = hyperv_init_vcpu(cpu);
+        if (r) {
+            return r;
+        }
+
         cpuid_i = hyperv_fill_cpuids(cs, cpuid_data.entries);
         kvm_base = KVM_CPUID_SIGNATURE_NEXT;
         has_msr_hv_hypercall = true;
@@ -1879,11 +1898,6 @@ int kvm_arch_init_vcpu(CPUState *cs)
 
     kvm_init_msrs(cpu);
 
-    r = hyperv_init_vcpu(cpu);
-    if (r) {
-        goto fail;
-    }
-
     return 0;
 
  fail:
-- 
2.25.4

[PATCH RFC v2 15/19] i386: switch hyperv_expand_features() to using error_setg()

[Vitaly Kuznetsov]

Use standard error_setg() mechanism in hyperv_expand_features().

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 101 ++++++++++++++++++++++++++++------------------
 1 file changed, 61 insertions(+), 40 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 181e034da701..32678154837f 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1149,7 +1149,7 @@ static bool hyperv_feature_supported(CPUState *cs, int feature)
     return true;
 }
 
-static int hv_cpuid_check_and_set(CPUState *cs, int feature)
+static int hv_cpuid_check_and_set(CPUState *cs, int feature, Error **errp)
 {
     X86CPU *cpu = X86_CPU(cs);
     uint64_t deps;
@@ -1163,20 +1163,18 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
     while (deps) {
         dep_feat = ctz64(deps);
         if (!(hyperv_feat_enabled(cpu, dep_feat))) {
-                fprintf(stderr,
-                        "Hyper-V %s requires Hyper-V %s\n",
-                        kvm_hyperv_properties[feature].desc,
-                        kvm_hyperv_properties[dep_feat].desc);
-                return 1;
+            error_setg(errp, "Hyper-V %s requires Hyper-V %s",
+                       kvm_hyperv_properties[feature].desc,
+                       kvm_hyperv_properties[dep_feat].desc);
+            return 1;
         }
         deps &= ~(1ull << dep_feat);
     }
 
     if (!hyperv_feature_supported(cs, feature)) {
         if (hyperv_feat_enabled(cpu, feature)) {
-            fprintf(stderr,
-                    "Hyper-V %s is not supported by kernel\n",
-                    kvm_hyperv_properties[feature].desc);
+            error_setg(errp, "Hyper-V %s is not supported by kernel",
+                       kvm_hyperv_properties[feature].desc);
             return 1;
         } else {
             return 0;
@@ -1223,13 +1221,12 @@ static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
  * of 'hv_passthrough' mode and fills the environment with all supported
  * Hyper-V features.
  */
-static int hyperv_expand_features(CPUState *cs)
+static void hyperv_expand_features(CPUState *cs, Error **errp)
 {
     X86CPU *cpu = X86_CPU(cs);
-    int r;
 
     if (!hyperv_enabled(cpu))
-        return 0;
+        return;
 
     if (cpu->hyperv_passthrough) {
         cpu->hyperv_vendor_id[0] =
@@ -1275,37 +1272,60 @@ static int hyperv_expand_features(CPUState *cs)
     }
 
     /* Features */
-    r = hv_cpuid_check_and_set(cs, HYPERV_FEAT_RELAXED);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_VAPIC);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_TIME);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_CRASH);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_RESET);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_VPINDEX);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_RUNTIME);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_SYNIC);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_FREQUENCIES);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_REENLIGHTENMENT);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_TLBFLUSH);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_EVMCS);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_IPI);
-    r |= hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER_DIRECT);
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_RELAXED, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_VAPIC, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_TIME, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_CRASH, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_RESET, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_VPINDEX, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_RUNTIME, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_SYNIC, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_FREQUENCIES, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_REENLIGHTENMENT, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_TLBFLUSH, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_EVMCS, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_IPI, errp)) {
+        return;
+    }
+    if (hv_cpuid_check_and_set(cs, HYPERV_FEAT_STIMER_DIRECT, errp)) {
+        return;
+    }
 
     /* Additional dependencies not covered by kvm_hyperv_properties[] */
     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) &&
         !cpu->hyperv_synic_kvm_only &&
         !hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)) {
-        fprintf(stderr, "Hyper-V %s requires Hyper-V %s\n",
-                kvm_hyperv_properties[HYPERV_FEAT_SYNIC].desc,
-                kvm_hyperv_properties[HYPERV_FEAT_VPINDEX].desc);
-        r |= 1;
-    }
-
-    if (r) {
-        return -ENOSYS;
+        error_setg(errp, "Hyper-V %s requires Hyper-V %s",
+                   kvm_hyperv_properties[HYPERV_FEAT_SYNIC].desc,
+                   kvm_hyperv_properties[HYPERV_FEAT_VPINDEX].desc);
     }
-
-    return 0;
 }
 
 /*
@@ -1540,9 +1560,10 @@ int kvm_arch_init_vcpu(CPUState *cs)
     env->apic_bus_freq = KVM_APIC_BUS_FREQUENCY;
 
     /* Paravirtualization CPUIDs */
-    r = hyperv_expand_features(cs);
-    if (r < 0) {
-        return r;
+    hyperv_expand_features(cs, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+        return -ENOSYS;
     }
 
     if (hyperv_enabled(cpu)) {
-- 
2.25.4

[PATCH RFC v2 16/19] i386: adjust the expected KVM_GET_SUPPORTED_HV_CPUID array size

[Vitaly Kuznetsov]

SYNDBG leaves were recently (Linux-5.8) added to KVM but we haven't
updated the expected size of KVM_GET_SUPPORTED_HV_CPUID output in
KVM so we now make serveral tries before succeeding. Update the
default.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 32678154837f..39a6e89dbb60 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -975,7 +975,8 @@ static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max)
 static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
 {
     struct kvm_cpuid2 *cpuid;
-    int max = 7; /* 0x40000000..0x40000005, 0x4000000A */
+    /* 0x40000000..0x40000005, 0x4000000A, 0x40000080..0x40000080 leaves */
+    int max = 10;
     int i;
 
     /*
-- 
2.25.4

[PATCH RFC v2 17/19] i386: prefer system KVM_GET_SUPPORTED_HV_CPUID ioctl over vCPU's one

[Vitaly Kuznetsov]

KVM_GET_SUPPORTED_HV_CPUID was made a system wide ioctl which can be called
prior to creating vCPUs and we are going to use that to expand Hyper-V cpu
features early. Use it when it is supported by KVM.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 39a6e89dbb60..af59b5f1a935 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -942,7 +942,8 @@ static struct {
     },
 };
 
-static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max)
+static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max,
+                                           bool do_sys_ioctl)
 {
     struct kvm_cpuid2 *cpuid;
     int r, size;
@@ -951,7 +952,11 @@ static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max)
     cpuid = g_malloc0(size);
     cpuid->nent = max;
 
-    r = kvm_vcpu_ioctl(cs, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
+    if (do_sys_ioctl) {
+        r = kvm_ioctl(kvm_state, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
+    } else {
+        r = kvm_vcpu_ioctl(cs, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
+    }
     if (r == 0 && cpuid->nent >= max) {
         r = -E2BIG;
     }
@@ -978,13 +983,17 @@ static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
     /* 0x40000000..0x40000005, 0x4000000A, 0x40000080..0x40000080 leaves */
     int max = 10;
     int i;
+    bool do_sys_ioctl;
+
+    do_sys_ioctl =
+        kvm_check_extension(kvm_state, KVM_CAP_SYS_HYPERV_CPUID) > 0;
 
     /*
      * When the buffer is too small, KVM_GET_SUPPORTED_HV_CPUID fails with
      * -E2BIG, however, it doesn't report back the right size. Keep increasing
      * it and re-trying until we succeed.
      */
-    while ((cpuid = try_get_hv_cpuid(cs, max)) == NULL) {
+    while ((cpuid = try_get_hv_cpuid(cs, max, do_sys_ioctl)) == NULL) {
         max++;
     }
 
@@ -994,7 +1003,7 @@ static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
      * information early, just check for the capability and set the bit
      * manually.
      */
-    if (kvm_check_extension(cs->kvm_state,
+    if (!do_sys_ioctl && kvm_check_extension(cs->kvm_state,
                             KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) {
         for (i = 0; i < cpuid->nent; i++) {
             if (cpuid->entries[i].function == HV_CPUID_ENLIGHTMENT_INFO) {
-- 
2.25.4

[PATCH RFC v2 18/19] i386: use global kvm_state in hyperv_enabled() check

[Vitaly Kuznetsov]

There is no need to use vCPU-specific kvm state in hyperv_enabled() check
and we need to do that when feature expansion happens early, before vCPU
specific KVM state is created.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/kvm.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index af59b5f1a935..2c62122a1bc8 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -729,8 +729,7 @@ unsigned long kvm_arch_vcpu_id(CPUState *cs)
 
 static bool hyperv_enabled(X86CPU *cpu)
 {
-    CPUState *cs = CPU(cpu);
-    return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 &&
+    return kvm_check_extension(kvm_state, KVM_CAP_HYPERV) > 0 &&
         ((cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_NOTIFY) ||
          cpu->hyperv_features || cpu->hyperv_passthrough);
 }
-- 
2.25.4

[PATCH RFC v2 19/19] i386: expand Hyper-V features during CPU feature expansion time

[Vitaly Kuznetsov]

To make Hyper-V features appear in e.g. QMP query-cpu-model-expansion we
need to expand and set the corresponding CPUID leaves early. Modify
x86_cpu_get_supported_feature_word() to call newly intoduced Hyper-V
specific kvm_hv_get_supported_cpuid() instead of
kvm_arch_get_supported_cpuid(). We can't use kvm_arch_get_supported_cpuid()
as Hyper-V specific CPUID leaves intersect with KVM's.

Note, early expansion will only happen when KVM supports system wide
KVM_GET_SUPPORTED_HV_CPUID ioctl (KVM_CAP_SYS_HYPERV_CPUID).

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 target/i386/cpu.c      |  4 ++++
 target/i386/kvm-stub.c |  5 +++++
 target/i386/kvm.c      | 15 ++++++++++++---
 target/i386/kvm_i386.h |  1 +
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index a178db255641..272091eab057 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -6345,6 +6345,10 @@ static void x86_cpu_expand_features(X86CPU *cpu, Error **errp)
     if (env->cpuid_xlevel2 == UINT32_MAX) {
         env->cpuid_xlevel2 = env->cpuid_min_xlevel2;
     }
+
+    if (kvm_enabled()) {
+        kvm_hyperv_expand_features(cpu, errp);
+    }
 }
 
 /*
diff --git a/target/i386/kvm-stub.c b/target/i386/kvm-stub.c
index 872ef7df4c88..4abb5afa1168 100644
--- a/target/i386/kvm-stub.c
+++ b/target/i386/kvm-stub.c
@@ -44,3 +44,8 @@ bool kvm_hv_vpindex_settable(void)
 {
     return false;
 }
+
+void kvm_hyperv_expand_features(X86CPU *cpu, Error **errp)
+{
+    return;
+}
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 2c62122a1bc8..ba99aecb5a64 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -1230,13 +1230,22 @@ static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
  * of 'hv_passthrough' mode and fills the environment with all supported
  * Hyper-V features.
  */
-static void hyperv_expand_features(CPUState *cs, Error **errp)
+void kvm_hyperv_expand_features(X86CPU *cpu, Error **errp)
 {
-    X86CPU *cpu = X86_CPU(cs);
+    CPUState *cs = CPU(cpu);
 
     if (!hyperv_enabled(cpu))
         return;
 
+    /*
+     * When kvm_hyperv_expand_features is called at CPU feature expansion
+     * time per-CPU kvm_state is not available yet so we can only proceed
+     * when KVM_CAP_SYS_HYPERV_CPUID is supported.
+     */
+    if (!cs->kvm_state &&
+        !kvm_check_extension(kvm_state, KVM_CAP_SYS_HYPERV_CPUID))
+        return;
+
     if (cpu->hyperv_passthrough) {
         cpu->hyperv_vendor_id[0] =
             hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EBX);
@@ -1569,7 +1578,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
     env->apic_bus_freq = KVM_APIC_BUS_FREQUENCY;
 
     /* Paravirtualization CPUIDs */
-    hyperv_expand_features(cs, &local_err);
+    kvm_hyperv_expand_features(cpu, &local_err);
     if (local_err) {
         error_report_err(local_err);
         return -ENOSYS;
diff --git a/target/i386/kvm_i386.h b/target/i386/kvm_i386.h
index 064b8798a26c..126a1bcef2dd 100644
--- a/target/i386/kvm_i386.h
+++ b/target/i386/kvm_i386.h
@@ -47,5 +47,6 @@ bool kvm_has_x2apic_api(void);
 bool kvm_has_waitpkg(void);
 
 bool kvm_hv_vpindex_settable(void);
+void kvm_hyperv_expand_features(X86CPU *cpu, Error **errp);
 
 #endif
-- 
2.25.4

Re: [PATCH RFC v2 12/19] i386: drop FEAT_HYPERV feature leaves

[Eduardo Habkost]

On Wed, Sep 30, 2020 at 03:40:20PM +0200, Vitaly Kuznetsov wrote:
> Hyper-V feature leaves are weird. We have some of them in
> feature_word_info[] array but we don't use feature_word_info
> magic to enable them. Neither do we use feature_dependencies[]
> mechanism to validate the configuration as it doesn't allign
> well with Hyper-V's many-to-many dependency chains. Some of
> the feature leaves hold not only feature bits, but also values.
> E.g. FEAT_HV_NESTED_EAX contains both features and the supported
> Enlightened VMCS range.
> 
> Hyper-V features are already represented in 'struct X86CPU' with
> uint64_t hyperv_features so duplicating them in env->features adds
> little (or zero) benefits. THe other half of Hyper-V emulation features
> is also stored with values in hyperv_vendor_id[], hyperv_limits[],...
> so env->features[] is already incomplete.
> 
> Remove Hyper-V feature leaves from env->features[] completely.
> kvm_hyperv_properties[] is converted to using raw CPUID func/reg
> pairs for features, this allows us to get rid of hv_cpuid_get_fw()
> conversion.
> 
> Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>

So, my first reaction to this was "we don't want to revert this,
having hyperv features in env->features is useful", but seeing
that there's only one line where env->features[] is actually used
in target/i386/cpu.c, I agree with this change.  We still have
the option to recreate those FeatureWord entries in the future,
if necessary.

I'm still reviewing the patch and it looks good so far, but I
have one question: have you considered doing this at the
beginning of the series?  There are
  env->features[...] = hv_cpuid_get_host(cs, ...)
lines you have added in the previous patches, just to remove them
in this patch.

> ---
>  target/i386/cpu.c |  90 +----------------------
>  target/i386/cpu.h |   6 +-
>  target/i386/kvm.c | 181 ++++++++++++++++++++++------------------------
>  3 files changed, 90 insertions(+), 187 deletions(-)
> 
> diff --git a/target/i386/cpu.c b/target/i386/cpu.c
> index 8ec0af0a6d48..a178db255641 100644
> --- a/target/i386/cpu.c
> +++ b/target/i386/cpu.c
> @@ -828,94 +828,6 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
>           */
>          .no_autoenable_flags = ~0U,
>      },
> -    /*
> -     * .feat_names are commented out for Hyper-V enlightenments because we
> -     * don't want to have two different ways for enabling them on QEMU command
> -     * line. Some features (e.g. "hyperv_time", "hyperv_vapic", ...) require
> -     * enabling several feature bits simultaneously, exposing these bits
> -     * individually may just confuse guests.
> -     */
> -    [FEAT_HYPERV_EAX] = {
> -        .type = CPUID_FEATURE_WORD,
> -        .feat_names = {
> -            NULL /* hv_msr_vp_runtime_access */, NULL /* hv_msr_time_refcount_access */,
> -            NULL /* hv_msr_synic_access */, NULL /* hv_msr_stimer_access */,
> -            NULL /* hv_msr_apic_access */, NULL /* hv_msr_hypercall_access */,
> -            NULL /* hv_vpindex_access */, NULL /* hv_msr_reset_access */,
> -            NULL /* hv_msr_stats_access */, NULL /* hv_reftsc_access */,
> -            NULL /* hv_msr_idle_access */, NULL /* hv_msr_frequency_access */,
> -            NULL /* hv_msr_debug_access */, NULL /* hv_msr_reenlightenment_access */,
> -            NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -        },
> -        .cpuid = { .eax = 0x40000003, .reg = R_EAX, },
> -    },
> -    [FEAT_HYPERV_EBX] = {
> -        .type = CPUID_FEATURE_WORD,
> -        .feat_names = {
> -            NULL /* hv_create_partitions */, NULL /* hv_access_partition_id */,
> -            NULL /* hv_access_memory_pool */, NULL /* hv_adjust_message_buffers */,
> -            NULL /* hv_post_messages */, NULL /* hv_signal_events */,
> -            NULL /* hv_create_port */, NULL /* hv_connect_port */,
> -            NULL /* hv_access_stats */, NULL, NULL, NULL /* hv_debugging */,
> -            NULL /* hv_cpu_power_management */, NULL /* hv_configure_profiler */,
> -            NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -        },
> -        .cpuid = { .eax = 0x40000003, .reg = R_EBX, },
> -    },
> -    [FEAT_HYPERV_EDX] = {
> -        .type = CPUID_FEATURE_WORD,
> -        .feat_names = {
> -            NULL /* hv_mwait */, NULL /* hv_guest_debugging */,
> -            NULL /* hv_perf_monitor */, NULL /* hv_cpu_dynamic_part */,
> -            NULL /* hv_hypercall_params_xmm */, NULL /* hv_guest_idle_state */,
> -            NULL, NULL,
> -            NULL, NULL, NULL /* hv_guest_crash_msr */, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -        },
> -        .cpuid = { .eax = 0x40000003, .reg = R_EDX, },
> -    },
> -    [FEAT_HV_RECOMM_EAX] = {
> -        .type = CPUID_FEATURE_WORD,
> -        .feat_names = {
> -            NULL /* hv_recommend_pv_as_switch */,
> -            NULL /* hv_recommend_pv_tlbflush_local */,
> -            NULL /* hv_recommend_pv_tlbflush_remote */,
> -            NULL /* hv_recommend_msr_apic_access */,
> -            NULL /* hv_recommend_msr_reset */,
> -            NULL /* hv_recommend_relaxed_timing */,
> -            NULL /* hv_recommend_dma_remapping */,
> -            NULL /* hv_recommend_int_remapping */,
> -            NULL /* hv_recommend_x2apic_msrs */,
> -            NULL /* hv_recommend_autoeoi_deprecation */,
> -            NULL /* hv_recommend_pv_ipi */,
> -            NULL /* hv_recommend_ex_hypercalls */,
> -            NULL /* hv_hypervisor_is_nested */,
> -            NULL /* hv_recommend_int_mbec */,
> -            NULL /* hv_recommend_evmcs */,
> -            NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -            NULL, NULL, NULL, NULL,
> -        },
> -        .cpuid = { .eax = 0x40000004, .reg = R_EAX, },
> -    },
> -    [FEAT_HV_NESTED_EAX] = {
> -        .type = CPUID_FEATURE_WORD,
> -        .cpuid = { .eax = 0x4000000A, .reg = R_EAX, },
> -    },
>      [FEAT_SVM] = {
>          .type = CPUID_FEATURE_WORD,
>          .feat_names = {
> @@ -6906,7 +6818,7 @@ static GuestPanicInformation *x86_cpu_get_crash_info(CPUState *cs)
>      CPUX86State *env = &cpu->env;
>      GuestPanicInformation *panic_info = NULL;
>  
> -    if (env->features[FEAT_HYPERV_EDX] & HV_GUEST_CRASH_MSR_AVAILABLE) {
> +    if (hyperv_feat_enabled(cpu, HYPERV_FEAT_CRASH)) {
>          panic_info = g_malloc0(sizeof(GuestPanicInformation));
>  
>          panic_info->type = GUEST_PANIC_INFORMATION_TYPE_HYPER_V;
> diff --git a/target/i386/cpu.h b/target/i386/cpu.h
> index 187e4a09e65d..b07325745bcb 100644
> --- a/target/i386/cpu.h
> +++ b/target/i386/cpu.h
> @@ -519,11 +519,6 @@ typedef enum FeatureWord {
>      FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
>      FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
>      FEAT_KVM_HINTS,     /* CPUID[4000_0001].EDX */
> -    FEAT_HYPERV_EAX,    /* CPUID[4000_0003].EAX */
> -    FEAT_HYPERV_EBX,    /* CPUID[4000_0003].EBX */
> -    FEAT_HYPERV_EDX,    /* CPUID[4000_0003].EDX */
> -    FEAT_HV_RECOMM_EAX, /* CPUID[4000_0004].EAX */
> -    FEAT_HV_NESTED_EAX, /* CPUID[4000_000A].EAX */
>      FEAT_SVM,           /* CPUID[8000_000A].EDX */
>      FEAT_XSAVE,         /* CPUID[EAX=0xd,ECX=1].EAX */
>      FEAT_6_EAX,         /* CPUID[6].EAX */
> @@ -1663,6 +1658,7 @@ struct X86CPU {
>      uint32_t hyperv_interface_id[4];
>      uint32_t hyperv_version_id[4];
>      uint32_t hyperv_limits[3];
> +    uint32_t hyperv_nested[4];
>  
>      bool check_cpuid;
>      bool enforce_cpuid;
> diff --git a/target/i386/kvm.c b/target/i386/kvm.c
> index 0e6eef6a52c2..86a5cfa1e12b 100644
> --- a/target/i386/kvm.c
> +++ b/target/i386/kvm.c
> @@ -815,7 +815,8 @@ static bool tsc_is_stable_and_known(CPUX86State *env)
>  static struct {
>      const char *desc;
>      struct {
> -        uint32_t fw;
> +        uint32_t func;
> +        int reg;
>          uint32_t bits;
>      } flags[2];
>      uint64_t dependencies;
> @@ -823,25 +824,25 @@ static struct {
>      [HYPERV_FEAT_RELAXED] = {
>          .desc = "relaxed timing (hv-relaxed)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_HYPERCALL_AVAILABLE},
> -            {.fw = FEAT_HV_RECOMM_EAX,
> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>               .bits = HV_RELAXED_TIMING_RECOMMENDED}
>          }
>      },
>      [HYPERV_FEAT_VAPIC] = {
>          .desc = "virtual APIC (hv-vapic)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_HYPERCALL_AVAILABLE | HV_APIC_ACCESS_AVAILABLE},
> -            {.fw = FEAT_HV_RECOMM_EAX,
> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>               .bits = HV_APIC_ACCESS_RECOMMENDED}
>          }
>      },
>      [HYPERV_FEAT_TIME] = {
>          .desc = "clocksources (hv-time)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_HYPERCALL_AVAILABLE | HV_TIME_REF_COUNT_AVAILABLE |
>               HV_REFERENCE_TSC_AVAILABLE}
>          }
> @@ -849,42 +850,42 @@ static struct {
>      [HYPERV_FEAT_CRASH] = {
>          .desc = "crash MSRs (hv-crash)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EDX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>               .bits = HV_GUEST_CRASH_MSR_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_RESET] = {
>          .desc = "reset MSR (hv-reset)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_RESET_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_VPINDEX] = {
>          .desc = "VP_INDEX MSR (hv-vpindex)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_VP_INDEX_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_RUNTIME] = {
>          .desc = "VP_RUNTIME MSR (hv-runtime)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_VP_RUNTIME_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_SYNIC] = {
>          .desc = "synthetic interrupt controller (hv-synic)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_SYNIC_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_STIMER] = {
>          .desc = "synthetic timers (hv-stimer)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_SYNTIMERS_AVAILABLE}
>          },
>          .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME)
> @@ -892,23 +893,23 @@ static struct {
>      [HYPERV_FEAT_FREQUENCIES] = {
>          .desc = "frequency MSRs (hv-frequencies)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_ACCESS_FREQUENCY_MSRS},
> -            {.fw = FEAT_HYPERV_EDX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>               .bits = HV_FREQUENCY_MSRS_AVAILABLE}
>          }
>      },
>      [HYPERV_FEAT_REENLIGHTENMENT] = {
>          .desc = "reenlightenment MSRs (hv-reenlightenment)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EAX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>               .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL}
>          }
>      },
>      [HYPERV_FEAT_TLBFLUSH] = {
>          .desc = "paravirtualized TLB flush (hv-tlbflush)",
>          .flags = {
> -            {.fw = FEAT_HV_RECOMM_EAX,
> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>               .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED |
>               HV_EX_PROCESSOR_MASKS_RECOMMENDED}
>          },
> @@ -917,7 +918,7 @@ static struct {
>      [HYPERV_FEAT_EVMCS] = {
>          .desc = "enlightened VMCS (hv-evmcs)",
>          .flags = {
> -            {.fw = FEAT_HV_RECOMM_EAX,
> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>               .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED}
>          },
>          .dependencies = BIT(HYPERV_FEAT_VAPIC)
> @@ -925,7 +926,7 @@ static struct {
>      [HYPERV_FEAT_IPI] = {
>          .desc = "paravirtualized IPI (hv-ipi)",
>          .flags = {
> -            {.fw = FEAT_HV_RECOMM_EAX,
> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>               .bits = HV_CLUSTER_IPI_RECOMMENDED |
>               HV_EX_PROCESSOR_MASKS_RECOMMENDED}
>          },
> @@ -934,7 +935,7 @@ static struct {
>      [HYPERV_FEAT_STIMER_DIRECT] = {
>          .desc = "direct mode synthetic timers (hv-stimer-direct)",
>          .flags = {
> -            {.fw = FEAT_HYPERV_EDX,
> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>               .bits = HV_STIMER_DIRECT_MODE_AVAILABLE}
>          },
>          .dependencies = BIT(HYPERV_FEAT_STIMER)
> @@ -1108,38 +1109,34 @@ static uint32_t hv_cpuid_get_host(CPUState *cs, uint32_t func, int reg)
>      return cpuid_entry_get_reg(entry, reg);
>  }
>  
> -static uint32_t hv_cpuid_get_fw(CPUState *cs, int fw)
> +static bool hyperv_feature_supported(CPUState *cs, int feature)
>  {
> -    uint32_t func;
> -    int reg;
> +    uint32_t func, bits;
> +    int i, reg;
>  
> -    switch (fw) {
> -    case FEAT_HYPERV_EAX:
> -        reg = R_EAX;
> -        func = HV_CPUID_FEATURES;
> -        break;
> -    case FEAT_HYPERV_EDX:
> -        reg = R_EDX;
> -        func = HV_CPUID_FEATURES;
> -        break;
> -    case FEAT_HV_RECOMM_EAX:
> -        reg = R_EAX;
> -        func = HV_CPUID_ENLIGHTMENT_INFO;
> -        break;
> -    default:
> -        return 0;
> +    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
> +
> +        func = kvm_hyperv_properties[feature].flags[i].func;
> +        reg = kvm_hyperv_properties[feature].flags[i].reg;
> +        bits = kvm_hyperv_properties[feature].flags[i].bits;
> +
> +        if (!func) {
> +            continue;
> +        }
> +
> +        if ((hv_cpuid_get_host(cs, func, reg) & bits) != bits) {
> +            return false;
> +        }
>      }
>  
> -    return hv_cpuid_get_host(cs, func, reg);
> +    return true;
>  }
>  
>  static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>  {
>      X86CPU *cpu = X86_CPU(cs);
> -    CPUX86State *env = &cpu->env;
> -    uint32_t r, fw, bits;
>      uint64_t deps;
> -    int i, dep_feat;
> +    int dep_feat;
>  
>      if (!hyperv_feat_enabled(cpu, feature) && !cpu->hyperv_passthrough) {
>          return 0;
> @@ -1158,27 +1155,15 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>          deps &= ~(1ull << dep_feat);
>      }
>  
> -    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
> -        fw = kvm_hyperv_properties[feature].flags[i].fw;
> -        bits = kvm_hyperv_properties[feature].flags[i].bits;
> -
> -        if (!fw) {
> -            continue;
> -        }
> -
> -        r = hv_cpuid_get_fw(cs, fw);
> -        if ((r & bits) != bits) {
> -            if (hyperv_feat_enabled(cpu, feature)) {
> -                fprintf(stderr,
> -                        "Hyper-V %s is not supported by kernel\n",
> -                        kvm_hyperv_properties[feature].desc);
> -                return 1;
> -            } else {
> -                return 0;
> -            }
> +    if (!hyperv_feature_supported(cs, feature)) {
> +        if (hyperv_feat_enabled(cpu, feature)) {
> +            fprintf(stderr,
> +                    "Hyper-V %s is not supported by kernel\n",
> +                    kvm_hyperv_properties[feature].desc);
> +            return 1;
> +        } else {
> +            return 0;
>          }
> -
> -        env->features[fw] |= bits;
>      }
>  
>      if (cpu->hyperv_passthrough) {
> @@ -1188,6 +1173,32 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>      return 0;
>  }
>  
> +static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
> +{
> +    X86CPU *cpu = X86_CPU(cs);
> +    uint32_t r = 0;
> +    int i, j;
> +
> +    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties); i++) {
> +        if (!hyperv_feat_enabled(cpu, i)) {
> +            continue;
> +        }
> +
> +        for (j = 0; j < ARRAY_SIZE(kvm_hyperv_properties[i].flags); j++) {
> +            if (kvm_hyperv_properties[i].flags[j].func != func) {
> +                continue;
> +            }
> +            if (kvm_hyperv_properties[i].flags[j].reg != reg) {
> +                continue;
> +            }
> +
> +            r |= kvm_hyperv_properties[i].flags[j].bits;
> +        }
> +    }
> +
> +    return r;
> +}
> +
>  /*
>   * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in
>   * case of success, errno < 0 in case of failure and 0 when no Hyper-V
> @@ -1197,7 +1208,6 @@ static int hyperv_handle_properties(CPUState *cs,
>                                      struct kvm_cpuid_entry2 *cpuid_ent)
>  {
>      X86CPU *cpu = X86_CPU(cs);
> -    CPUX86State *env = &cpu->env;
>      struct kvm_cpuid_entry2 *c;
>      uint32_t cpuid_i = 0;
>      int r;
> @@ -1219,9 +1229,7 @@ static int hyperv_handle_properties(CPUState *cs,
>          }
>  
>          if (!r) {
> -            env->features[FEAT_HV_RECOMM_EAX] |=
> -                HV_ENLIGHTENED_VMCS_RECOMMENDED;
> -            env->features[FEAT_HV_NESTED_EAX] = evmcs_version;
> +            cpu->hyperv_nested[0] = evmcs_version;
>          }
>      }
>  
> @@ -1255,13 +1263,6 @@ static int hyperv_handle_properties(CPUState *cs,
>          cpu->hyperv_version_id[3] =
>              hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX);
>  
> -        env->features[FEAT_HYPERV_EAX] =
> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EAX);
> -        env->features[FEAT_HYPERV_EBX] =
> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EBX);
> -        env->features[FEAT_HYPERV_EDX] =
> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EDX);
> -
>          cpu->hv_max_vps = hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS,
>                                              R_EAX);
>          cpu->hyperv_limits[0] =
> @@ -1271,21 +1272,8 @@ static int hyperv_handle_properties(CPUState *cs,
>          cpu->hyperv_limits[2] =
>              hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EDX);
>  
> -        env->features[FEAT_HV_RECOMM_EAX] =
> -            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
>          cpu->hyperv_spinlock_attempts =
>              hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EBX);
> -
> -        env->features[FEAT_HV_NESTED_EAX] =
> -            hv_cpuid_get_host(cs, HV_CPUID_NESTED_FEATURES, R_EAX);
> -    }
> -
> -    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
> -        env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING;
> -    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
> -        env->features[FEAT_HV_RECOMM_EAX] |=
> -            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
> -            HV_NO_NONARCH_CORESHARING;
>      }
>  
>      /* Features */
> @@ -1315,9 +1303,6 @@ static int hyperv_handle_properties(CPUState *cs,
>          r |= 1;
>      }
>  
> -    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
> -    env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
> -
>      if (r) {
>          return -ENOSYS;
>      }
> @@ -1346,15 +1331,25 @@ static int hyperv_handle_properties(CPUState *cs,
>  
>      c = &cpuid_ent[cpuid_i++];
>      c->function = HV_CPUID_FEATURES;
> -    c->eax = env->features[FEAT_HYPERV_EAX];
> -    c->ebx = env->features[FEAT_HYPERV_EBX];
> -    c->edx = env->features[FEAT_HYPERV_EDX];
> +    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EAX);
> +    c->ebx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EBX);
> +    c->edx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EDX);
> +
> +    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
> +    c->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
>  
>      c = &cpuid_ent[cpuid_i++];
>      c->function = HV_CPUID_ENLIGHTMENT_INFO;
> -    c->eax = env->features[FEAT_HV_RECOMM_EAX];
> +    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
>      c->ebx = cpu->hyperv_spinlock_attempts;
>  
> +    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
> +        c->eax |= HV_NO_NONARCH_CORESHARING;
> +    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
> +        c->eax |= hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
> +            HV_NO_NONARCH_CORESHARING;
> +    }
> +
>      c = &cpuid_ent[cpuid_i++];
>      c->function = HV_CPUID_IMPLEMENT_LIMITS;
>      c->eax = cpu->hv_max_vps;
> @@ -1374,7 +1369,7 @@ static int hyperv_handle_properties(CPUState *cs,
>  
>          c = &cpuid_ent[cpuid_i++];
>          c->function = HV_CPUID_NESTED_FEATURES;
> -        c->eax = env->features[FEAT_HV_NESTED_EAX];
> +        c->eax = cpu->hyperv_nested[0];
>      }
>  
>      return cpuid_i;
> -- 
> 2.25.4
> 

-- 
Eduardo

Re: [PATCH RFC v2 12/19] i386: drop FEAT_HYPERV feature leaves

[Vitaly Kuznetsov]

Eduardo Habkost <ehabkost@redhat.com> writes:

> On Wed, Sep 30, 2020 at 03:40:20PM +0200, Vitaly Kuznetsov wrote:
>> Hyper-V feature leaves are weird. We have some of them in
>> feature_word_info[] array but we don't use feature_word_info
>> magic to enable them. Neither do we use feature_dependencies[]
>> mechanism to validate the configuration as it doesn't allign
>> well with Hyper-V's many-to-many dependency chains. Some of
>> the feature leaves hold not only feature bits, but also values.
>> E.g. FEAT_HV_NESTED_EAX contains both features and the supported
>> Enlightened VMCS range.
>> 
>> Hyper-V features are already represented in 'struct X86CPU' with
>> uint64_t hyperv_features so duplicating them in env->features adds
>> little (or zero) benefits. THe other half of Hyper-V emulation features
>> is also stored with values in hyperv_vendor_id[], hyperv_limits[],...
>> so env->features[] is already incomplete.
>> 
>> Remove Hyper-V feature leaves from env->features[] completely.
>> kvm_hyperv_properties[] is converted to using raw CPUID func/reg
>> pairs for features, this allows us to get rid of hv_cpuid_get_fw()
>> conversion.
>> 
>> Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
>
> So, my first reaction to this was "we don't want to revert this,
> having hyperv features in env->features is useful", but seeing
> that there's only one line where env->features[] is actually used
> in target/i386/cpu.c, I agree with this change.  We still have
> the option to recreate those FeatureWord entries in the future,
> if necessary.
>
> I'm still reviewing the patch and it looks good so far, but I
> have one question: have you considered doing this at the
> beginning of the series?  There are
>   env->features[...] = hv_cpuid_get_host(cs, ...)
> lines you have added in the previous patches, just to remove them
> in this patch.
>

In case the general idea (removing Hyper-V feature leaves from
env->features[]) is acceptable I can try to minimize code churn in
v1. Maybe separating hv_build_cpuid_leaf() into its own patch will make
the removal cleaner, I'll definitely give it a try.

>> ---
>>  target/i386/cpu.c |  90 +----------------------
>>  target/i386/cpu.h |   6 +-
>>  target/i386/kvm.c | 181 ++++++++++++++++++++++------------------------
>>  3 files changed, 90 insertions(+), 187 deletions(-)
>> 
>> diff --git a/target/i386/cpu.c b/target/i386/cpu.c
>> index 8ec0af0a6d48..a178db255641 100644
>> --- a/target/i386/cpu.c
>> +++ b/target/i386/cpu.c
>> @@ -828,94 +828,6 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
>>           */
>>          .no_autoenable_flags = ~0U,
>>      },
>> -    /*
>> -     * .feat_names are commented out for Hyper-V enlightenments because we
>> -     * don't want to have two different ways for enabling them on QEMU command
>> -     * line. Some features (e.g. "hyperv_time", "hyperv_vapic", ...) require
>> -     * enabling several feature bits simultaneously, exposing these bits
>> -     * individually may just confuse guests.
>> -     */
>> -    [FEAT_HYPERV_EAX] = {
>> -        .type = CPUID_FEATURE_WORD,
>> -        .feat_names = {
>> -            NULL /* hv_msr_vp_runtime_access */, NULL /* hv_msr_time_refcount_access */,
>> -            NULL /* hv_msr_synic_access */, NULL /* hv_msr_stimer_access */,
>> -            NULL /* hv_msr_apic_access */, NULL /* hv_msr_hypercall_access */,
>> -            NULL /* hv_vpindex_access */, NULL /* hv_msr_reset_access */,
>> -            NULL /* hv_msr_stats_access */, NULL /* hv_reftsc_access */,
>> -            NULL /* hv_msr_idle_access */, NULL /* hv_msr_frequency_access */,
>> -            NULL /* hv_msr_debug_access */, NULL /* hv_msr_reenlightenment_access */,
>> -            NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -        },
>> -        .cpuid = { .eax = 0x40000003, .reg = R_EAX, },
>> -    },
>> -    [FEAT_HYPERV_EBX] = {
>> -        .type = CPUID_FEATURE_WORD,
>> -        .feat_names = {
>> -            NULL /* hv_create_partitions */, NULL /* hv_access_partition_id */,
>> -            NULL /* hv_access_memory_pool */, NULL /* hv_adjust_message_buffers */,
>> -            NULL /* hv_post_messages */, NULL /* hv_signal_events */,
>> -            NULL /* hv_create_port */, NULL /* hv_connect_port */,
>> -            NULL /* hv_access_stats */, NULL, NULL, NULL /* hv_debugging */,
>> -            NULL /* hv_cpu_power_management */, NULL /* hv_configure_profiler */,
>> -            NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -        },
>> -        .cpuid = { .eax = 0x40000003, .reg = R_EBX, },
>> -    },
>> -    [FEAT_HYPERV_EDX] = {
>> -        .type = CPUID_FEATURE_WORD,
>> -        .feat_names = {
>> -            NULL /* hv_mwait */, NULL /* hv_guest_debugging */,
>> -            NULL /* hv_perf_monitor */, NULL /* hv_cpu_dynamic_part */,
>> -            NULL /* hv_hypercall_params_xmm */, NULL /* hv_guest_idle_state */,
>> -            NULL, NULL,
>> -            NULL, NULL, NULL /* hv_guest_crash_msr */, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -        },
>> -        .cpuid = { .eax = 0x40000003, .reg = R_EDX, },
>> -    },
>> -    [FEAT_HV_RECOMM_EAX] = {
>> -        .type = CPUID_FEATURE_WORD,
>> -        .feat_names = {
>> -            NULL /* hv_recommend_pv_as_switch */,
>> -            NULL /* hv_recommend_pv_tlbflush_local */,
>> -            NULL /* hv_recommend_pv_tlbflush_remote */,
>> -            NULL /* hv_recommend_msr_apic_access */,
>> -            NULL /* hv_recommend_msr_reset */,
>> -            NULL /* hv_recommend_relaxed_timing */,
>> -            NULL /* hv_recommend_dma_remapping */,
>> -            NULL /* hv_recommend_int_remapping */,
>> -            NULL /* hv_recommend_x2apic_msrs */,
>> -            NULL /* hv_recommend_autoeoi_deprecation */,
>> -            NULL /* hv_recommend_pv_ipi */,
>> -            NULL /* hv_recommend_ex_hypercalls */,
>> -            NULL /* hv_hypervisor_is_nested */,
>> -            NULL /* hv_recommend_int_mbec */,
>> -            NULL /* hv_recommend_evmcs */,
>> -            NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -            NULL, NULL, NULL, NULL,
>> -        },
>> -        .cpuid = { .eax = 0x40000004, .reg = R_EAX, },
>> -    },
>> -    [FEAT_HV_NESTED_EAX] = {
>> -        .type = CPUID_FEATURE_WORD,
>> -        .cpuid = { .eax = 0x4000000A, .reg = R_EAX, },
>> -    },
>>      [FEAT_SVM] = {
>>          .type = CPUID_FEATURE_WORD,
>>          .feat_names = {
>> @@ -6906,7 +6818,7 @@ static GuestPanicInformation *x86_cpu_get_crash_info(CPUState *cs)
>>      CPUX86State *env = &cpu->env;
>>      GuestPanicInformation *panic_info = NULL;
>>  
>> -    if (env->features[FEAT_HYPERV_EDX] & HV_GUEST_CRASH_MSR_AVAILABLE) {
>> +    if (hyperv_feat_enabled(cpu, HYPERV_FEAT_CRASH)) {
>>          panic_info = g_malloc0(sizeof(GuestPanicInformation));
>>  
>>          panic_info->type = GUEST_PANIC_INFORMATION_TYPE_HYPER_V;
>> diff --git a/target/i386/cpu.h b/target/i386/cpu.h
>> index 187e4a09e65d..b07325745bcb 100644
>> --- a/target/i386/cpu.h
>> +++ b/target/i386/cpu.h
>> @@ -519,11 +519,6 @@ typedef enum FeatureWord {
>>      FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
>>      FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
>>      FEAT_KVM_HINTS,     /* CPUID[4000_0001].EDX */
>> -    FEAT_HYPERV_EAX,    /* CPUID[4000_0003].EAX */
>> -    FEAT_HYPERV_EBX,    /* CPUID[4000_0003].EBX */
>> -    FEAT_HYPERV_EDX,    /* CPUID[4000_0003].EDX */
>> -    FEAT_HV_RECOMM_EAX, /* CPUID[4000_0004].EAX */
>> -    FEAT_HV_NESTED_EAX, /* CPUID[4000_000A].EAX */
>>      FEAT_SVM,           /* CPUID[8000_000A].EDX */
>>      FEAT_XSAVE,         /* CPUID[EAX=0xd,ECX=1].EAX */
>>      FEAT_6_EAX,         /* CPUID[6].EAX */
>> @@ -1663,6 +1658,7 @@ struct X86CPU {
>>      uint32_t hyperv_interface_id[4];
>>      uint32_t hyperv_version_id[4];
>>      uint32_t hyperv_limits[3];
>> +    uint32_t hyperv_nested[4];
>>  
>>      bool check_cpuid;
>>      bool enforce_cpuid;
>> diff --git a/target/i386/kvm.c b/target/i386/kvm.c
>> index 0e6eef6a52c2..86a5cfa1e12b 100644
>> --- a/target/i386/kvm.c
>> +++ b/target/i386/kvm.c
>> @@ -815,7 +815,8 @@ static bool tsc_is_stable_and_known(CPUX86State *env)
>>  static struct {
>>      const char *desc;
>>      struct {
>> -        uint32_t fw;
>> +        uint32_t func;
>> +        int reg;
>>          uint32_t bits;
>>      } flags[2];
>>      uint64_t dependencies;
>> @@ -823,25 +824,25 @@ static struct {
>>      [HYPERV_FEAT_RELAXED] = {
>>          .desc = "relaxed timing (hv-relaxed)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_HYPERCALL_AVAILABLE},
>> -            {.fw = FEAT_HV_RECOMM_EAX,
>> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>>               .bits = HV_RELAXED_TIMING_RECOMMENDED}
>>          }
>>      },
>>      [HYPERV_FEAT_VAPIC] = {
>>          .desc = "virtual APIC (hv-vapic)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_HYPERCALL_AVAILABLE | HV_APIC_ACCESS_AVAILABLE},
>> -            {.fw = FEAT_HV_RECOMM_EAX,
>> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>>               .bits = HV_APIC_ACCESS_RECOMMENDED}
>>          }
>>      },
>>      [HYPERV_FEAT_TIME] = {
>>          .desc = "clocksources (hv-time)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_HYPERCALL_AVAILABLE | HV_TIME_REF_COUNT_AVAILABLE |
>>               HV_REFERENCE_TSC_AVAILABLE}
>>          }
>> @@ -849,42 +850,42 @@ static struct {
>>      [HYPERV_FEAT_CRASH] = {
>>          .desc = "crash MSRs (hv-crash)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EDX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>>               .bits = HV_GUEST_CRASH_MSR_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_RESET] = {
>>          .desc = "reset MSR (hv-reset)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_RESET_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_VPINDEX] = {
>>          .desc = "VP_INDEX MSR (hv-vpindex)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_VP_INDEX_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_RUNTIME] = {
>>          .desc = "VP_RUNTIME MSR (hv-runtime)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_VP_RUNTIME_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_SYNIC] = {
>>          .desc = "synthetic interrupt controller (hv-synic)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_SYNIC_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_STIMER] = {
>>          .desc = "synthetic timers (hv-stimer)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_SYNTIMERS_AVAILABLE}
>>          },
>>          .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME)
>> @@ -892,23 +893,23 @@ static struct {
>>      [HYPERV_FEAT_FREQUENCIES] = {
>>          .desc = "frequency MSRs (hv-frequencies)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_ACCESS_FREQUENCY_MSRS},
>> -            {.fw = FEAT_HYPERV_EDX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>>               .bits = HV_FREQUENCY_MSRS_AVAILABLE}
>>          }
>>      },
>>      [HYPERV_FEAT_REENLIGHTENMENT] = {
>>          .desc = "reenlightenment MSRs (hv-reenlightenment)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EAX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EAX,
>>               .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL}
>>          }
>>      },
>>      [HYPERV_FEAT_TLBFLUSH] = {
>>          .desc = "paravirtualized TLB flush (hv-tlbflush)",
>>          .flags = {
>> -            {.fw = FEAT_HV_RECOMM_EAX,
>> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>>               .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED |
>>               HV_EX_PROCESSOR_MASKS_RECOMMENDED}
>>          },
>> @@ -917,7 +918,7 @@ static struct {
>>      [HYPERV_FEAT_EVMCS] = {
>>          .desc = "enlightened VMCS (hv-evmcs)",
>>          .flags = {
>> -            {.fw = FEAT_HV_RECOMM_EAX,
>> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>>               .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED}
>>          },
>>          .dependencies = BIT(HYPERV_FEAT_VAPIC)
>> @@ -925,7 +926,7 @@ static struct {
>>      [HYPERV_FEAT_IPI] = {
>>          .desc = "paravirtualized IPI (hv-ipi)",
>>          .flags = {
>> -            {.fw = FEAT_HV_RECOMM_EAX,
>> +            {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
>>               .bits = HV_CLUSTER_IPI_RECOMMENDED |
>>               HV_EX_PROCESSOR_MASKS_RECOMMENDED}
>>          },
>> @@ -934,7 +935,7 @@ static struct {
>>      [HYPERV_FEAT_STIMER_DIRECT] = {
>>          .desc = "direct mode synthetic timers (hv-stimer-direct)",
>>          .flags = {
>> -            {.fw = FEAT_HYPERV_EDX,
>> +            {.func = HV_CPUID_FEATURES, .reg = R_EDX,
>>               .bits = HV_STIMER_DIRECT_MODE_AVAILABLE}
>>          },
>>          .dependencies = BIT(HYPERV_FEAT_STIMER)
>> @@ -1108,38 +1109,34 @@ static uint32_t hv_cpuid_get_host(CPUState *cs, uint32_t func, int reg)
>>      return cpuid_entry_get_reg(entry, reg);
>>  }
>>  
>> -static uint32_t hv_cpuid_get_fw(CPUState *cs, int fw)
>> +static bool hyperv_feature_supported(CPUState *cs, int feature)
>>  {
>> -    uint32_t func;
>> -    int reg;
>> +    uint32_t func, bits;
>> +    int i, reg;
>>  
>> -    switch (fw) {
>> -    case FEAT_HYPERV_EAX:
>> -        reg = R_EAX;
>> -        func = HV_CPUID_FEATURES;
>> -        break;
>> -    case FEAT_HYPERV_EDX:
>> -        reg = R_EDX;
>> -        func = HV_CPUID_FEATURES;
>> -        break;
>> -    case FEAT_HV_RECOMM_EAX:
>> -        reg = R_EAX;
>> -        func = HV_CPUID_ENLIGHTMENT_INFO;
>> -        break;
>> -    default:
>> -        return 0;
>> +    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
>> +
>> +        func = kvm_hyperv_properties[feature].flags[i].func;
>> +        reg = kvm_hyperv_properties[feature].flags[i].reg;
>> +        bits = kvm_hyperv_properties[feature].flags[i].bits;
>> +
>> +        if (!func) {
>> +            continue;
>> +        }
>> +
>> +        if ((hv_cpuid_get_host(cs, func, reg) & bits) != bits) {
>> +            return false;
>> +        }
>>      }
>>  
>> -    return hv_cpuid_get_host(cs, func, reg);
>> +    return true;
>>  }
>>  
>>  static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>>  {
>>      X86CPU *cpu = X86_CPU(cs);
>> -    CPUX86State *env = &cpu->env;
>> -    uint32_t r, fw, bits;
>>      uint64_t deps;
>> -    int i, dep_feat;
>> +    int dep_feat;
>>  
>>      if (!hyperv_feat_enabled(cpu, feature) && !cpu->hyperv_passthrough) {
>>          return 0;
>> @@ -1158,27 +1155,15 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>>          deps &= ~(1ull << dep_feat);
>>      }
>>  
>> -    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
>> -        fw = kvm_hyperv_properties[feature].flags[i].fw;
>> -        bits = kvm_hyperv_properties[feature].flags[i].bits;
>> -
>> -        if (!fw) {
>> -            continue;
>> -        }
>> -
>> -        r = hv_cpuid_get_fw(cs, fw);
>> -        if ((r & bits) != bits) {
>> -            if (hyperv_feat_enabled(cpu, feature)) {
>> -                fprintf(stderr,
>> -                        "Hyper-V %s is not supported by kernel\n",
>> -                        kvm_hyperv_properties[feature].desc);
>> -                return 1;
>> -            } else {
>> -                return 0;
>> -            }
>> +    if (!hyperv_feature_supported(cs, feature)) {
>> +        if (hyperv_feat_enabled(cpu, feature)) {
>> +            fprintf(stderr,
>> +                    "Hyper-V %s is not supported by kernel\n",
>> +                    kvm_hyperv_properties[feature].desc);
>> +            return 1;
>> +        } else {
>> +            return 0;
>>          }
>> -
>> -        env->features[fw] |= bits;
>>      }
>>  
>>      if (cpu->hyperv_passthrough) {
>> @@ -1188,6 +1173,32 @@ static int hv_cpuid_check_and_set(CPUState *cs, int feature)
>>      return 0;
>>  }
>>  
>> +static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
>> +{
>> +    X86CPU *cpu = X86_CPU(cs);
>> +    uint32_t r = 0;
>> +    int i, j;
>> +
>> +    for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties); i++) {
>> +        if (!hyperv_feat_enabled(cpu, i)) {
>> +            continue;
>> +        }
>> +
>> +        for (j = 0; j < ARRAY_SIZE(kvm_hyperv_properties[i].flags); j++) {
>> +            if (kvm_hyperv_properties[i].flags[j].func != func) {
>> +                continue;
>> +            }
>> +            if (kvm_hyperv_properties[i].flags[j].reg != reg) {
>> +                continue;
>> +            }
>> +
>> +            r |= kvm_hyperv_properties[i].flags[j].bits;
>> +        }
>> +    }
>> +
>> +    return r;
>> +}
>> +
>>  /*
>>   * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in
>>   * case of success, errno < 0 in case of failure and 0 when no Hyper-V
>> @@ -1197,7 +1208,6 @@ static int hyperv_handle_properties(CPUState *cs,
>>                                      struct kvm_cpuid_entry2 *cpuid_ent)
>>  {
>>      X86CPU *cpu = X86_CPU(cs);
>> -    CPUX86State *env = &cpu->env;
>>      struct kvm_cpuid_entry2 *c;
>>      uint32_t cpuid_i = 0;
>>      int r;
>> @@ -1219,9 +1229,7 @@ static int hyperv_handle_properties(CPUState *cs,
>>          }
>>  
>>          if (!r) {
>> -            env->features[FEAT_HV_RECOMM_EAX] |=
>> -                HV_ENLIGHTENED_VMCS_RECOMMENDED;
>> -            env->features[FEAT_HV_NESTED_EAX] = evmcs_version;
>> +            cpu->hyperv_nested[0] = evmcs_version;
>>          }
>>      }
>>  
>> @@ -1255,13 +1263,6 @@ static int hyperv_handle_properties(CPUState *cs,
>>          cpu->hyperv_version_id[3] =
>>              hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX);
>>  
>> -        env->features[FEAT_HYPERV_EAX] =
>> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EAX);
>> -        env->features[FEAT_HYPERV_EBX] =
>> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EBX);
>> -        env->features[FEAT_HYPERV_EDX] =
>> -            hv_cpuid_get_host(cs, HV_CPUID_FEATURES, R_EDX);
>> -
>>          cpu->hv_max_vps = hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS,
>>                                              R_EAX);
>>          cpu->hyperv_limits[0] =
>> @@ -1271,21 +1272,8 @@ static int hyperv_handle_properties(CPUState *cs,
>>          cpu->hyperv_limits[2] =
>>              hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EDX);
>>  
>> -        env->features[FEAT_HV_RECOMM_EAX] =
>> -            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
>>          cpu->hyperv_spinlock_attempts =
>>              hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EBX);
>> -
>> -        env->features[FEAT_HV_NESTED_EAX] =
>> -            hv_cpuid_get_host(cs, HV_CPUID_NESTED_FEATURES, R_EAX);
>> -    }
>> -
>> -    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
>> -        env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING;
>> -    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
>> -        env->features[FEAT_HV_RECOMM_EAX] |=
>> -            hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
>> -            HV_NO_NONARCH_CORESHARING;
>>      }
>>  
>>      /* Features */
>> @@ -1315,9 +1303,6 @@ static int hyperv_handle_properties(CPUState *cs,
>>          r |= 1;
>>      }
>>  
>> -    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
>> -    env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
>> -
>>      if (r) {
>>          return -ENOSYS;
>>      }
>> @@ -1346,15 +1331,25 @@ static int hyperv_handle_properties(CPUState *cs,
>>  
>>      c = &cpuid_ent[cpuid_i++];
>>      c->function = HV_CPUID_FEATURES;
>> -    c->eax = env->features[FEAT_HYPERV_EAX];
>> -    c->ebx = env->features[FEAT_HYPERV_EBX];
>> -    c->edx = env->features[FEAT_HYPERV_EDX];
>> +    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EAX);
>> +    c->ebx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EBX);
>> +    c->edx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EDX);
>> +
>> +    /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
>> +    c->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
>>  
>>      c = &cpuid_ent[cpuid_i++];
>>      c->function = HV_CPUID_ENLIGHTMENT_INFO;
>> -    c->eax = env->features[FEAT_HV_RECOMM_EAX];
>> +    c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
>>      c->ebx = cpu->hyperv_spinlock_attempts;
>>  
>> +    if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
>> +        c->eax |= HV_NO_NONARCH_CORESHARING;
>> +    } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
>> +        c->eax |= hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
>> +            HV_NO_NONARCH_CORESHARING;
>> +    }
>> +
>>      c = &cpuid_ent[cpuid_i++];
>>      c->function = HV_CPUID_IMPLEMENT_LIMITS;
>>      c->eax = cpu->hv_max_vps;
>> @@ -1374,7 +1369,7 @@ static int hyperv_handle_properties(CPUState *cs,
>>  
>>          c = &cpuid_ent[cpuid_i++];
>>          c->function = HV_CPUID_NESTED_FEATURES;
>> -        c->eax = env->features[FEAT_HV_NESTED_EAX];
>> +        c->eax = cpu->hyperv_nested[0];
>>      }
>>  
>>      return cpuid_i;
>> -- 
>> 2.25.4
>> 

-- 
Vitaly