[PATCH v2 0/2] add support for vNVDIMM - Part 1: enable instructions

[PATCH v2 0/2] add support for vNVDIMM - Part 1: enable instructions

[Haozhong Zhang]

I split the previous patch series "[PATCH 0/4] add support for vNVDIMM"
into two parts. This is the first part that enables clflushopt/clwb/pcommit
instructions for HVM guests.

[The second part will be sent separately after v1 patches get reviewed.]

Changes in v2:
 * Refactor modifications in hvm_cpuid() per Andrew Cooper's comments.

Haozhong Zhang (2):
  x86/hvm: allow guest to use clflushopt and clwb
  x86/hvm: add support for pcommit instruction

 tools/libxc/xc_cpufeature.h        |  4 +++-
 tools/libxc/xc_cpuid_x86.c         |  5 ++++-
 xen/arch/x86/hvm/hvm.c             | 36 ++++++++++++++++++++++++------------
 xen/arch/x86/hvm/vmx/vmcs.c        |  6 +++++-
 xen/arch/x86/hvm/vmx/vmx.c         |  1 +
 xen/arch/x86/hvm/vmx/vvmx.c        |  3 +++
 xen/include/asm-x86/cpufeature.h   |  7 +++++++
 xen/include/asm-x86/hvm/vmx/vmcs.h |  3 +++
 xen/include/asm-x86/hvm/vmx/vmx.h  |  1 +
 9 files changed, 51 insertions(+), 15 deletions(-)

-- 
2.4.8

[PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Haozhong Zhang]

Pass CPU features CLFLUSHOPT and CLWB into HVM domain so that those two
instructions can be used by guest.

The specification of above two instructions can be found in
https://software.intel.com/sites/default/files/managed/0d/53/319433-022.pdf

Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
 tools/libxc/xc_cpufeature.h      |  3 ++-
 tools/libxc/xc_cpuid_x86.c       |  4 +++-
 xen/arch/x86/hvm/hvm.c           | 33 +++++++++++++++++++++------------
 xen/include/asm-x86/cpufeature.h |  5 +++++
 4 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/tools/libxc/xc_cpufeature.h b/tools/libxc/xc_cpufeature.h
index c3ddc80..5288ac6 100644
--- a/tools/libxc/xc_cpufeature.h
+++ b/tools/libxc/xc_cpufeature.h
@@ -140,6 +140,7 @@
 #define X86_FEATURE_RDSEED      18 /* RDSEED instruction */
 #define X86_FEATURE_ADX         19 /* ADCX, ADOX instructions */
 #define X86_FEATURE_SMAP        20 /* Supervisor Mode Access Protection */
-
+#define X86_FEATURE_CLFLUSHOPT  23 /* CLFLUSHOPT instruction */
+#define X86_FEATURE_CLWB        24 /* CLWB instruction */
 
 #endif /* __LIBXC_CPUFEATURE_H */
diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c
index 8882c01..fecfd6c 100644
--- a/tools/libxc/xc_cpuid_x86.c
+++ b/tools/libxc/xc_cpuid_x86.c
@@ -426,7 +426,9 @@ static void xc_cpuid_hvm_policy(xc_interface *xch,
                         bitmaskof(X86_FEATURE_RDSEED)  |
                         bitmaskof(X86_FEATURE_ADX)  |
                         bitmaskof(X86_FEATURE_SMAP) |
-                        bitmaskof(X86_FEATURE_FSGSBASE));
+                        bitmaskof(X86_FEATURE_FSGSBASE) |
+                        bitmaskof(X86_FEATURE_CLWB) |
+                        bitmaskof(X86_FEATURE_CLFLUSHOPT));
         } else
             regs[1] = 0;
         regs[0] = regs[2] = regs[3] = 0;
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 21470ec..9099188 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
             *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
         break;
     case 0x7:
-        if ( (count == 0) && !cpu_has_smep )
-            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
+        if ( count == 0 )
+        {
+            if ( !cpu_has_smep )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
+
+            if ( !cpu_has_smap )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
+
+            /* Don't expose MPX to hvm when VMX support is not available */
+            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
+                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
 
-        if ( (count == 0) && !cpu_has_smap )
-            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
+            /* Don't expose INVPCID to non-hap hvm. */
+            if ( !hap_enabled(d) )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
 
-        /* Don't expose MPX to hvm when VMX support is not available */
-        if ( (count == 0) &&
-             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
-              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
-            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
+            if ( !cpu_has_clflushopt )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
+
+            if ( !cpu_has_clwb )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
+        }
 
-        /* Don't expose INVPCID to non-hap hvm. */
-        if ( (count == 0) && !hap_enabled(d) )
-            *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
         break;
     case 0xb:
         /* Fix the x2APIC identifier. */
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index ef96514..5818228 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -162,6 +162,8 @@
 #define X86_FEATURE_RDSEED	(7*32+18) /* RDSEED instruction */
 #define X86_FEATURE_ADX		(7*32+19) /* ADCX, ADOX instructions */
 #define X86_FEATURE_SMAP	(7*32+20) /* Supervisor Mode Access Prevention */
+#define X86_FEATURE_CLFLUSHOPT	(7*32+23) /* CLFLUSHOPT instruction */
+#define X86_FEATURE_CLWB	(7*32+24) /* CLWB instruction */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (ecx), word 8 */
 #define X86_FEATURE_PKU	(8*32+ 3) /* Protection Keys for Userspace */
@@ -234,6 +236,9 @@
 #define cpu_has_xgetbv1		boot_cpu_has(X86_FEATURE_XGETBV1)
 #define cpu_has_xsaves		boot_cpu_has(X86_FEATURE_XSAVES)
 
+#define cpu_has_clflushopt  boot_cpu_has(X86_FEATURE_CLFLUSHOPT)
+#define cpu_has_clwb        boot_cpu_has(X86_FEATURE_CLWB)
+
 enum _cache_type {
     CACHE_TYPE_NULL = 0,
     CACHE_TYPE_DATA = 1,
-- 
2.4.8

[PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Haozhong Zhang]

Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
pcommit instruction for L1 guest, and allow L1 to intercept pcommit
instruction for L2 guest.

The specification of pcommit instruction can be found in
https://software.intel.com/sites/default/files/managed/0d/53/319433-022.pdf

Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>
---
 tools/libxc/xc_cpufeature.h        | 1 +
 tools/libxc/xc_cpuid_x86.c         | 1 +
 xen/arch/x86/hvm/hvm.c             | 3 +++
 xen/arch/x86/hvm/vmx/vmcs.c        | 6 +++++-
 xen/arch/x86/hvm/vmx/vmx.c         | 1 +
 xen/arch/x86/hvm/vmx/vvmx.c        | 3 +++
 xen/include/asm-x86/cpufeature.h   | 2 ++
 xen/include/asm-x86/hvm/vmx/vmcs.h | 3 +++
 xen/include/asm-x86/hvm/vmx/vmx.h  | 1 +
 9 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tools/libxc/xc_cpufeature.h b/tools/libxc/xc_cpufeature.h
index 5288ac6..ee53679 100644
--- a/tools/libxc/xc_cpufeature.h
+++ b/tools/libxc/xc_cpufeature.h
@@ -140,6 +140,7 @@
 #define X86_FEATURE_RDSEED      18 /* RDSEED instruction */
 #define X86_FEATURE_ADX         19 /* ADCX, ADOX instructions */
 #define X86_FEATURE_SMAP        20 /* Supervisor Mode Access Protection */
+#define X86_FEATURE_PCOMMIT     22 /* PCOMMIT instruction */
 #define X86_FEATURE_CLFLUSHOPT  23 /* CLFLUSHOPT instruction */
 #define X86_FEATURE_CLWB        24 /* CLWB instruction */
 
diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c
index fecfd6c..c142595 100644
--- a/tools/libxc/xc_cpuid_x86.c
+++ b/tools/libxc/xc_cpuid_x86.c
@@ -427,6 +427,7 @@ static void xc_cpuid_hvm_policy(xc_interface *xch,
                         bitmaskof(X86_FEATURE_ADX)  |
                         bitmaskof(X86_FEATURE_SMAP) |
                         bitmaskof(X86_FEATURE_FSGSBASE) |
+                        bitmaskof(X86_FEATURE_PCOMMIT) |
                         bitmaskof(X86_FEATURE_CLWB) |
                         bitmaskof(X86_FEATURE_CLFLUSHOPT));
         } else
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 9099188..ae8f8dc 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -4605,6 +4605,9 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
 
             if ( !cpu_has_clwb )
                 *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
+
+            if ( !cpu_has_pcommit )
+                *ebx &= ~cpufeat_mask(X86_FEATURE_PCOMMIT);
         }
 
         break;
diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c
index edd4c8d..9092a98 100644
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -242,7 +242,8 @@ static int vmx_init_vmcs_config(void)
                SECONDARY_EXEC_ENABLE_INVPCID |
                SECONDARY_EXEC_ENABLE_VM_FUNCTIONS |
                SECONDARY_EXEC_ENABLE_VIRT_EXCEPTIONS |
-               SECONDARY_EXEC_XSAVES);
+               SECONDARY_EXEC_XSAVES |
+               SECONDARY_EXEC_PCOMMIT);
         rdmsrl(MSR_IA32_VMX_MISC, _vmx_misc_cap);
         if ( _vmx_misc_cap & VMX_MISC_VMWRITE_ALL )
             opt |= SECONDARY_EXEC_ENABLE_VMCS_SHADOWING;
@@ -1075,6 +1076,9 @@ static int construct_vmcs(struct vcpu *v)
         __vmwrite(PLE_WINDOW, ple_window);
     }
 
+    if ( cpu_has_vmx_pcommit )
+        v->arch.hvm_vmx.secondary_exec_control &= ~SECONDARY_EXEC_PCOMMIT;
+
     if ( cpu_has_vmx_secondary_exec_control )
         __vmwrite(SECONDARY_VM_EXEC_CONTROL,
                   v->arch.hvm_vmx.secondary_exec_control);
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index b918b8a..0991cdf 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -3517,6 +3517,7 @@ void vmx_vmexit_handler(struct cpu_user_regs *regs)
     case EXIT_REASON_ACCESS_LDTR_OR_TR:
     case EXIT_REASON_VMX_PREEMPTION_TIMER_EXPIRED:
     case EXIT_REASON_INVPCID:
+    case EXIT_REASON_PCOMMIT:
     /* fall through */
     default:
     exit_and_crash:
diff --git a/xen/arch/x86/hvm/vmx/vvmx.c b/xen/arch/x86/hvm/vmx/vvmx.c
index ea1052e..271ec70 100644
--- a/xen/arch/x86/hvm/vmx/vvmx.c
+++ b/xen/arch/x86/hvm/vmx/vvmx.c
@@ -1950,6 +1950,8 @@ int nvmx_msr_read_intercept(unsigned int msr, u64 *msr_content)
                SECONDARY_EXEC_ENABLE_VPID |
                SECONDARY_EXEC_UNRESTRICTED_GUEST |
                SECONDARY_EXEC_ENABLE_EPT;
+        if ( cpu_has_vmx_pcommit )
+            data |= SECONDARY_EXEC_PCOMMIT;
         data = gen_vmx_msr(data, 0, host_data);
         break;
     case MSR_IA32_VMX_EXIT_CTLS:
@@ -2226,6 +2228,7 @@ int nvmx_n2_vmexit_handler(struct cpu_user_regs *regs,
     case EXIT_REASON_VMXON:
     case EXIT_REASON_INVEPT:
     case EXIT_REASON_XSETBV:
+    case EXIT_REASON_PCOMMIT:
         /* inject to L1 */
         nvcpu->nv_vmexit_pending = 1;
         break;
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 5818228..7491e37 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -162,6 +162,7 @@
 #define X86_FEATURE_RDSEED	(7*32+18) /* RDSEED instruction */
 #define X86_FEATURE_ADX		(7*32+19) /* ADCX, ADOX instructions */
 #define X86_FEATURE_SMAP	(7*32+20) /* Supervisor Mode Access Prevention */
+#define X86_FEATURE_PCOMMIT	(7*32+22) /* PCOMMIT instruction */
 #define X86_FEATURE_CLFLUSHOPT	(7*32+23) /* CLFLUSHOPT instruction */
 #define X86_FEATURE_CLWB	(7*32+24) /* CLWB instruction */
 
@@ -238,6 +239,7 @@
 
 #define cpu_has_clflushopt  boot_cpu_has(X86_FEATURE_CLFLUSHOPT)
 #define cpu_has_clwb        boot_cpu_has(X86_FEATURE_CLWB)
+#define cpu_has_pcommit     boot_cpu_has(X86_FEATURE_PCOMMIT)
 
 enum _cache_type {
     CACHE_TYPE_NULL = 0,
diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h
index d1496b8..77cf8da 100644
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h
@@ -236,6 +236,7 @@ extern u32 vmx_vmentry_control;
 #define SECONDARY_EXEC_ENABLE_PML               0x00020000
 #define SECONDARY_EXEC_ENABLE_VIRT_EXCEPTIONS   0x00040000
 #define SECONDARY_EXEC_XSAVES                   0x00100000
+#define SECONDARY_EXEC_PCOMMIT                  0x00200000
 extern u32 vmx_secondary_exec_control;
 
 #define VMX_EPT_EXEC_ONLY_SUPPORTED                         0x00000001
@@ -303,6 +304,8 @@ extern u64 vmx_ept_vpid_cap;
     (vmx_secondary_exec_control & SECONDARY_EXEC_ENABLE_PML)
 #define cpu_has_vmx_xsaves \
     (vmx_secondary_exec_control & SECONDARY_EXEC_XSAVES)
+#define cpu_has_vmx_pcommit \
+    (vmx_secondary_exec_control & SECONDARY_EXEC_PCOMMIT)
 
 #define VMCS_RID_TYPE_MASK              0x80000000
 
diff --git a/xen/include/asm-x86/hvm/vmx/vmx.h b/xen/include/asm-x86/hvm/vmx/vmx.h
index 1719965..14f3d32 100644
--- a/xen/include/asm-x86/hvm/vmx/vmx.h
+++ b/xen/include/asm-x86/hvm/vmx/vmx.h
@@ -213,6 +213,7 @@ static inline void pi_clear_sn(struct pi_desc *pi_desc)
 #define EXIT_REASON_PML_FULL            62
 #define EXIT_REASON_XSAVES              63
 #define EXIT_REASON_XRSTORS             64
+#define EXIT_REASON_PCOMMIT             65
 
 /*
  * Interruption-information format
-- 
2.4.8

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Andrew Cooper]

On 30/12/2015 11:48, Haozhong Zhang wrote:
> Pass CPU features CLFLUSHOPT and CLWB into HVM domain so that those two
> instructions can be used by guest.
>
> The specification of above two instructions can be found in
> https://software.intel.com/sites/default/files/managed/0d/53/319433-022.pdf
>
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Andrew Cooper]

On 30/12/2015 11:48, Haozhong Zhang wrote:
> Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> instruction for L2 guest.
>
> The specification of pcommit instruction can be found in
> https://software.intel.com/sites/default/files/managed/0d/53/319433-022.pdf
>
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

Re: [PATCH v2 0/2] add support for vNVDIMM - Part 1: enable instructions

[Wei Liu]

On Wed, Dec 30, 2015 at 07:48:57PM +0800, Haozhong Zhang wrote:
> I split the previous patch series "[PATCH 0/4] add support for vNVDIMM"
> into two parts. This is the first part that enables clflushopt/clwb/pcommit
> instructions for HVM guests.
> 
> [The second part will be sent separately after v1 patches get reviewed.]
> 
> Changes in v2:
>  * Refactor modifications in hvm_cpuid() per Andrew Cooper's comments.
> 
> Haozhong Zhang (2):
>   x86/hvm: allow guest to use clflushopt and clwb
>   x86/hvm: add support for pcommit instruction
> 
>  tools/libxc/xc_cpufeature.h        |  4 +++-
>  tools/libxc/xc_cpuid_x86.c         |  5 ++++-

Tools bits:

Acked-by: Wei Liu <wei.liu2@citrix.com>

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Tian, Kevin]

> From: Zhang, Haozhong
> Sent: Wednesday, December 30, 2015 7:49 PM
> 
> Pass CPU features CLFLUSHOPT and CLWB into HVM domain so that those two
> instructions can be used by guest.
> 
> The specification of above two instructions can be found in
> https://software.intel.com/sites/default/files/managed/0d/53/319433-022.pdf
> 
> Signed-off-by: Haozhong Zhang <haozhong.zhang@intel.com>

Reviewed-by: Kevin Tian <kevin.tian@intel.com>

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Tian, Kevin]

> From: Zhang, Haozhong
> Sent: Wednesday, December 30, 2015 7:49 PM
> 
> Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> instruction for L2 guest.

Could you elaborate why different policies are used for L1/L2? And better
add a comment in code (at least for vvmx) to describe the intention.

Thanks
Kevin

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Zhang, Haozhong]

On 01/05/16 15:08, Tian, Kevin wrote:
> > From: Zhang, Haozhong
> > Sent: Wednesday, December 30, 2015 7:49 PM
> > 
> > Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> > pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> > instruction for L2 guest.
> 
> Could you elaborate why different policies are used for L1/L2? And better
> add a comment in code (at least for vvmx) to describe the intention.
> 
> Thanks
> Kevin

The intention is that we completely expose pcommit (both the
instruction and VMEXIT caused by pcommit) to L1.

Haozhong

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Tian, Kevin]

> From: Zhang, Haozhong
> Sent: Tuesday, January 05, 2016 3:15 PM
> 
> On 01/05/16 15:08, Tian, Kevin wrote:
> > > From: Zhang, Haozhong
> > > Sent: Wednesday, December 30, 2015 7:49 PM
> > >
> > > Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> > > pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> > > instruction for L2 guest.
> >
> > Could you elaborate why different policies are used for L1/L2? And better
> > add a comment in code (at least for vvmx) to describe the intention.
> >
> > Thanks
> > Kevin
> 
> The intention is that we completely expose pcommit (both the
> instruction and VMEXIT caused by pcommit) to L1.
> 
> Haozhong

My question is why pcommit can't not be directly in L2 w/o interception?

Thanks
Kevin

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Zhang, Haozhong]

On 01/05/16 15:19, Tian, Kevin wrote:
> > From: Zhang, Haozhong
> > Sent: Tuesday, January 05, 2016 3:15 PM
> > 
> > On 01/05/16 15:08, Tian, Kevin wrote:
> > > > From: Zhang, Haozhong
> > > > Sent: Wednesday, December 30, 2015 7:49 PM
> > > >
> > > > Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> > > > pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> > > > instruction for L2 guest.
> > >
> > > Could you elaborate why different policies are used for L1/L2? And better
> > > add a comment in code (at least for vvmx) to describe the intention.
> > >
> > > Thanks
> > > Kevin
> > 
> > The intention is that we completely expose pcommit (both the
> > instruction and VMEXIT caused by pcommit) to L1.
> > 
> > Haozhong
> 
> My question is why pcommit can't not be directly in L2 w/o interception?
> 
> Thanks
> Kevin
> 

Isn't it because L1 hypervisor may decide to intercept L2 pcommit?

Haozhong

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Tian, Kevin]

> From: Zhang, Haozhong
> Sent: Tuesday, January 05, 2016 3:33 PM
> 
> On 01/05/16 15:19, Tian, Kevin wrote:
> > > From: Zhang, Haozhong
> > > Sent: Tuesday, January 05, 2016 3:15 PM
> > >
> > > On 01/05/16 15:08, Tian, Kevin wrote:
> > > > > From: Zhang, Haozhong
> > > > > Sent: Wednesday, December 30, 2015 7:49 PM
> > > > >
> > > > > Pass PCOMMIT CPU feature into HMV domain. Currently, we do not intercept
> > > > > pcommit instruction for L1 guest, and allow L1 to intercept pcommit
> > > > > instruction for L2 guest.
> > > >
> > > > Could you elaborate why different policies are used for L1/L2? And better
> > > > add a comment in code (at least for vvmx) to describe the intention.
> > > >
> > > > Thanks
> > > > Kevin
> > >
> > > The intention is that we completely expose pcommit (both the
> > > instruction and VMEXIT caused by pcommit) to L1.
> > >
> > > Haozhong
> >
> > My question is why pcommit can't not be directly in L2 w/o interception?
> >
> > Thanks
> > Kevin
> >
> 
> Isn't it because L1 hypervisor may decide to intercept L2 pcommit?
> 

Well, I misunderstood the original statement and actual patch w/ the
impression that pcommit interception is always enabled for L2 guest.
Now I see that itt's just the intrinsic result when we don't intercept 
L1 pcommit (either a L1 guest or L1 hypervisor). So,

Acked-by: Kevin Tian <kevin.tian@intel.com>

Thanks
Kevin

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Jan Beulich]

>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
> --- a/xen/arch/x86/hvm/hvm.c
> +++ b/xen/arch/x86/hvm/hvm.c
> @@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
>              *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
>          break;
>      case 0x7:
> -        if ( (count == 0) && !cpu_has_smep )
> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
> +        if ( count == 0 )
> +        {
> +            if ( !cpu_has_smep )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
> +
> +            if ( !cpu_has_smap )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
> +
> +            /* Don't expose MPX to hvm when VMX support is not available */
> +            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
> +                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>  
> -        if ( (count == 0) && !cpu_has_smap )
> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
> +            /* Don't expose INVPCID to non-hap hvm. */
> +            if ( !hap_enabled(d) )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
>  
> -        /* Don't expose MPX to hvm when VMX support is not available */
> -        if ( (count == 0) &&
> -             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
> -              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
> -            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
> +            if ( !cpu_has_clflushopt )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
> +
> +            if ( !cpu_has_clwb )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);

I don't think we need this: Other than other things adjusted here,
there's nothing disabling these two features when found available,
and there are no extra conditions to consider. Otherwise, if we
were to follow this route, quite a bit of code would need to be
added to other case statements in this function. But that's all (I
think) going to be taken care of by Andrew's CPUID leveling series.

Jan

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Jan Beulich]

>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
> --- a/xen/arch/x86/hvm/hvm.c
> +++ b/xen/arch/x86/hvm/hvm.c
> @@ -4605,6 +4605,9 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
>  
>              if ( !cpu_has_clwb )
>                  *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
> +
> +            if ( !cpu_has_pcommit )
> +                *ebx &= ~cpufeat_mask(X86_FEATURE_PCOMMIT);

Other than for patch 1, this not only need to stay, but needs to be
extended along the lines of X86_FEATURE_MPX handling.

> @@ -1075,6 +1076,9 @@ static int construct_vmcs(struct vcpu *v)
>          __vmwrite(PLE_WINDOW, ple_window);
>      }
>  
> +    if ( cpu_has_vmx_pcommit )
> +        v->arch.hvm_vmx.secondary_exec_control &= ~SECONDARY_EXEC_PCOMMIT;

Why is this conditional? Instead of the if() there should be a comment
imo.

Jan

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Andrew Cooper]

On 07/01/16 13:49, Jan Beulich wrote:
>>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
>> --- a/xen/arch/x86/hvm/hvm.c
>> +++ b/xen/arch/x86/hvm/hvm.c
>> @@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
>>              *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
>>          break;
>>      case 0x7:
>> -        if ( (count == 0) && !cpu_has_smep )
>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>> +        if ( count == 0 )
>> +        {
>> +            if ( !cpu_has_smep )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>> +
>> +            if ( !cpu_has_smap )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>> +
>> +            /* Don't expose MPX to hvm when VMX support is not available */
>> +            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>> +                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>>  
>> -        if ( (count == 0) && !cpu_has_smap )
>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>> +            /* Don't expose INVPCID to non-hap hvm. */
>> +            if ( !hap_enabled(d) )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
>>  
>> -        /* Don't expose MPX to hvm when VMX support is not available */
>> -        if ( (count == 0) &&
>> -             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>> -              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>> +            if ( !cpu_has_clflushopt )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
>> +
>> +            if ( !cpu_has_clwb )
>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
> I don't think we need this: Other than other things adjusted here,
> there's nothing disabling these two features when found available,
> and there are no extra conditions to consider. Otherwise, if we
> were to follow this route, quite a bit of code would need to be
> added to other case statements in this function. But that's all (I
> think) going to be taken care of by Andrew's CPUID leveling series.

My series does take care of all of this.

I would prefer that these two changes get taken as soon as they are
ready, so I can rebase.

~Andrew

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Jan Beulich]

>>> On 07.01.16 at 15:01, <andrew.cooper3@citrix.com> wrote:
> On 07/01/16 13:49, Jan Beulich wrote:
>>>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
>>> --- a/xen/arch/x86/hvm/hvm.c
>>> +++ b/xen/arch/x86/hvm/hvm.c
>>> @@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
> unsigned int *ebx,
>>>              *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
>>>          break;
>>>      case 0x7:
>>> -        if ( (count == 0) && !cpu_has_smep )
>>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>>> +        if ( count == 0 )
>>> +        {
>>> +            if ( !cpu_has_smep )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>>> +
>>> +            if ( !cpu_has_smap )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>>> +
>>> +            /* Don't expose MPX to hvm when VMX support is not available */
>>> +            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>>> +                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>>>  
>>> -        if ( (count == 0) && !cpu_has_smap )
>>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>>> +            /* Don't expose INVPCID to non-hap hvm. */
>>> +            if ( !hap_enabled(d) )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
>>>  
>>> -        /* Don't expose MPX to hvm when VMX support is not available */
>>> -        if ( (count == 0) &&
>>> -             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>>> -              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
>>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>>> +            if ( !cpu_has_clflushopt )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
>>> +
>>> +            if ( !cpu_has_clwb )
>>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
>> I don't think we need this: Other than other things adjusted here,
>> there's nothing disabling these two features when found available,
>> and there are no extra conditions to consider. Otherwise, if we
>> were to follow this route, quite a bit of code would need to be
>> added to other case statements in this function. But that's all (I
>> think) going to be taken care of by Andrew's CPUID leveling series.
> 
> My series does take care of all of this.
> 
> I would prefer that these two changes get taken as soon as they are
> ready, so I can rebase.

If we don't need the change quoted above, your rebase will actually
be easier to do.

Jan

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Haozhong Zhang]

On 01/07/16 06:53, Jan Beulich wrote:
> >>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
> > --- a/xen/arch/x86/hvm/hvm.c
> > +++ b/xen/arch/x86/hvm/hvm.c
> > @@ -4605,6 +4605,9 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
> >  
> >              if ( !cpu_has_clwb )
> >                  *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
> > +
> > +            if ( !cpu_has_pcommit )
> > +                *ebx &= ~cpufeat_mask(X86_FEATURE_PCOMMIT);
> 
> Other than for patch 1, this not only need to stay, but needs to be
> extended along the lines of X86_FEATURE_MPX handling.
>

In section "PCOMMIT - Virtualization Support" of Intel Architecture
Instruction Set Extensions Programming Reference, it says

    IA32_VMX_PROCBASED_CTLS2[53] (which enumerates support for the
    1-setting of “PCOMMIT exiting”) is always the same as
    CPUID.07H:EBX.PCOMMIT[bit 22].

so checking cpu_has_pcommit is enough here.

> > @@ -1075,6 +1076,9 @@ static int construct_vmcs(struct vcpu *v)
> >          __vmwrite(PLE_WINDOW, ple_window);
> >      }
> >  
> > +    if ( cpu_has_vmx_pcommit )
> > +        v->arch.hvm_vmx.secondary_exec_control &= ~SECONDARY_EXEC_PCOMMIT;
> 
> Why is this conditional? Instead of the if() there should be a comment
> imo.
>
The condition is not necessary. I'll remove it and add a comment in
the next version.

Haozhong

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Jan Beulich]

>>> On 08.01.16 at 02:15, <haozhong.zhang@intel.com> wrote:
> On 01/07/16 06:53, Jan Beulich wrote:
>> >>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
>> > --- a/xen/arch/x86/hvm/hvm.c
>> > +++ b/xen/arch/x86/hvm/hvm.c
>> > @@ -4605,6 +4605,9 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
> unsigned int *ebx,
>> >  
>> >              if ( !cpu_has_clwb )
>> >                  *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
>> > +
>> > +            if ( !cpu_has_pcommit )
>> > +                *ebx &= ~cpufeat_mask(X86_FEATURE_PCOMMIT);
>> 
>> Other than for patch 1, this not only need to stay, but needs to be
>> extended along the lines of X86_FEATURE_MPX handling.
>>
> 
> In section "PCOMMIT - Virtualization Support" of Intel Architecture
> Instruction Set Extensions Programming Reference, it says
> 
>     IA32_VMX_PROCBASED_CTLS2[53] (which enumerates support for the
>     1-setting of “PCOMMIT exiting”) is always the same as
>     CPUID.07H:EBX.PCOMMIT[bit 22].
> 
> so checking cpu_has_pcommit is enough here.

Even if the documentation says so, I think it is appropriate (as
indicated at the very least to match with MPX handling) to add
the check (here or elsewhere), just to make obvious that both
are a prereq.

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 2/2] x86/hvm: add support for pcommit instruction

[Haozhong Zhang]

On 01/08/16 00:54, Jan Beulich wrote:
> >>> On 08.01.16 at 02:15, <haozhong.zhang@intel.com> wrote:
> > On 01/07/16 06:53, Jan Beulich wrote:
> >> >>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
> >> > --- a/xen/arch/x86/hvm/hvm.c
> >> > +++ b/xen/arch/x86/hvm/hvm.c
> >> > @@ -4605,6 +4605,9 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
> > unsigned int *ebx,
> >> >  
> >> >              if ( !cpu_has_clwb )
> >> >                  *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
> >> > +
> >> > +            if ( !cpu_has_pcommit )
> >> > +                *ebx &= ~cpufeat_mask(X86_FEATURE_PCOMMIT);
> >> 
> >> Other than for patch 1, this not only need to stay, but needs to be
> >> extended along the lines of X86_FEATURE_MPX handling.
> >>
> > 
> > In section "PCOMMIT - Virtualization Support" of Intel Architecture
> > Instruction Set Extensions Programming Reference, it says
> > 
> >     IA32_VMX_PROCBASED_CTLS2[53] (which enumerates support for the
> >     1-setting of “PCOMMIT exiting”) is always the same as
> >     CPUID.07H:EBX.PCOMMIT[bit 22].
> > 
> > so checking cpu_has_pcommit is enough here.
> 
> Even if the documentation says so, I think it is appropriate (as
> indicated at the very least to match with MPX handling) to add
> the check (here or elsewhere), just to make obvious that both
> are a prereq.
> 
> Jan

OK, I'll modify in the next version.

Haozhong

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Haozhong Zhang]

On 01/07/16 07:34, Jan Beulich wrote:
> >>> On 07.01.16 at 15:01, <andrew.cooper3@citrix.com> wrote:
> > On 07/01/16 13:49, Jan Beulich wrote:
> >>>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
> >>> --- a/xen/arch/x86/hvm/hvm.c
> >>> +++ b/xen/arch/x86/hvm/hvm.c
> >>> @@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
> > unsigned int *ebx,
> >>>              *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
> >>>          break;
> >>>      case 0x7:
> >>> -        if ( (count == 0) && !cpu_has_smep )
> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
> >>> +        if ( count == 0 )
> >>> +        {
> >>> +            if ( !cpu_has_smep )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
> >>> +
> >>> +            if ( !cpu_has_smap )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
> >>> +
> >>> +            /* Don't expose MPX to hvm when VMX support is not available */
> >>> +            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
> >>> +                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
> >>>  
> >>> -        if ( (count == 0) && !cpu_has_smap )
> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
> >>> +            /* Don't expose INVPCID to non-hap hvm. */
> >>> +            if ( !hap_enabled(d) )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
> >>>  
> >>> -        /* Don't expose MPX to hvm when VMX support is not available */
> >>> -        if ( (count == 0) &&
> >>> -             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
> >>> -              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
> >>> +            if ( !cpu_has_clflushopt )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
> >>> +
> >>> +            if ( !cpu_has_clwb )
> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
> >> I don't think we need this: Other than other things adjusted here,
> >> there's nothing disabling these two features when found available,
> >> and there are no extra conditions to consider. Otherwise, if we
> >> were to follow this route, quite a bit of code would need to be
> >> added to other case statements in this function. But that's all (I
> >> think) going to be taken care of by Andrew's CPUID leveling series.
> > 
> > My series does take care of all of this.
> > 
> > I would prefer that these two changes get taken as soon as they are
> > ready, so I can rebase.
> 
> If we don't need the change quoted above, your rebase will actually
> be easier to do.
> 
> Jan
>
I'll remove changes in hvm_cpuid() in the next version. Changes in
xen/include/asm-x86/cpufeature.h will be removed as well, because
there will be no use of them.

Haozhong

Re: [PATCH v2 1/2] x86/hvm: allow guest to use clflushopt and clwb

[Jan Beulich]

>>> On 08.01.16 at 09:27, <haozhong.zhang@intel.com> wrote:
> On 01/07/16 07:34, Jan Beulich wrote:
>> >>> On 07.01.16 at 15:01, <andrew.cooper3@citrix.com> wrote:
>> > On 07/01/16 13:49, Jan Beulich wrote:
>> >>>>> On 30.12.15 at 12:48, <haozhong.zhang@intel.com> wrote:
>> >>> --- a/xen/arch/x86/hvm/hvm.c
>> >>> +++ b/xen/arch/x86/hvm/hvm.c
>> >>> @@ -4583,21 +4583,30 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
>> > unsigned int *ebx,
>> >>>              *edx &= ~cpufeat_mask(X86_FEATURE_PSE36);
>> >>>          break;
>> >>>      case 0x7:
>> >>> -        if ( (count == 0) && !cpu_has_smep )
>> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>> >>> +        if ( count == 0 )
>> >>> +        {
>> >>> +            if ( !cpu_has_smep )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMEP);
>> >>> +
>> >>> +            if ( !cpu_has_smap )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>> >>> +
>> >>> +            /* Don't expose MPX to hvm when VMX support is not available */
>> >>> +            if ( !(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>> >>> +                 !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS) )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>> >>>  
>> >>> -        if ( (count == 0) && !cpu_has_smap )
>> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_SMAP);
>> >>> +            /* Don't expose INVPCID to non-hap hvm. */
>> >>> +            if ( !hap_enabled(d) )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_INVPCID);
>> >>>  
>> >>> -        /* Don't expose MPX to hvm when VMX support is not available */
>> >>> -        if ( (count == 0) &&
>> >>> -             (!(vmx_vmexit_control & VM_EXIT_CLEAR_BNDCFGS) ||
>> >>> -              !(vmx_vmentry_control & VM_ENTRY_LOAD_BNDCFGS)) )
>> >>> -            *ebx &= ~cpufeat_mask(X86_FEATURE_MPX);
>> >>> +            if ( !cpu_has_clflushopt )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLFLUSHOPT);
>> >>> +
>> >>> +            if ( !cpu_has_clwb )
>> >>> +                *ebx &= ~cpufeat_mask(X86_FEATURE_CLWB);
>> >> I don't think we need this: Other than other things adjusted here,
>> >> there's nothing disabling these two features when found available,
>> >> and there are no extra conditions to consider. Otherwise, if we
>> >> were to follow this route, quite a bit of code would need to be
>> >> added to other case statements in this function. But that's all (I
>> >> think) going to be taken care of by Andrew's CPUID leveling series.
>> > 
>> > My series does take care of all of this.
>> > 
>> > I would prefer that these two changes get taken as soon as they are
>> > ready, so I can rebase.
>> 
>> If we don't need the change quoted above, your rebase will actually
>> be easier to do.
>>
> I'll remove changes in hvm_cpuid() in the next version. Changes in
> xen/include/asm-x86/cpufeature.h will be removed as well, because
> there will be no use of them.

Indeed - this should become a tools only patch.

Jan