[Qemu-devel] RESEND: [PATCH v3 0/5] This patch-set is to enable Guest

[Qemu-devel] RESEND: [PATCH v3 0/5] This patch-set is to enable Guest

[Yang Weijiang]

Control-flow Enforcement Technology (CET) provides protection against
return/jump-oriented programming (ROP) attacks. To make kvm Guest OS own
the capability, this patch-set is required. It enables CET related CPUID
report, xsaves/xrstors and live-migration etc. in Qemu.

Changelog:

 v3:
 - Add CET MSR save/restore support for live-migration.

 v2:
 - In CPUID.(EAX=d, ECX=1), set return ECX[n] = 0 if bit n corresponds
   to a bit in MSR_IA32_XSS.
 - In CPUID.(EAX=d, ECX=n), set return ECX = 1 if bit n corresponds
   to a bit in MSR_IA32_XSS.
 - Skip Supervisor mode xsave component when calculate User mode
   xave component size in xsave_area_size() and x86_cpu_reset().

Yang Weijiang (5):
  Add CET xsaves/xrstors related macros and structures.
  Add CET SHSTK and IBT CPUID feature-word definitions.
  Add hepler functions for CPUID xsave area size calculation.
  Report CPUID xsave area support for CET.
  Add CET MSR save/restore support for migration

 target/i386/cpu.c     |  73 ++++++++++++++++++++++++++++--
 target/i386/cpu.h     |  48 +++++++++++++++++++-
 target/i386/kvm.c     |  27 ++++++++++++
 target/i386/machine.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 244 insertions(+), 4 deletions(-)

-- 
2.17.1

[Qemu-devel] [PATCH v3 1/5] Add CET xsaves/xrstors related macros and structures.

[Yang Weijiang]

CET protection in user mode and kernel mode relies on
specific MSRs, these MSRs' contents are automatically
saved/restored by xsaves/xrstors instructions.

Signed-off-by: Zhang Yi <yi.z.zhang@linux.intel.com>
Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.h | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 9c52d0cbeb..f3f724d8e6 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -469,6 +469,9 @@ typedef enum X86Seg {
 #define XSTATE_ZMM_Hi256_BIT            6
 #define XSTATE_Hi16_ZMM_BIT             7
 #define XSTATE_PKRU_BIT                 9
+#define XSTATE_RESERVED_BIT             10
+#define XSTATE_CET_U_BIT                11
+#define XSTATE_CET_S_BIT                12
 
 #define XSTATE_FP_MASK                  (1ULL << XSTATE_FP_BIT)
 #define XSTATE_SSE_MASK                 (1ULL << XSTATE_SSE_BIT)
@@ -479,6 +482,19 @@ typedef enum X86Seg {
 #define XSTATE_ZMM_Hi256_MASK           (1ULL << XSTATE_ZMM_Hi256_BIT)
 #define XSTATE_Hi16_ZMM_MASK            (1ULL << XSTATE_Hi16_ZMM_BIT)
 #define XSTATE_PKRU_MASK                (1ULL << XSTATE_PKRU_BIT)
+#define XSTATE_RESERVED_MASK            (1ULL << XSTATE_RESERVED_BIT)
+#define XSTATE_CET_U_MASK               (1ULL << XSTATE_CET_U_BIT)
+#define XSTATE_CET_S_MASK               (1ULL << XSTATE_CET_S_BIT)
+
+/* CPUID feature bits available in XCR0 */
+#define CPUID_XSTATE_USER_MASK  (XSTATE_FP_MASK | XSTATE_SSE_MASK \
+                                | XSTATE_YMM_MASK | XSTATE_BNDREGS_MASK \
+                                | XSTATE_BNDCSR_MASK | XSTATE_OPMASK_MASK \
+                                | XSTATE_ZMM_Hi256_MASK \
+                                | XSTATE_Hi16_ZMM_MASK | XSTATE_PKRU_MASK)
+
+/* CPUID feature bits available in XSS */
+#define CPUID_XSTATE_KERNEL_MASK    (XSTATE_CET_U_MASK | XSTATE_CET_S_MASK)
 
 /* CPUID feature words */
 typedef enum FeatureWord {
@@ -503,6 +519,8 @@ typedef enum FeatureWord {
     FEAT_XSAVE_COMP_LO, /* CPUID[EAX=0xd,ECX=0].EAX */
     FEAT_XSAVE_COMP_HI, /* CPUID[EAX=0xd,ECX=0].EDX */
     FEAT_ARCH_CAPABILITIES,
+    FEAT_XSAVE_SV_LO,   /* CPUID[EAX=0xd,ECX=1].ECX */
+    FEAT_XSAVE_SV_HI,   /* CPUID[EAX=0xd,ECX=1].EDX */
     FEATURE_WORDS,
 } FeatureWord;
 
@@ -687,7 +705,7 @@ typedef uint32_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_7_0_ECX_LA57     (1U << 16)
 #define CPUID_7_0_ECX_RDPID    (1U << 22)
 #define CPUID_7_0_ECX_CLDEMOTE (1U << 25)  /* CLDEMOTE Instruction */
-
+#define CPUID_7_0_ECX_CET_SHSTK     (1U << 7)  /* CET SHSTK feature bit */
 #define CPUID_7_0_EDX_AVX512_4VNNIW (1U << 2) /* AVX512 Neural Network Instructions */
 #define CPUID_7_0_EDX_AVX512_4FMAPS (1U << 3) /* AVX512 Multiply Accumulation Single Precision */
 #define CPUID_7_0_EDX_PCONFIG (1U << 18)       /* Platform Configuration */
@@ -1021,6 +1039,19 @@ typedef struct XSavePKRU {
     uint32_t padding;
 } XSavePKRU;
 
+/* Ext. save area 11: User mode CET state */
+typedef struct XSaveCETU {
+    uint64_t u_cet;
+    uint64_t user_ssp;
+} XSaveCETU;
+
+/* Ext. save area 12: Supervisor mode CET state */
+typedef struct XSaveCETS {
+    uint64_t kernel_ssp;
+    uint64_t pl1_ssp;
+    uint64_t pl2_ssp;
+} XSaveCETS;
+
 typedef struct X86XSaveArea {
     X86LegacyXSaveArea legacy;
     X86XSaveHeader header;
@@ -1039,6 +1070,9 @@ typedef struct X86XSaveArea {
     XSaveHi16_ZMM hi16_zmm_state;
     /* PKRU State: */
     XSavePKRU pkru_state;
+    /* CET State: */
+    XSaveCETU cet_u;
+    XSaveCETS cet_s;
 } X86XSaveArea;
 
 QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, avx_state) != 0x240);
-- 
2.17.1

[Qemu-devel] [PATCH v3 2/5] Add CET SHSTK and IBT CPUID feature-word definitions.

[Yang Weijiang]

XSS[bit 11] and XSS[bit 12] correspond to CET
user mode area and supervisor mode area respectively.

Signed-off-by: Zhang Yi <yi.z.zhang@linux.intel.com>
Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.c | 37 +++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index f81d35e1f9..f6c7bdf6fe 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -1018,7 +1018,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .type = CPUID_FEATURE_WORD,
         .feat_names = {
             NULL, "avx512vbmi", "umip", "pku",
-            NULL /* ospke */, NULL, "avx512vbmi2", NULL,
+            NULL /* ospke */, NULL, "avx512vbmi2", "shstk",
             "gfni", "vaes", "vpclmulqdq", "avx512vnni",
             "avx512bitalg", NULL, "avx512-vpopcntdq", NULL,
             "la57", NULL, NULL, NULL,
@@ -1041,7 +1041,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
             NULL, NULL, NULL, NULL,
             NULL, NULL, NULL, NULL,
             NULL, NULL, "pconfig", NULL,
-            NULL, NULL, NULL, NULL,
+            "ibt", NULL, NULL, NULL,
             NULL, NULL, "spec-ctrl", NULL,
             NULL, "arch-capabilities", NULL, "ssbd",
         },
@@ -1162,6 +1162,25 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
             }
         },
     },
+    /* Below are CET supervisor xsave features */
+    [FEAT_XSAVE_SV_LO] = {
+        .type = CPUID_FEATURE_WORD,
+        .cpuid = {
+            .eax = 0xD,
+            .needs_ecx = true,
+            .ecx = 1,
+            .reg = R_ECX,
+        },
+    },
+    [FEAT_XSAVE_SV_HI] = {
+        .type = CPUID_FEATURE_WORD,
+        .cpuid = {
+            .eax = 0xD,
+            .needs_ecx = true,
+            .ecx = 1,
+            .reg = R_EDX
+        },
+    }
 };
 
 typedef struct X86RegisterInfo32 {
@@ -1233,6 +1252,14 @@ static const ExtSaveArea x86_ext_save_areas[] = {
           { .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_PKU,
             .offset = offsetof(X86XSaveArea, pkru_state),
             .size = sizeof(XSavePKRU) },
+    [XSTATE_CET_U_BIT] = {
+            .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_CET_SHSTK,
+            .offset = 0 /*supervisor mode component, offset = 0 */,
+            .size = sizeof(XSaveCETU) },
+    [XSTATE_CET_S_BIT] = {
+            .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_CET_SHSTK,
+            .offset = 0 /*supervisor mode component, offset = 0 */,
+            .size = sizeof(XSaveCETS) },
 };
 
 static uint32_t xsave_area_size(uint64_t mask)
@@ -1243,6 +1270,9 @@ static uint32_t xsave_area_size(uint64_t mask)
     for (i = 0; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
         const ExtSaveArea *esa = &x86_ext_save_areas[i];
         if ((mask >> i) & 1) {
+            if (i >= 2 && !esa->offset) {
+                continue;
+            }
             ret = MAX(ret, esa->offset + esa->size);
         }
     }
@@ -4657,6 +4687,9 @@ static void x86_cpu_reset(CPUState *s)
     }
     for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
         const ExtSaveArea *esa = &x86_ext_save_areas[i];
+        if (!esa->offset) {
+            continue;
+        }
         if (env->features[esa->feature] & esa->bits) {
             xcr0 |= 1ull << i;
         }
-- 
2.17.1

[Qemu-devel] [PATCH v3 3/5] Add hepler functions for CPUID xsave area size calculation.

[Yang Weijiang]

These functions are called when return CPUID xsave area
size information.

Signed-off-by: Zhang Yi <yi.z.zhang@linux.intel.com>
Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index f6c7bdf6fe..d8c36e0f2f 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -1284,12 +1284,34 @@ static inline bool accel_uses_host_cpuid(void)
     return kvm_enabled() || hvf_enabled();
 }
 
+static uint32_t xsave_area_size_compacted(uint64_t mask)
+{
+    int i;
+    uint64_t ret = 0;
+    uint32_t offset;
+
+    for (i = 0; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+        const ExtSaveArea *esa = &x86_ext_save_areas[i];
+        offset = i >= 2 ? ret : esa->offset;
+        if ((mask >> i) & 1) {
+            ret = MAX(ret, offset + esa->size);
+        }
+    }
+    return ret;
+}
+
 static inline uint64_t x86_cpu_xsave_components(X86CPU *cpu)
 {
     return ((uint64_t)cpu->env.features[FEAT_XSAVE_COMP_HI]) << 32 |
            cpu->env.features[FEAT_XSAVE_COMP_LO];
 }
 
+static inline uint64_t x86_cpu_xsave_sv_components(X86CPU *cpu)
+{
+    return ((uint64_t)cpu->env.features[FEAT_XSAVE_SV_HI]) << 32 |
+           cpu->env.features[FEAT_XSAVE_SV_LO];
+}
+
 const char *get_register_name_32(unsigned int reg)
 {
     if (reg >= CPU_NB_REGS32) {
@@ -4919,8 +4941,10 @@ static void x86_cpu_enable_xsave_components(X86CPU *cpu)
         }
     }
 
-    env->features[FEAT_XSAVE_COMP_LO] = mask;
+    env->features[FEAT_XSAVE_COMP_LO] = mask & CPUID_XSTATE_USER_MASK;
     env->features[FEAT_XSAVE_COMP_HI] = mask >> 32;
+    env->features[FEAT_XSAVE_SV_LO] = mask & CPUID_XSTATE_KERNEL_MASK;
+    env->features[FEAT_XSAVE_SV_HI] = mask >> 32;
 }
 
 /***** Steps involved on loading and filtering CPUID data
-- 
2.17.1

[Qemu-devel] [PATCH v3 4/5] Report CPUID xsave area support for CET.

[Yang Weijiang]

CPUID bit definition as below:
CPUID.(EAX=d, ECX=1):ECX.CET_U(bit 11): user mode state
CPUID.(EAX=d, ECX=1):ECX.CET_S(bit 12): kernel mode state

Signed-off-by: Zhang Yi <yi.z.zhang@linux.intel.com>
Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index d8c36e0f2f..15e2d5e009 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -4399,12 +4399,22 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
             *ebx = xsave_area_size(env->xcr0);
         } else if (count == 1) {
             *eax = env->features[FEAT_XSAVE];
+            *ecx = env->features[FEAT_XSAVE_SV_LO];
+            *edx = env->features[FEAT_XSAVE_SV_HI];
+            *ebx = xsave_area_size_compacted(x86_cpu_xsave_components(cpu) |
+                    x86_cpu_xsave_sv_components(cpu));
         } else if (count < ARRAY_SIZE(x86_ext_save_areas)) {
             if ((x86_cpu_xsave_components(cpu) >> count) & 1) {
                 const ExtSaveArea *esa = &x86_ext_save_areas[count];
                 *eax = esa->size;
                 *ebx = esa->offset;
             }
+            if ((x86_cpu_xsave_sv_components(cpu) >> count) & 1) {
+                const ExtSaveArea *esa_sv = &x86_ext_save_areas[count];
+                *eax = esa_sv->size;
+                *ebx = 0;
+                *ecx = 1;
+            }
         }
         break;
     }
-- 
2.17.1

[Qemu-devel] [PATCH v3 5/5] Add CET MSR save/restore support for migration

[Yang Weijiang]

To support features such as live-migration,
CET runtime MSRs need to be saved in source machine and
restored on destination machine, this patch is to save
and restore CET_U, CET_S, PL0_SSP, PL3_SSP and SSP_TABL_ADDR
MSRs.

Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.h     |  12 +++++
 target/i386/kvm.c     |  33 ++++++++++++++
 target/i386/machine.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index f3f724d8e6..f350684895 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -460,6 +460,12 @@ typedef enum X86Seg {
 #define MSR_IA32_BNDCFGS                0x00000d90
 #define MSR_IA32_XSS                    0x00000da0
 
+#define MSR_IA32_U_CET                  0x6a0
+#define MSR_IA32_S_CET                  0x6a2
+#define MSR_IA32_PL0_SSP                0x6a4
+#define MSR_IA32_PL3_SSP                0x6a7
+#define MSR_IA32_INTR_SSP_TABL          0x6a8
+
 #define XSTATE_FP_BIT                   0
 #define XSTATE_SSE_BIT                  1
 #define XSTATE_YMM_BIT                  2
@@ -1325,6 +1331,12 @@ typedef struct CPUX86State {
 
     uintptr_t retaddr;
 
+    uint64_t u_cet;
+    uint64_t s_cet;
+    uint64_t pl0_ssp;
+    uint64_t pl3_ssp;
+    uint64_t ssp_tabl_addr;
+
     /* Fields up to this point are cleared by a CPU reset */
     struct {} end_reset_fields;
 
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index f524e7d929..2ab3c977a4 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -63,6 +63,8 @@
 /* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus
  * 255 kvm_msr_entry structs */
 #define MSR_BUF_SIZE 4096
+#define HAS_CET_CAP(env)  (env->features[FEAT_7_0_ECX] & 0x80 || \
+                           env->features[FEAT_7_0_EDX] & 0x100000)
 
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_INFO(SET_TSS_ADDR),
@@ -2197,6 +2199,14 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
         }
     }
 
+    if (HAS_CET_CAP(env)) {
+        kvm_msr_entry_add(cpu, MSR_IA32_U_CET, env->u_cet);
+        kvm_msr_entry_add(cpu, MSR_IA32_S_CET, env->s_cet);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL0_SSP, env->pl0_ssp);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL3_SSP, env->pl3_ssp);
+        kvm_msr_entry_add(cpu, MSR_IA32_INTR_SSP_TABL, env->ssp_tabl_addr);
+    }
+
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
     if (ret < 0) {
         return ret;
@@ -2516,6 +2526,14 @@ static int kvm_get_msrs(X86CPU *cpu)
         }
     }
 
+    if (HAS_CET_CAP(env)) {
+        kvm_msr_entry_add(cpu, MSR_IA32_U_CET, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_S_CET, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL0_SSP, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL3_SSP, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_INTR_SSP_TABL, 0);
+    }
+
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf);
     if (ret < 0) {
         return ret;
@@ -2789,6 +2807,21 @@ static int kvm_get_msrs(X86CPU *cpu)
         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
             env->msr_rtit_addrs[index - MSR_IA32_RTIT_ADDR0_A] = msrs[i].data;
             break;
+        case MSR_IA32_U_CET:
+            env->u_cet = msrs[i].data;
+            break;
+        case MSR_IA32_S_CET:
+            env->s_cet = msrs[i].data;
+            break;
+        case MSR_IA32_PL0_SSP:
+            env->pl0_ssp = msrs[i].data;
+            break;
+        case MSR_IA32_PL3_SSP:
+            env->pl3_ssp = msrs[i].data;
+            break;
+        case MSR_IA32_INTR_SSP_TABL:
+            env->ssp_tabl_addr = msrs[i].data;
+            break;
         }
     }
 
diff --git a/target/i386/machine.c b/target/i386/machine.c
index 225b5d433b..5f8a12ca30 100644
--- a/target/i386/machine.c
+++ b/target/i386/machine.c
@@ -810,6 +810,101 @@ static const VMStateDescription vmstate_xss = {
     }
 };
 
+static bool u_cet_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->u_cet != 0;
+}
+
+static const VMStateDescription vmstate_u_cet = {
+    .name = "cpu/u_cet",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = u_cet_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.u_cet, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool s_cet_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->s_cet != 0;
+}
+
+static const VMStateDescription vmstate_s_cet = {
+    .name = "cpu/s_cet",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = s_cet_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.s_cet, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool pl0_ssp_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pl0_ssp != 0;
+}
+
+static const VMStateDescription vmstate_pl0_ssp = {
+    .name = "cpu/pl0_ssp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pl0_ssp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.pl0_ssp, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool pl3_ssp_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pl3_ssp != 0;
+}
+
+static const VMStateDescription vmstate_pl3_ssp = {
+    .name = "cpu/pl3_ssp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pl3_ssp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.pl3_ssp, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool ssp_tabl_addr_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->ssp_tabl_addr != 0;
+}
+
+static const VMStateDescription vmstate_ssp_tabl_addr = {
+    .name = "cpu/ssp_tabl_addr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ssp_tabl_addr_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.ssp_tabl_addr, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 #ifdef TARGET_X86_64
 static bool pkru_needed(void *opaque)
 {
@@ -1089,6 +1184,11 @@ VMStateDescription vmstate_x86_cpu = {
         &vmstate_msr_intel_pt,
         &vmstate_msr_virt_ssbd,
         &vmstate_svm_npt,
+        &vmstate_u_cet,
+        &vmstate_s_cet,
+        &vmstate_pl0_ssp,
+        &vmstate_pl3_ssp,
+        &vmstate_ssp_tabl_addr,
         NULL
     }
 };
-- 
2.17.1

Re: [Qemu-devel] [PATCH v3 5/5] Add CET MSR save/restore support for migration

[Paolo Bonzini]

On 25/02/19 14:37, Yang Weijiang wrote:
> +#define MSR_IA32_PL0_SSP                0x6a4
> +#define MSR_IA32_PL3_SSP                0x6a7

Please add PL1 and PL2 SSP too.

Paolo

Re: [Qemu-devel] [PATCH v3 1/5] Add CET xsaves/xrstors related macros and structures.

[Paolo Bonzini]

On 25/02/19 14:37, Yang Weijiang wrote:
> +
>  typedef struct X86XSaveArea {
>      X86LegacyXSaveArea legacy;
>      X86XSaveHeader header;
> @@ -1039,6 +1070,9 @@ typedef struct X86XSaveArea {
>      XSaveHi16_ZMM hi16_zmm_state;
>      /* PKRU State: */
>      XSavePKRU pkru_state;
> +    /* CET State: */
> +    XSaveCETU cet_u;
> +    XSaveCETS cet_s;
>  } X86XSaveArea;
>  

Hi,

CET state should be retrieved or set not via XSAVE, but rather with
KVM_GET/SET_MSR, so this part is not needed.

Paolo

Re: [Qemu-devel] [PATCH v3 4/5] Report CPUID xsave area support for CET.

[Paolo Bonzini]

On 25/02/19 14:37, Yang Weijiang wrote:
> @@ -4399,12 +4399,22 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
>              *ebx = xsave_area_size(env->xcr0);
>          } else if (count == 1) {
>              *eax = env->features[FEAT_XSAVE];
> +            *ecx = env->features[FEAT_XSAVE_SV_LO];
> +            *edx = env->features[FEAT_XSAVE_SV_HI];
> +            *ebx = xsave_area_size_compacted(x86_cpu_xsave_components(cpu) |
> +                    x86_cpu_xsave_sv_components(cpu));

*ebx should not be necessary, KVM computes it automatically.  In fact it
doesn't depend on x86_cpu_xsave_components and
x86_cpu_xsave_sv_components, it depends on XCR0 and MSR_IA32_XSS, so you
can just drop it.

Paolo

[Qemu-devel] [PATCH v3 5/5] Add CET MSR save/restore support for migration

[Yang Weijiang]

To support features such as live-migration,
CET runtime MSRs need to be saved in source machine and
restored on destination machine, this patch is to save
and restore CET_U, CET_S, PL0_SSP, PL3_SSP and SSP_TABL_ADDR
MSRs.

Signed-off-by: Yang Weijiang <weijiang.yang@intel.com>
---
 target/i386/cpu.h     |  12 +++++
 target/i386/kvm.c     |  33 ++++++++++++++
 target/i386/machine.c | 100 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index f3f724d8e6..f350684895 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -460,6 +460,12 @@ typedef enum X86Seg {
 #define MSR_IA32_BNDCFGS                0x00000d90
 #define MSR_IA32_XSS                    0x00000da0
 
+#define MSR_IA32_U_CET                  0x6a0
+#define MSR_IA32_S_CET                  0x6a2
+#define MSR_IA32_PL0_SSP                0x6a4
+#define MSR_IA32_PL3_SSP                0x6a7
+#define MSR_IA32_INTR_SSP_TABL          0x6a8
+
 #define XSTATE_FP_BIT                   0
 #define XSTATE_SSE_BIT                  1
 #define XSTATE_YMM_BIT                  2
@@ -1325,6 +1331,12 @@ typedef struct CPUX86State {
 
     uintptr_t retaddr;
 
+    uint64_t u_cet;
+    uint64_t s_cet;
+    uint64_t pl0_ssp;
+    uint64_t pl3_ssp;
+    uint64_t ssp_tabl_addr;
+
     /* Fields up to this point are cleared by a CPU reset */
     struct {} end_reset_fields;
 
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index f524e7d929..2ab3c977a4 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -63,6 +63,8 @@
 /* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus
  * 255 kvm_msr_entry structs */
 #define MSR_BUF_SIZE 4096
+#define HAS_CET_CAP(env)  (env->features[FEAT_7_0_ECX] & 0x80 || \
+                           env->features[FEAT_7_0_EDX] & 0x100000)
 
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_INFO(SET_TSS_ADDR),
@@ -2197,6 +2199,14 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
         }
     }
 
+    if (HAS_CET_CAP(env)) {
+        kvm_msr_entry_add(cpu, MSR_IA32_U_CET, env->u_cet);
+        kvm_msr_entry_add(cpu, MSR_IA32_S_CET, env->s_cet);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL0_SSP, env->pl0_ssp);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL3_SSP, env->pl3_ssp);
+        kvm_msr_entry_add(cpu, MSR_IA32_INTR_SSP_TABL, env->ssp_tabl_addr);
+    }
+
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
     if (ret < 0) {
         return ret;
@@ -2516,6 +2526,14 @@ static int kvm_get_msrs(X86CPU *cpu)
         }
     }
 
+    if (HAS_CET_CAP(env)) {
+        kvm_msr_entry_add(cpu, MSR_IA32_U_CET, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_S_CET, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL0_SSP, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_PL3_SSP, 0);
+        kvm_msr_entry_add(cpu, MSR_IA32_INTR_SSP_TABL, 0);
+    }
+
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf);
     if (ret < 0) {
         return ret;
@@ -2789,6 +2807,21 @@ static int kvm_get_msrs(X86CPU *cpu)
         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
             env->msr_rtit_addrs[index - MSR_IA32_RTIT_ADDR0_A] = msrs[i].data;
             break;
+        case MSR_IA32_U_CET:
+            env->u_cet = msrs[i].data;
+            break;
+        case MSR_IA32_S_CET:
+            env->s_cet = msrs[i].data;
+            break;
+        case MSR_IA32_PL0_SSP:
+            env->pl0_ssp = msrs[i].data;
+            break;
+        case MSR_IA32_PL3_SSP:
+            env->pl3_ssp = msrs[i].data;
+            break;
+        case MSR_IA32_INTR_SSP_TABL:
+            env->ssp_tabl_addr = msrs[i].data;
+            break;
         }
     }
 
diff --git a/target/i386/machine.c b/target/i386/machine.c
index 225b5d433b..5f8a12ca30 100644
--- a/target/i386/machine.c
+++ b/target/i386/machine.c
@@ -810,6 +810,101 @@ static const VMStateDescription vmstate_xss = {
     }
 };
 
+static bool u_cet_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->u_cet != 0;
+}
+
+static const VMStateDescription vmstate_u_cet = {
+    .name = "cpu/u_cet",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = u_cet_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.u_cet, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool s_cet_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->s_cet != 0;
+}
+
+static const VMStateDescription vmstate_s_cet = {
+    .name = "cpu/s_cet",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = s_cet_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.s_cet, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool pl0_ssp_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pl0_ssp != 0;
+}
+
+static const VMStateDescription vmstate_pl0_ssp = {
+    .name = "cpu/pl0_ssp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pl0_ssp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.pl0_ssp, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool pl3_ssp_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pl3_ssp != 0;
+}
+
+static const VMStateDescription vmstate_pl3_ssp = {
+    .name = "cpu/pl3_ssp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pl3_ssp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.pl3_ssp, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool ssp_tabl_addr_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->ssp_tabl_addr != 0;
+}
+
+static const VMStateDescription vmstate_ssp_tabl_addr = {
+    .name = "cpu/ssp_tabl_addr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = ssp_tabl_addr_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.ssp_tabl_addr, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 #ifdef TARGET_X86_64
 static bool pkru_needed(void *opaque)
 {
@@ -1089,6 +1184,11 @@ VMStateDescription vmstate_x86_cpu = {
         &vmstate_msr_intel_pt,
         &vmstate_msr_virt_ssbd,
         &vmstate_svm_npt,
+        &vmstate_u_cet,
+        &vmstate_s_cet,
+        &vmstate_pl0_ssp,
+        &vmstate_pl3_ssp,
+        &vmstate_ssp_tabl_addr,
         NULL
     }
 };
-- 
2.17.1