[PATCH kvm-unit-tests v2 0/3] svm: INIT test and test_run on selected vcpu

[PATCH kvm-unit-tests v2 0/3] svm: INIT test and test_run on selected vcpu

[Cathy Avery]

INIT intercept test and the ability to execute test_run
on a selected vcpu

Changes from v1:

1) Incorporated feedback:
	- DR6/DR7/CR2/DEBUGCTL should not be need.
	- HSAVE should be set to a different page for each vCPU
	- The on_cpu to set EFER should be in setup_svm
	- The on_cpu to set cr0/cr3/cr4 should be in setup_vm.

2) Execute tests on selected vcpu using on_cpu_async so the tests
may use the on_cpu functions without causing an ipi_lock deadlock.

3) Added additional test svm_init_startup_test which inits the vcpu and
restarts with sipi.

Cathy Avery (3):
  svm: Add ability to execute test via test_run on a vcpu other than
    vcpu 0
  svm: INIT and STARTUP ipi test
  svm: INIT intercept test

 lib/x86/vm.c    | 18 +++++++++
 lib/x86/vm.h    |  7 ++++
 x86/cstart64.S  |  1 +
 x86/svm.c       | 24 +++++++++++-
 x86/svm.h       |  2 +
 x86/svm_tests.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 148 insertions(+), 1 deletion(-)

-- 
2.20.1

[PATCH kvm-unit-tests v2 1/3] svm: Add ability to execute test via test_run on a vcpu other than vcpu 0

[Cathy Avery]

When running tests that can result in a vcpu being left in an
indeterminate state it is useful to be able to run the test on
a vcpu other than 0. This patch allows test_run to be executed
on any vcpu indicated by the on_vcpu member of the svm_test struct.
The initialized state of the vcpu0 registers used to populate the
vmcb is carried forward to the other vcpus.

Signed-off-by: Cathy Avery <cavery@redhat.com>
---
 lib/x86/vm.c | 18 ++++++++++++++++++
 lib/x86/vm.h |  7 +++++++
 x86/svm.c    | 24 +++++++++++++++++++++++-
 x86/svm.h    |  2 ++
 4 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/lib/x86/vm.c b/lib/x86/vm.c
index 41d6d96..e223bb4 100644
--- a/lib/x86/vm.c
+++ b/lib/x86/vm.c
@@ -2,6 +2,7 @@
 #include "libcflat.h"
 #include "vmalloc.h"
 #include "alloc_page.h"
+#include "smp.h"
 
 pteval_t *install_pte(pgd_t *cr3,
 		      int pte_level,
@@ -139,9 +140,18 @@ static void setup_mmu_range(pgd_t *cr3, phys_addr_t start, size_t len)
 	install_pages(cr3, phys, max - phys, (void *)(ulong)phys);
 }
 
+static void set_additional_vcpu_vmregs(struct vm_vcpu_info *info)
+{
+	write_cr3(info->cr3);
+	write_cr4(info->cr4);
+	write_cr0(info->cr0);
+}
+
 void *setup_mmu(phys_addr_t end_of_memory)
 {
     pgd_t *cr3 = alloc_page();
+    struct vm_vcpu_info info;
+    int i;
 
     memset(cr3, 0, PAGE_SIZE);
 
@@ -166,6 +176,14 @@ void *setup_mmu(phys_addr_t end_of_memory)
     printf("cr0 = %lx\n", read_cr0());
     printf("cr3 = %lx\n", read_cr3());
     printf("cr4 = %lx\n", read_cr4());
+
+    info.cr3 = read_cr3();
+    info.cr4 = read_cr4();
+    info.cr0 = read_cr0();
+
+    for (i = 1; i < cpu_count(); i++)
+        on_cpu(i, (void *)set_additional_vcpu_vmregs, &info);
+
     return cr3;
 }
 
diff --git a/lib/x86/vm.h b/lib/x86/vm.h
index 8750a1e..3a1432f 100644
--- a/lib/x86/vm.h
+++ b/lib/x86/vm.h
@@ -45,4 +45,11 @@ static inline void *current_page_table(void)
 
 void split_large_page(unsigned long *ptep, int level);
 void force_4k_page(void *addr);
+
+struct vm_vcpu_info {
+        u64 cr3;
+        u64 cr4;
+        u64 cr0;
+};
+
 #endif
diff --git a/x86/svm.c b/x86/svm.c
index d8c8272..975c477 100644
--- a/x86/svm.c
+++ b/x86/svm.c
@@ -275,6 +275,17 @@ static void test_run(struct svm_test *test)
 	irq_enable();
 
 	report(test->succeeded(test), "%s", test->name);
+
+        if (test->on_vcpu)
+	    test->on_vcpu_done = true;
+}
+
+static void set_additional_vpcu_msr(void *msr_efer)
+{
+	void *hsave = alloc_page();
+
+	wrmsr(MSR_VM_HSAVE_PA, virt_to_phys(hsave));
+	wrmsr(MSR_EFER, (ulong)msr_efer | EFER_SVME | EFER_NX);
 }
 
 static void setup_svm(void)
@@ -294,6 +305,9 @@ static void setup_svm(void)
 	if (!npt_supported())
 		return;
 
+	for (i = 1; i < cpu_count(); i++)
+		on_cpu(i, (void *)set_additional_vpcu_msr, (void *)rdmsr(MSR_EFER));
+
 	printf("NPT detected - running all tests with NPT enabled\n");
 
 	/*
@@ -396,7 +410,15 @@ int main(int ac, char **av)
 		if (svm_tests[i].supported && !svm_tests[i].supported())
 			continue;
 		if (svm_tests[i].v2 == NULL) {
-			test_run(&svm_tests[i]);
+			if (svm_tests[i].on_vcpu) {
+				if (cpu_count() <= svm_tests[i].on_vcpu)
+					continue;
+				on_cpu_async(svm_tests[i].on_vcpu, (void *)test_run, &svm_tests[i]);
+				while (!svm_tests[i].on_vcpu_done)
+					cpu_relax();
+			}
+			else
+				test_run(&svm_tests[i]);
 		} else {
 			vmcb_ident(vmcb);
 			v2_test = &(svm_tests[i]);
diff --git a/x86/svm.h b/x86/svm.h
index f8e7429..1e60d52 100644
--- a/x86/svm.h
+++ b/x86/svm.h
@@ -348,6 +348,8 @@ struct svm_test {
 	ulong scratch;
 	/* Alternative test interface. */
 	void (*v2)(void);
+	int on_vcpu;
+	bool on_vcpu_done;
 };
 
 struct regs {
-- 
2.20.1

[PATCH kvm-unit-tests v2 2/3] svm: INIT and STARTUP ipi test

[Cathy Avery]

Init the vcpu and issue the STARTUP ipi to indicate the vcpu
should execute its startup routine.

Signed-off-by: Cathy Avery <cavery@redhat.com>
---
 x86/cstart64.S  |  1 +
 x86/svm_tests.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/x86/cstart64.S b/x86/cstart64.S
index 3ae98d3..dfd7320 100644
--- a/x86/cstart64.S
+++ b/x86/cstart64.S
@@ -7,6 +7,7 @@
 .globl tss_descr
 .globl gdt64_desc
 .globl online_cpus
+.globl cpu_online_count
 
 ipi_vector = 0x20
 
diff --git a/x86/svm_tests.c b/x86/svm_tests.c
index 3b0d019..698eb20 100644
--- a/x86/svm_tests.c
+++ b/x86/svm_tests.c
@@ -17,6 +17,8 @@ static void *scratch_page;
 
 #define LATENCY_RUNS 1000000
 
+extern u16 cpu_online_count;
+
 u64 tsc_start;
 u64 tsc_end;
 
@@ -1885,6 +1887,58 @@ static bool reg_corruption_check(struct svm_test *test)
     return get_test_stage(test) == 1;
 }
 
+static void get_tss_entry(void *data)
+{
+    struct descriptor_table_ptr gdt;
+    struct segment_desc64 *gdt_table;
+    struct segment_desc64 *tss_entry;
+    u16 tr = 0;
+
+    sgdt(&gdt);
+    tr = str();
+    gdt_table = (struct segment_desc64 *) gdt.base;
+    tss_entry = &gdt_table[tr / sizeof(struct segment_desc64)];
+    *((struct segment_desc64 **)data) = tss_entry;
+}
+
+static int orig_cpu_count;
+
+static void init_startup_prepare(struct svm_test *test)
+{
+    struct segment_desc64 *tss_entry;
+    int i;
+
+    vmcb_ident(vmcb);
+
+    on_cpu(1, get_tss_entry, &tss_entry);
+
+    orig_cpu_count = cpu_online_count;
+
+    apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT,
+                   id_map[1]);
+
+    delay(100000000ULL);
+
+    --cpu_online_count;
+
+    *(uint64_t *)tss_entry &= ~DESC_BUSY;
+
+    apic_icr_write(APIC_DEST_PHYSICAL | APIC_DM_STARTUP, id_map[1]);
+
+    for (i = 0; i < 5 && cpu_online_count < orig_cpu_count; i++)
+       delay(100000000ULL);
+}
+
+static bool init_startup_finished(struct svm_test *test)
+{
+    return true;
+}
+
+static bool init_startup_check(struct svm_test *test)
+{
+    return cpu_online_count == orig_cpu_count;
+}
+
 #define TEST(name) { #name, .v2 = name }
 
 /*
@@ -2198,6 +2252,9 @@ struct svm_test svm_tests[] = {
     { "reg_corruption", default_supported, reg_corruption_prepare,
       default_prepare_gif_clear, reg_corruption_test,
       reg_corruption_finished, reg_corruption_check },
+    { "svm_init_startup_test", smp_supported, init_startup_prepare,
+      default_prepare_gif_clear, null_test,
+      init_startup_finished, init_startup_check },
     TEST(svm_guest_state_test),
     { NULL, NULL, NULL, NULL, NULL, NULL, NULL }
 };
-- 
2.20.1

[PATCH kvm-unit-tests v2 3/3] svm: INIT intercept test

[Cathy Avery]

INIT vcpu 2 and intercept the INIT. This test
will leave the vcpu in an unusable state.

Signed-off-by: Cathy Avery <cavery@redhat.com>
---
 x86/svm_tests.c | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/x86/svm_tests.c b/x86/svm_tests.c
index 698eb20..b43c19f 100644
--- a/x86/svm_tests.c
+++ b/x86/svm_tests.c
@@ -1939,6 +1939,43 @@ static bool init_startup_check(struct svm_test *test)
     return cpu_online_count == orig_cpu_count;
 }
 
+static volatile bool init_intercept;
+
+static void init_intercept_prepare(struct svm_test *test)
+{
+    init_intercept = false;
+    vmcb_ident(vmcb);
+    vmcb->control.intercept |= (1ULL << INTERCEPT_INIT);
+}
+
+static void init_intercept_test(struct svm_test *test)
+{
+    apic_icr_write(APIC_DEST_SELF | APIC_DEST_PHYSICAL | APIC_DM_INIT | APIC_INT_ASSERT, 0);
+}
+
+static bool init_intercept_finished(struct svm_test *test)
+{
+    vmcb->save.rip += 3;
+
+    if (vmcb->control.exit_code != SVM_EXIT_INIT) {
+        report(false, "VMEXIT not due to init intercept. Exit reason 0x%x",
+               vmcb->control.exit_code);
+
+        return true;
+        }
+
+    init_intercept = true;
+
+    report(true, "INIT to vcpu intercepted");
+
+    return true;
+}
+
+static bool init_intercept_check(struct svm_test *test)
+{
+    return init_intercept;
+}
+
 #define TEST(name) { #name, .v2 = name }
 
 /*
@@ -2255,6 +2292,9 @@ struct svm_test svm_tests[] = {
     { "svm_init_startup_test", smp_supported, init_startup_prepare,
       default_prepare_gif_clear, null_test,
       init_startup_finished, init_startup_check },
+    { "svm_init_intercept_test", smp_supported, init_intercept_prepare,
+      default_prepare_gif_clear, init_intercept_test,
+      init_intercept_finished, init_intercept_check, .on_vcpu = 2 },
     TEST(svm_guest_state_test),
     { NULL, NULL, NULL, NULL, NULL, NULL, NULL }
 };
-- 
2.20.1

Re: [PATCH kvm-unit-tests v2 1/3] svm: Add ability to execute test via test_run on a vcpu other than vcpu 0

[Paolo Bonzini]

On 17/07/20 13:34, Cathy Avery wrote:
> When running tests that can result in a vcpu being left in an
> indeterminate state it is useful to be able to run the test on
> a vcpu other than 0. This patch allows test_run to be executed
> on any vcpu indicated by the on_vcpu member of the svm_test struct.
> The initialized state of the vcpu0 registers used to populate the
> vmcb is carried forward to the other vcpus.
> 
> Signed-off-by: Cathy Avery <cavery@redhat.com>
> ---
>   lib/x86/vm.c | 18 ++++++++++++++++++
>   lib/x86/vm.h |  7 +++++++
>   x86/svm.c    | 24 +++++++++++++++++++++++-
>   x86/svm.h    |  2 ++
>   4 files changed, 50 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/x86/vm.c b/lib/x86/vm.c
> index 41d6d96..e223bb4 100644
> --- a/lib/x86/vm.c
> +++ b/lib/x86/vm.c
> @@ -2,6 +2,7 @@
>   #include "libcflat.h"
>   #include "vmalloc.h"
>   #include "alloc_page.h"
> +#include "smp.h"
>   
>   pteval_t *install_pte(pgd_t *cr3,
>   		      int pte_level,
> @@ -139,9 +140,18 @@ static void setup_mmu_range(pgd_t *cr3, phys_addr_t start, size_t len)
>   	install_pages(cr3, phys, max - phys, (void *)(ulong)phys);
>   }
>   
> +static void set_additional_vcpu_vmregs(struct vm_vcpu_info *info)
> +{
> +	write_cr3(info->cr3);
> +	write_cr4(info->cr4);
> +	write_cr0(info->cr0);
> +}
> +
>   void *setup_mmu(phys_addr_t end_of_memory)
>   {
>       pgd_t *cr3 = alloc_page();
> +    struct vm_vcpu_info info;
> +    int i;
>   
>       memset(cr3, 0, PAGE_SIZE);
>   
> @@ -166,6 +176,14 @@ void *setup_mmu(phys_addr_t end_of_memory)
>       printf("cr0 = %lx\n", read_cr0());
>       printf("cr3 = %lx\n", read_cr3());
>       printf("cr4 = %lx\n", read_cr4());
> +
> +    info.cr3 = read_cr3();
> +    info.cr4 = read_cr4();
> +    info.cr0 = read_cr0();
> +
> +    for (i = 1; i < cpu_count(); i++)
> +        on_cpu(i, (void *)set_additional_vcpu_vmregs, &info);
> +
>       return cr3;
>   }
>   
> diff --git a/lib/x86/vm.h b/lib/x86/vm.h
> index 8750a1e..3a1432f 100644
> --- a/lib/x86/vm.h
> +++ b/lib/x86/vm.h
> @@ -45,4 +45,11 @@ static inline void *current_page_table(void)
>   
>   void split_large_page(unsigned long *ptep, int level);
>   void force_4k_page(void *addr);
> +
> +struct vm_vcpu_info {
> +        u64 cr3;
> +        u64 cr4;
> +        u64 cr0;
> +};
> +
>   #endif
> diff --git a/x86/svm.c b/x86/svm.c
> index d8c8272..975c477 100644
> --- a/x86/svm.c
> +++ b/x86/svm.c
> @@ -275,6 +275,17 @@ static void test_run(struct svm_test *test)
>   	irq_enable();
>   
>   	report(test->succeeded(test), "%s", test->name);
> +
> +        if (test->on_vcpu)
> +	    test->on_vcpu_done = true;
> +}
> +
> +static void set_additional_vpcu_msr(void *msr_efer)
> +{
> +	void *hsave = alloc_page();
> +
> +	wrmsr(MSR_VM_HSAVE_PA, virt_to_phys(hsave));
> +	wrmsr(MSR_EFER, (ulong)msr_efer | EFER_SVME | EFER_NX);
>   }
>   
>   static void setup_svm(void)
> @@ -294,6 +305,9 @@ static void setup_svm(void)
>   	if (!npt_supported())
>   		return;
>   
> +	for (i = 1; i < cpu_count(); i++)
> +		on_cpu(i, (void *)set_additional_vpcu_msr, (void *)rdmsr(MSR_EFER));
> +
>   	printf("NPT detected - running all tests with NPT enabled\n");
>   
>   	/*
> @@ -396,7 +410,15 @@ int main(int ac, char **av)
>   		if (svm_tests[i].supported && !svm_tests[i].supported())
>   			continue;
>   		if (svm_tests[i].v2 == NULL) {
> -			test_run(&svm_tests[i]);
> +			if (svm_tests[i].on_vcpu) {
> +				if (cpu_count() <= svm_tests[i].on_vcpu)
> +					continue;
> +				on_cpu_async(svm_tests[i].on_vcpu, (void *)test_run, &svm_tests[i]);
> +				while (!svm_tests[i].on_vcpu_done)
> +					cpu_relax();
> +			}
> +			else
> +				test_run(&svm_tests[i]);
>   		} else {
>   			vmcb_ident(vmcb);
>   			v2_test = &(svm_tests[i]);
> diff --git a/x86/svm.h b/x86/svm.h
> index f8e7429..1e60d52 100644
> --- a/x86/svm.h
> +++ b/x86/svm.h
> @@ -348,6 +348,8 @@ struct svm_test {
>   	ulong scratch;
>   	/* Alternative test interface. */
>   	void (*v2)(void);
> +	int on_vcpu;
> +	bool on_vcpu_done;
>   };
>   
>   struct regs {
> 

Queued, thanks.

Paolo