[kvm-unit-tests PATCH 39/39] x86/access: nVMX: Add "access" test variants to invalidate via (INV)VPID

[Sean Christopherson <seanjc@google.com>]

Add three variants of the #PF interception access test to handle TLB
invalidations by relying on VPID rules.  Intercept the access test's
INVLPG and perform invalidation by:

  1. Implicity flush on VM-Enter by disabling VPID
  2. Explicitly perform INVVPID on the target address
  3. Implicitly "flush" by moving to a new VPID

Case #3 exposes a bug where KVM fails to update unsync SPTEs when using
shadow paging and L1 changes the VPID it uses for L2, i.e. vmcs12->vpid.

Signed-off-by: Sean Christopherson <seanjc@google.com>
---
 x86/unittests.cfg |  6 ++--
 x86/vmx_tests.c   | 89 ++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/x86/unittests.cfg b/x86/unittests.cfg
index f3f9f17..80875d2 100644
--- a/x86/unittests.cfg
+++ b/x86/unittests.cfg
@@ -284,7 +284,7 @@ arch = i386
 
 [vmx]
 file = vmx.flat
-extra_params = -cpu max,+vmx -append "-exit_monitor_from_l2_test -ept_access* -vmx_smp* -vmx_vmcs_shadow_test -atomic_switch_overflow_msrs_test -vmx_init_signal_test -vmx_apic_passthrough_tpr_threshold_test -apic_reg_virt_test -virt_x2apic_mode_test -vmx_pf_exception_test"
+extra_params = -cpu max,+vmx -append "-exit_monitor_from_l2_test -ept_access* -vmx_smp* -vmx_vmcs_shadow_test -atomic_switch_overflow_msrs_test -vmx_init_signal_test -vmx_apic_passthrough_tpr_threshold_test -apic_reg_virt_test -virt_x2apic_mode_test -vmx_pf_exception_test -vmx_pf_no_vpid_test -vmx_pf_vpid_test"
 arch = x86_64
 groups = vmx
 
@@ -353,13 +353,13 @@ groups = vmx
 
 [vmx_pf_exception_test]
 file = vmx.flat
-extra_params = -cpu max,+vmx -append vmx_pf_exception_test
+extra_params = -cpu max,+vmx -append "vmx_pf_exception_test vmx_pf_no_vpid_test vmx_pf_vpid_test vmx_pf_invvpid_test"
 arch = x86_64
 groups = vmx nested_exception
 
 [vmx_pf_exception_test_reduced_maxphyaddr]
 file = vmx.flat
-extra_params = -cpu IvyBridge,phys-bits=36,host-phys-bits=off,+vmx -append vmx_pf_exception_test
+extra_params = -cpu IvyBridge,phys-bits=36,host-phys-bits=off,+vmx -append "vmx_pf_exception_test vmx_pf_no_vpid_test vmx_pf_vpid_test vmx_pf_invvpid_test"
 arch = x86_64
 groups = vmx nested_exception
 check = /sys/module/kvm_intel/parameters/allow_smaller_maxphyaddr=Y
diff --git a/x86/vmx_tests.c b/x86/vmx_tests.c
index 172d385..3d57ed6 100644
--- a/x86/vmx_tests.c
+++ b/x86/vmx_tests.c
@@ -10575,13 +10575,21 @@ static void vmx_pf_exception_test_guest(void)
 	ac_test_run(PT_LEVEL_PML4);
 }
 
-static void vmx_pf_exception_test(void)
+typedef void (*invalidate_tlb_t)(void *data);
+
+static void __vmx_pf_exception_test(invalidate_tlb_t inv_fn, void *data)
 {
 	u64 efer;
 	struct cpuid cpuid;
 
 	test_set_guest(vmx_pf_exception_test_guest);
 
+	/* Intercept INVLPG when to perform TLB invalidation from L1 (this). */
+	if (inv_fn)
+		vmcs_set_bits(CPU_EXEC_CTRL0, CPU_INVLPG);
+	else
+		vmcs_clear_bits(CPU_EXEC_CTRL0, CPU_INVLPG);
+
 	enter_guest();
 
 	while (vmcs_read(EXI_REASON) != VMX_VMCALL) {
@@ -10605,6 +10613,9 @@ static void vmx_pf_exception_test(void)
 			regs.rcx = cpuid.c;
 			regs.rdx = cpuid.d;
 			break;
+		case VMX_INVLPG:
+			inv_fn(data);
+			break;
 		default:
 			assert_msg(false,
 				"Unexpected exit to L1, exit_reason: %s (0x%lx)",
@@ -10617,6 +10628,79 @@ static void vmx_pf_exception_test(void)
 
 	assert_exit_reason(VMX_VMCALL);
 }
+
+static void vmx_pf_exception_test(void)
+{
+	__vmx_pf_exception_test(NULL, NULL);
+}
+
+static void invalidate_tlb_no_vpid(void *data)
+{
+	/* If VPID is disabled, the TLB is flushed on VM-Enter and VM-Exit. */
+}
+
+static void vmx_pf_no_vpid_test(void)
+{
+	if (is_vpid_supported())
+		vmcs_clear_bits(CPU_EXEC_CTRL1, CPU_VPID);
+
+	__vmx_pf_exception_test(invalidate_tlb_no_vpid, NULL);
+}
+
+static void invalidate_tlb_invvpid_addr(void *data)
+{
+	invvpid(INVVPID_ALL, *(u16 *)data, vmcs_read(EXI_QUALIFICATION));
+}
+
+static void invalidate_tlb_new_vpid(void *data)
+{
+	u16 *vpid = data;
+
+	/*
+	 * Bump VPID to effectively flush L2's TLB from L0's perspective.
+	 * Invalidate all VPIDs when the VPID wraps to zero as hardware/KVM is
+	 * architecturally allowed to keep TLB entries indefinitely.
+	 */
+	++(*vpid);
+	if (*vpid == 0) {
+		++(*vpid);
+		invvpid(INVVPID_ALL, 0, 0);
+	}
+	vmcs_write(VPID, *vpid);
+}
+
+static void __vmx_pf_vpid_test(invalidate_tlb_t inv_fn, u16 vpid)
+{
+	if (!is_vpid_supported())
+		test_skip("VPID unsupported");
+
+	if (!is_invvpid_supported())
+		test_skip("INVVPID unsupported");
+
+	vmcs_set_bits(CPU_EXEC_CTRL0, CPU_SECONDARY);
+	vmcs_set_bits(CPU_EXEC_CTRL1, CPU_VPID);
+	vmcs_write(VPID, vpid);
+
+	__vmx_pf_exception_test(inv_fn, &vpid);
+}
+
+static void vmx_pf_invvpid_test(void)
+{
+	if (!is_invvpid_type_supported(INVVPID_ADDR))
+		test_skip("INVVPID ADDR unsupported");
+
+	__vmx_pf_vpid_test(invalidate_tlb_invvpid_addr, 0xaaaa);
+}
+
+static void vmx_pf_vpid_test(void)
+{
+	/* Need INVVPID(ALL) to flush VPIDs upon wrap/reuse. */
+	if (!is_invvpid_type_supported(INVVPID_ALL))
+		test_skip("INVVPID ALL unsupported");
+
+	__vmx_pf_vpid_test(invalidate_tlb_new_vpid, 1);
+}
+
 #define TEST(name) { #name, .v2 = name }
 
 /* name/init/guest_main/exit_handler/syscall_handler/guest_regs */
@@ -10723,5 +10807,8 @@ struct vmx_test vmx_tests[] = {
 	TEST(vmx_mtf_test),
 	TEST(vmx_mtf_pdpte_test),
 	TEST(vmx_pf_exception_test),
+	TEST(vmx_pf_no_vpid_test),
+	TEST(vmx_pf_invvpid_test),
+	TEST(vmx_pf_vpid_test),
 	{ NULL, NULL, NULL, NULL, NULL, {0} },
 };
-- 
2.34.0.rc2.393.gf8c9666880-goog