[kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[Jim Mattson]

Add tests of "instructions retired" and "branch instructions retired,"
to ensure that these events count emulated instructions.

Signed-off-by: Eric Hankland <ehankland@google.com>
[jmattson:
  - Added command-line parameter to conditionally run the new tests.
  - Added pmu-emulation test to unittests.cfg
]
Signed-off-by: Jim Mattson <jmattson@google.com>
---
 x86/pmu.c         | 80 +++++++++++++++++++++++++++++++++++++++++++++++
 x86/unittests.cfg |  7 +++++
 2 files changed, 87 insertions(+)

diff --git a/x86/pmu.c b/x86/pmu.c
index ec61ac956a55..a159333b0c73 100644
--- a/x86/pmu.c
+++ b/x86/pmu.c
@@ -33,6 +33,12 @@
 
 #define N 1000000
 
+#define KVM_FEP "ud2; .byte 'k', 'v', 'm';"
+// These values match the number of instructions and branches in the
+// assembly block in check_emulated_instr().
+#define EXPECTED_INSTR 17
+#define EXPECTED_BRNCH 5
+
 typedef struct {
 	uint32_t ctr;
 	uint32_t config;
@@ -468,6 +474,77 @@ static void check_running_counter_wrmsr(void)
 	report_prefix_pop();
 }
 
+static void check_emulated_instr(void)
+{
+	uint64_t status, instr_start, brnch_start;
+	pmu_counter_t brnch_cnt = {
+		.ctr = MSR_IA32_PERFCTR0,
+		/* branch instructions */
+		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[5].unit_sel,
+		.count = 0,
+	};
+	pmu_counter_t instr_cnt = {
+		.ctr = MSR_IA32_PERFCTR0 + 1,
+		/* instructions */
+		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[1].unit_sel,
+		.count = 0,
+	};
+	report_prefix_push("emulated instruction");
+
+	wrmsr(MSR_CORE_PERF_GLOBAL_OVF_CTRL,
+	      rdmsr(MSR_CORE_PERF_GLOBAL_STATUS));
+
+	start_event(&brnch_cnt);
+	start_event(&instr_cnt);
+
+	brnch_start = -EXPECTED_BRNCH;
+	instr_start = -EXPECTED_INSTR;
+	wrmsr(MSR_IA32_PERFCTR0, brnch_start);
+	wrmsr(MSR_IA32_PERFCTR0 + 1, instr_start);
+	// KVM_FEP is a magic prefix that forces emulation so
+	// 'KVM_FEP "jne label\n"' just counts as a single instruction.
+	asm volatile(
+		"mov $0x0, %%eax\n"
+		"cmp $0x0, %%eax\n"
+		KVM_FEP "jne label\n"
+		KVM_FEP "jne label\n"
+		KVM_FEP "jne label\n"
+		KVM_FEP "jne label\n"
+		KVM_FEP "jne label\n"
+		"mov $0xa, %%eax\n"
+		"cpuid\n"
+		"mov $0xa, %%eax\n"
+		"cpuid\n"
+		"mov $0xa, %%eax\n"
+		"cpuid\n"
+		"mov $0xa, %%eax\n"
+		"cpuid\n"
+		"mov $0xa, %%eax\n"
+		"cpuid\n"
+		"label:\n"
+		:
+		:
+		: "eax", "ebx", "ecx", "edx");
+
+	wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
+
+	stop_event(&brnch_cnt);
+	stop_event(&instr_cnt);
+
+	// Check that the end count - start count is at least the expected
+	// number of instructions and branches.
+	report(instr_cnt.count - instr_start >= EXPECTED_INSTR,
+	       "instruction count");
+	report(brnch_cnt.count - brnch_start >= EXPECTED_BRNCH,
+	       "branch count");
+	// Additionally check that those counters overflowed properly.
+	status = rdmsr(MSR_CORE_PERF_GLOBAL_STATUS);
+	report(status & 1, "instruction counter overflow");
+	report(status & 2, "branch counter overflow");
+
+	report_prefix_pop();
+}
+
 static void check_counters(void)
 {
 	check_gp_counters();
@@ -563,6 +640,9 @@ int main(int ac, char **av)
 
 	check_counters();
 
+	if (ac > 1 && !strcmp(av[1], "emulation"))
+		check_emulated_instr();
+
 	if (rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) {
 		gp_counter_base = MSR_IA32_PMC0;
 		report_prefix_push("full-width writes");
diff --git a/x86/unittests.cfg b/x86/unittests.cfg
index 3000e53c790f..2aedb24dc4ff 100644
--- a/x86/unittests.cfg
+++ b/x86/unittests.cfg
@@ -185,6 +185,13 @@ extra_params = -cpu host,migratable=no
 check = /sys/module/kvm/parameters/ignore_msrs=N
 check = /proc/sys/kernel/nmi_watchdog=0
 
+[pmu_emulation]
+file = pmu.flat
+arch = x86_64
+extra_params = -cpu max -append emulation
+check = /sys/module/kvm_intel/parameters/force_emulation_prefix=Y
+check = /proc/sys/kernel/nmi_watchdog=0
+
 [vmware_backdoors]
 file = vmware_backdoors.flat
 extra_params = -machine vmport=on -cpu max
-- 
2.34.0.rc1.387.gb447b232ab-goog

Re: [kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[Like Xu]

On 13/11/2021 7:56 am, Jim Mattson wrote:
> Add tests of "instructions retired" and "branch instructions retired,"
> to ensure that these events count emulated instructions.
> 
> Signed-off-by: Eric Hankland <ehankland@google.com>
> [jmattson:
>    - Added command-line parameter to conditionally run the new tests.
>    - Added pmu-emulation test to unittests.cfg
> ]
> Signed-off-by: Jim Mattson <jmattson@google.com>
> ---
>   x86/pmu.c         | 80 +++++++++++++++++++++++++++++++++++++++++++++++
>   x86/unittests.cfg |  7 +++++
>   2 files changed, 87 insertions(+)
> 
> diff --git a/x86/pmu.c b/x86/pmu.c
> index ec61ac956a55..a159333b0c73 100644
> --- a/x86/pmu.c
> +++ b/x86/pmu.c
> @@ -33,6 +33,12 @@
>   
>   #define N 1000000
>   
> +#define KVM_FEP "ud2; .byte 'k', 'v', 'm';"
> +// These values match the number of instructions and branches in the
> +// assembly block in check_emulated_instr().
> +#define EXPECTED_INSTR 17
> +#define EXPECTED_BRNCH 5
> +
>   typedef struct {
>   	uint32_t ctr;
>   	uint32_t config;
> @@ -468,6 +474,77 @@ static void check_running_counter_wrmsr(void)
>   	report_prefix_pop();
>   }
>   
> +static void check_emulated_instr(void)
> +{
> +	uint64_t status, instr_start, brnch_start;
> +	pmu_counter_t brnch_cnt = {
> +		.ctr = MSR_IA32_PERFCTR0,
> +		/* branch instructions */
> +		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[5].unit_sel,
> +		.count = 0,
> +	};
> +	pmu_counter_t instr_cnt = {
> +		.ctr = MSR_IA32_PERFCTR0 + 1,
> +		/* instructions */
> +		.config = EVNTSEL_OS | EVNTSEL_USR | gp_events[1].unit_sel,
> +		.count = 0,
> +	};
> +	report_prefix_push("emulated instruction");
> +
> +	wrmsr(MSR_CORE_PERF_GLOBAL_OVF_CTRL,
> +	      rdmsr(MSR_CORE_PERF_GLOBAL_STATUS));
> +
> +	start_event(&brnch_cnt);
> +	start_event(&instr_cnt);
> +
> +	brnch_start = -EXPECTED_BRNCH;
> +	instr_start = -EXPECTED_INSTR;
> +	wrmsr(MSR_IA32_PERFCTR0, brnch_start);
> +	wrmsr(MSR_IA32_PERFCTR0 + 1, instr_start);
> +	// KVM_FEP is a magic prefix that forces emulation so
> +	// 'KVM_FEP "jne label\n"' just counts as a single instruction.
> +	asm volatile(
> +		"mov $0x0, %%eax\n"
> +		"cmp $0x0, %%eax\n"
> +		KVM_FEP "jne label\n"
> +		KVM_FEP "jne label\n"
> +		KVM_FEP "jne label\n"
> +		KVM_FEP "jne label\n"
> +		KVM_FEP "jne label\n"
> +		"mov $0xa, %%eax\n"
> +		"cpuid\n"
> +		"mov $0xa, %%eax\n"
> +		"cpuid\n"
> +		"mov $0xa, %%eax\n"
> +		"cpuid\n"
> +		"mov $0xa, %%eax\n"
> +		"cpuid\n"
> +		"mov $0xa, %%eax\n"
> +		"cpuid\n"
> +		"label:\n"
> +		:
> +		:
> +		: "eax", "ebx", "ecx", "edx");
> +
> +	wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, 0);
> +
> +	stop_event(&brnch_cnt);
> +	stop_event(&instr_cnt);
> +
> +	// Check that the end count - start count is at least the expected
> +	// number of instructions and branches.
> +	report(instr_cnt.count - instr_start >= EXPECTED_INSTR,
> +	       "instruction count");
> +	report(brnch_cnt.count - brnch_start >= EXPECTED_BRNCH,
> +	       "branch count");
> +	// Additionally check that those counters overflowed properly.
> +	status = rdmsr(MSR_CORE_PERF_GLOBAL_STATUS);
> +	report(status & 1, "instruction counter overflow");
> +	report(status & 2, "branch counter overflow");
> +
> +	report_prefix_pop();
> +}
> +
>   static void check_counters(void)
>   {
>   	check_gp_counters();
> @@ -563,6 +640,9 @@ int main(int ac, char **av)
>   
>   	check_counters();
>   
> +	if (ac > 1 && !strcmp(av[1], "emulation"))
> +		check_emulated_instr();
> +
>   	if (rdmsr(MSR_IA32_PERF_CAPABILITIES) & PMU_CAP_FW_WRITES) {
>   		gp_counter_base = MSR_IA32_PMC0;
>   		report_prefix_push("full-width writes");
> diff --git a/x86/unittests.cfg b/x86/unittests.cfg
> index 3000e53c790f..2aedb24dc4ff 100644
> --- a/x86/unittests.cfg
> +++ b/x86/unittests.cfg
> @@ -185,6 +185,13 @@ extra_params = -cpu host,migratable=no
>   check = /sys/module/kvm/parameters/ignore_msrs=N
>   check = /proc/sys/kernel/nmi_watchdog=0
>   
> +[pmu_emulation]
> +file = pmu.flat
> +arch = x86_64
> +extra_params = -cpu max -append emulation
> +check = /sys/module/kvm_intel/parameters/force_emulation_prefix=Y

It's "/sys/module/kvm/parameters/force_emulation_prefix=Y",

If it's N, we need a output like "FAIL: check_emulated_instr"
rather than:

Unhandled exception 6 #UD at ip 0000000000401387
error_code=0000      rflags=00010046      cs=00000008
rax=0000000000000000 rcx=00000000000000c2 rdx=00000000ffffffff rbx=00000000009509f4
rbp=0000000000513730 rsi=0000000000000020 rdi=0000000000000034
  r8=0000000000000000  r9=0000000000000020 r10=000000000000000d r11=0000000000000000
r12=000000000000038e r13=0000000000000002 r14=0000000000513d80 r15=0000000000008603
cr0=0000000080010011 cr2=0000000000000000 cr3=0000000001007000 cr4=0000000000000020
cr8=0000000000000000
	STACK: @401387 400384

> +check = /proc/sys/kernel/nmi_watchdog=0
> +
>   [vmware_backdoors]
>   file = vmware_backdoors.flat
>   extra_params = -machine vmport=on -cpu max
> 

Re: [kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[Paolo Bonzini]

On 11/16/21 11:04, Like Xu wrote:
>>
>> +check = /sys/module/kvm_intel/parameters/force_emulation_prefix=Y
> 
> It's "/sys/module/kvm/parameters/force_emulation_prefix=Y",
> 
> If it's N, we need a output like "FAIL: check_emulated_instr"
> rather than:
> 
> Unhandled exception 6 #UD at ip 0000000000401387
> error_code=0000      rflags=00010046      cs=00000008
> rax=0000000000000000 rcx=00000000000000c2 rdx=00000000ffffffff 
> rbx=00000000009509f4
> rbp=0000000000513730 rsi=0000000000000020 rdi=0000000000000034
>   r8=0000000000000000  r9=0000000000000020 r10=000000000000000d 
> r11=0000000000000000
> r12=000000000000038e r13=0000000000000002 r14=0000000000513d80 
> r15=0000000000008603
> cr0=0000000080010011 cr2=0000000000000000 cr3=0000000001007000 
> cr4=0000000000000020
> cr8=0000000000000000
>      STACK: @401387 400384
> 
>> +check = /proc/sys/kernel/nmi_watchdog=0 

Only one check is supported.  However, this test does not need all 
counters, so we can remove the NMI watchdog line.  I'll send a patch 
shortly.

Paolo

Re: [kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[Paolo Bonzini]

On 12/17/21 10:47, Ma Xinjian wrote:
> Hi, Jim
> 
> I am from Intel LKP team, we noticed that pmu_emulation was new added 
> recently by you.
> 
> We tested it and finished with 2 unexpected failures

The patch for this is not yet in kvm.git, since there were some 
discussions on the list.  It will be fixed before the merge window.

Paolo

Re: [kvm-unit-tests PATCH] x86/pmu: Test PMU virtualization on emulated instructions

[Ma Xinjian]

Hi, Jim

I am from Intel LKP team, we noticed that pmu_emulation was new added 
recently by you.

We tested it and finished with 2 unexpected failures

```

timeout -k 1s --foreground 90s /usr/bin/qemu-system-x86_64 --no-reboot 
-nodefaults -device pc-testdev -device 
isa-debug-exit,iobase=0xf4,iosize=0x4 -vnc none -serial stdio -device 
pci-testdev -machine accel=kvm -kernel x86/pmu.flat -smp 1 -cpu max 
-append emulation # -initrd /tmp/tmp.Y0jCDA5Jlw
enabling apic
paging enabled
cr0 = 80010011
cr3 = 1007000
cr4 = 20
PMU version:         2
GP counters:         4
GP counter width:    48
Mask length:         7
Fixed counters:      3
Fixed counter width: 48
PASS: emulated instruction: instruction count
PASS: emulated instruction: branch count
FAIL: emulated instruction: instruction counter overflow
FAIL: emulated instruction: branch counter overflow
SUMMARY: 4 tests, 2 unexpected failures

```

we have tried on kernel v5.15 v5.16-rc5

on 2 different machine

machine 1:

```

model: Haswell
cpu: 8
memory: 16G
brand: Intel(R) Core(TM) i7-4790T CPU @ 2.70GH

```

machine2:

```

model: Ice Lake
cpu: 96
memory: 256G
kernel_cmdline_hw: acpi_rsdp=0x667fd014
rootfs_partition: 
/dev/disk/by-id/ata-INTEL_SSDSC2BB800G4_PHWL4204005K800RGN-part3

```

If you confirm and fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>


Thanks

Ma Xinjian