[PATCH QEMU-KVM 0/7] test: more IPI tests

[PATCH QEMU-KVM 0/7] test: more IPI tests

[Avi Kivity]

The following patchset, besides fixing IPIs to actually work, adds two
IPI latency tests (back-to-back IPIs and IPI-then-spin) and improves
the framework a bit.

Avi Kivity (7):
  test: issue EOI after IPI
  test: set up per-cpu area
  test: optimize smp_id()
  test: add conditional execution for vmexit tests
  test: add ipi latency test
  test: Add ipi_halt benchmark
  test: Auto-tune vmexit test

 kvm/user/test/lib/x86/smp.c  |   18 +++++++++++++-
 kvm/user/test/x86/cstart64.S |   11 +++++++++
 kvm/user/test/x86/vmexit.c   |   51 +++++++++++++++++++++++++++++++++++++-----
 3 files changed, 72 insertions(+), 8 deletions(-)

[PATCH QEMU-KVM 1/7] test: issue EOI after IPI

[Avi Kivity]

Otherwise, we can't take more than one IPI.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/lib/x86/smp.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/kvm/user/test/lib/x86/smp.c b/kvm/user/test/lib/x86/smp.c
index a3a5472..25f0cae 100644
--- a/kvm/user/test/lib/x86/smp.c
+++ b/kvm/user/test/lib/x86/smp.c
@@ -14,6 +14,7 @@ static volatile int ipi_done;
 static __attribute__((used)) void ipi()
 {
     ipi_function(ipi_data);
+    apic_write(APIC_EOI, 0);
     ipi_done = 1;
 }
 
-- 
1.6.4.1

[PATCH QEMU-KVM 2/7] test: set up per-cpu area

[Avi Kivity]

Currently sharing space with the stack.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/x86/cstart64.S |   11 +++++++++++
 1 files changed, 11 insertions(+), 0 deletions(-)

diff --git a/kvm/user/test/x86/cstart64.S b/kvm/user/test/x86/cstart64.S
index e5554ba..f1a9d09 100644
--- a/kvm/user/test/x86/cstart64.S
+++ b/kvm/user/test/x86/cstart64.S
@@ -80,9 +80,19 @@ mb_flags = 0x0
 	# multiboot header
 	.long mb_magic, mb_flags, 0 - (mb_magic + mb_flags)
 
+MSR_GS_BASE = 0xc0000101
+
+.macro setup_percpu_area
+	lea -4096(%esp), %eax
+	mov $0, %edx
+	mov $MSR_GS_BASE, %ecx
+	wrmsr
+.endm
+
 .globl start
 start:
 	mov $stacktop, %esp
+	setup_percpu_area
 	call prepare_64
 	jmpl $8, $start64
 
@@ -142,6 +152,7 @@ ap_start32:
 	mov %ax, %ss
 	mov $-4096, %esp
 	lock/xaddl %esp, smp_stacktop
+	setup_percpu_area
 	call prepare_64
 	ljmpl $8, $ap_start64
 
-- 
1.6.4.1

[PATCH QEMU-KVM 3/7] test: optimize smp_id()

[Avi Kivity]

Rather than reading it from the APIC, read it from the per-cpu
area.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/lib/x86/smp.c |   17 +++++++++++++++--
 1 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/kvm/user/test/lib/x86/smp.c b/kvm/user/test/lib/x86/smp.c
index 25f0cae..9eface5 100644
--- a/kvm/user/test/lib/x86/smp.c
+++ b/kvm/user/test/lib/x86/smp.c
@@ -71,13 +71,21 @@ int cpu_count(void)
 
 int smp_id(void)
 {
-    return apic_read(APIC_ID);
+    unsigned id;
+
+    asm ("mov %%gs:0, %0" : "=r"(id));
+    return id;
+}
+
+static void setup_smp_id(void *data)
+{
+    asm ("mov %0, %%gs:0" : : "r"(apic_id()) : "memory");
 }
 
 void on_cpu(int cpu, void (*function)(void *data), void *data)
 {
     spin_lock(&ipi_lock);
-    if (cpu == apic_id())
+    if (cpu == smp_id())
 	function(data);
     else {
 	ipi_function = function;
@@ -98,4 +106,9 @@ void smp_init(void)
     void ipi_entry(void);
 
     set_ipi_descriptor(ipi_entry);
+
+    setup_smp_id(0);
+    for (i = 1; i < cpu_count(); ++i)
+        on_cpu(i, setup_smp_id, 0);
+
 }
-- 
1.6.4.1

[PATCH QEMU-KVM 4/7] test: add conditional execution for vmexit tests

[Avi Kivity]

ipi tests can only run on smp for example.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/x86/vmexit.c |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)

diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
index cce26d9..76f676d 100644
--- a/kvm/user/test/x86/vmexit.c
+++ b/kvm/user/test/x86/vmexit.c
@@ -54,6 +54,7 @@ static void mov_to_cr8(void)
 static struct test {
 	void (*func)(void);
 	const char *name;
+	int (*valid)(void);
 } tests[] = {
 	{ cpuid, "cpuid", },
 	{ vmcall, "vmcall", },
@@ -67,6 +68,11 @@ static void do_test(struct test *test)
 	unsigned long long t1, t2;
         void (*func)(void) = test->func;
 
+        if (test->valid && !test->valid()) {
+		printf("%s (skipped)\n", test->name);
+		return;
+	}
+
 	t1 = rdtsc();
 	for (i = 0; i < N; ++i)
             func();
-- 
1.6.4.1

[PATCH QEMU-KVM 5/7] test: add ipi latency test

[Avi Kivity]

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/x86/vmexit.c |   18 ++++++++++++++++++
 1 files changed, 18 insertions(+), 0 deletions(-)

diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
index 76f676d..29bb32a 100644
--- a/kvm/user/test/x86/vmexit.c
+++ b/kvm/user/test/x86/vmexit.c
@@ -1,5 +1,6 @@
 
 #include "libcflat.h"
+#include "smp.h"
 
 static inline unsigned long long rdtsc()
 {
@@ -51,6 +52,20 @@ static void mov_to_cr8(void)
 	asm volatile ("mov %0, %%cr8" : : "r"(cr8));
 }
 
+static int is_smp(void)
+{
+	return cpu_count() > 1;
+}
+
+static void nop(void *junk)
+{
+}
+
+static void ipi(void)
+{
+	on_cpu(1, nop, 0);
+}
+
 static struct test {
 	void (*func)(void);
 	const char *name;
@@ -60,6 +75,7 @@ static struct test {
 	{ vmcall, "vmcall", },
 	{ mov_from_cr8, "mov_from_cr8" },
 	{ mov_to_cr8, "mov_to_cr8" },
+	{ ipi, "ipi", is_smp },
 };
 
 static void do_test(struct test *test)
@@ -86,6 +102,8 @@ int main(void)
 {
 	int i;
 
+	smp_init();
+
 	for (i = 0; i < ARRAY_SIZE(tests); ++i)
 		do_test(&tests[i]);
 
-- 
1.6.4.1

[PATCH QEMU-KVM 6/7] test: Add ipi_halt benchmark

[Avi Kivity]

Wait for 2000 cycles after the IPI to allow the host to schedule out.
Measures wake-from-idle overhead.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/x86/vmexit.c |   11 +++++++++++
 1 files changed, 11 insertions(+), 0 deletions(-)

diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
index 29bb32a..5088dc9 100644
--- a/kvm/user/test/x86/vmexit.c
+++ b/kvm/user/test/x86/vmexit.c
@@ -66,6 +66,16 @@ static void ipi(void)
 	on_cpu(1, nop, 0);
 }
 
+static void ipi_halt(void)
+{
+	unsigned long long t;
+
+	on_cpu(1, nop, 0);
+	t = rdtsc() + 2000;
+	while (rdtsc() < t)
+		;
+}
+
 static struct test {
 	void (*func)(void);
 	const char *name;
@@ -76,6 +86,7 @@ static struct test {
 	{ mov_from_cr8, "mov_from_cr8" },
 	{ mov_to_cr8, "mov_to_cr8" },
 	{ ipi, "ipi", is_smp },
+	{ ipi_halt, "ipi+halt", is_smp },
 };
 
 static void do_test(struct test *test)
-- 
1.6.4.1

[PATCH QEMU-KVM 7/7] test: Auto-tune vmexit test

[Avi Kivity]

Scale up iterations until we measure at least 1G cycles.

Signed-off-by: Avi Kivity <avi@redhat.com>
---
 kvm/user/test/x86/vmexit.c |   16 ++++++++++------
 1 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
index 5088dc9..e7cb5ef 100644
--- a/kvm/user/test/x86/vmexit.c
+++ b/kvm/user/test/x86/vmexit.c
@@ -17,7 +17,7 @@ static inline unsigned long long rdtsc()
 	return r;
 }
 
-#define N (1 << 22)
+#define GOAL (1ull << 30)
 
 #ifdef __x86_64__
 #  define R "r"
@@ -93,6 +93,7 @@ static void do_test(struct test *test)
 {
 	int i;
 	unsigned long long t1, t2;
+	unsigned iterations = 32;
         void (*func)(void) = test->func;
 
         if (test->valid && !test->valid()) {
@@ -100,11 +101,14 @@ static void do_test(struct test *test)
 		return;
 	}
 
-	t1 = rdtsc();
-	for (i = 0; i < N; ++i)
-            func();
-	t2 = rdtsc();
-	printf("%s %d\n", test->name, (int)((t2 - t1) / N));
+	do {
+		iterations *= 2;
+		t1 = rdtsc();
+		for (i = 0; i < iterations; ++i)
+			func();
+		t2 = rdtsc();
+	} while ((t2 - t1) < GOAL);
+	printf("%s %d\n", test->name, (int)((t2 - t1) / iterations));
 }
 
 #define ARRAY_SIZE(_x) (sizeof(_x) / sizeof((_x)[0]))
-- 
1.6.4.1

Re: [PATCH QEMU-KVM 6/7] test: Add ipi_halt benchmark

[Marcelo Tosatti]

On Mon, Sep 21, 2009 at 11:55:37AM +0300, Avi Kivity wrote:
> Wait for 2000 cycles after the IPI to allow the host to schedule out.
> Measures wake-from-idle overhead.
> 
> Signed-off-by: Avi Kivity <avi@redhat.com>
> ---
>  kvm/user/test/x86/vmexit.c |   11 +++++++++++
>  1 files changed, 11 insertions(+), 0 deletions(-)
> 
> diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
> index 29bb32a..5088dc9 100644
> --- a/kvm/user/test/x86/vmexit.c
> +++ b/kvm/user/test/x86/vmexit.c
> @@ -66,6 +66,16 @@ static void ipi(void)
>  	on_cpu(1, nop, 0);
>  }
>  
> +static void ipi_halt(void)
> +{
> +	unsigned long long t;
> +
> +	on_cpu(1, nop, 0);
> +	t = rdtsc() + 2000;
> +	while (rdtsc() < t)
> +		;

I don't get the 2000 cycle thing. Since vcpu1 is on hlt by default, just
IPI'ing it to do nop() already includes wake-from-idle overhead?

Other than this the remaining is very nice, please apply.

> +}
> +
>  static struct test {
>  	void (*func)(void);
>  	const char *name;
> @@ -76,6 +86,7 @@ static struct test {
>  	{ mov_from_cr8, "mov_from_cr8" },
>  	{ mov_to_cr8, "mov_to_cr8" },
>  	{ ipi, "ipi", is_smp },
> +	{ ipi_halt, "ipi+halt", is_smp },
>  };
>  
>  static void do_test(struct test *test)
> -- 
> 1.6.4.1

Re: [PATCH QEMU-KVM 6/7] test: Add ipi_halt benchmark

[Avi Kivity]

On 09/22/2009 04:54 AM, Marcelo Tosatti wrote:
> On Mon, Sep 21, 2009 at 11:55:37AM +0300, Avi Kivity wrote:
>    
>> Wait for 2000 cycles after the IPI to allow the host to schedule out.
>> Measures wake-from-idle overhead.
>>
>> Signed-off-by: Avi Kivity<avi@redhat.com>
>> ---
>>   kvm/user/test/x86/vmexit.c |   11 +++++++++++
>>   1 files changed, 11 insertions(+), 0 deletions(-)
>>
>> diff --git a/kvm/user/test/x86/vmexit.c b/kvm/user/test/x86/vmexit.c
>> index 29bb32a..5088dc9 100644
>> --- a/kvm/user/test/x86/vmexit.c
>> +++ b/kvm/user/test/x86/vmexit.c
>> @@ -66,6 +66,16 @@ static void ipi(void)
>>   	on_cpu(1, nop, 0);
>>   }
>>
>> +static void ipi_halt(void)
>> +{
>> +	unsigned long long t;
>> +
>> +	on_cpu(1, nop, 0);
>> +	t = rdtsc() + 2000;
>> +	while (rdtsc()<  t)
>> +		;
>>      
> I don't get the 2000 cycle thing. Since vcpu1 is on hlt by default, just
> IPI'ing it to do nop() already includes wake-from-idle overhead?
>    

Without the delay, by the time vcpu1 gets to schedule() it will already 
have been woken up by vcpu0.  So it never switches to the idle thread.

The purpose of this test was to measure the just-in-time MSR switching, 
and that only helps if a context switch takes place.

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.