[RFC PATCH kvm-unit-tests 0/4] add generic stress test

[RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Paolo Bonzini]

This short series adds a generic stress test to KVM unit tests that runs a
series of

The test could grow a lot more features, including:

- wrapping the stress test with a VMX or SVM veneer which would forward
  or inject interrupts periodically

- test perf events

- do some work in the MSI handler, so that they have a chance
  of overlapping

- use PV EOI

- play with TPR and self IPIs, similar to Windows DPCs.

The configuration of the test is set individually for each VCPU on
the command line, for example:

   ./x86/run x86/chaos.flat -smp 2 \
      -append 'invtlb=1,mem=12,hz=100  hz=250,edu=1,edu_hz=53,hlt' -device edu

runs a continuous INVLPG+write test on 1<<12 pages on CPU 0, interrupted
by a 100 Hz timer tick; and keeps CPU 1 mostly idle except for 250 timer
ticks and 53 edu device interrupts per second.

For now, the test runs for an infinite time so it's not included in
unittests.cfg.  Do you think this is worth including in kvm-unit-tests,
and if so are you interested in non-x86 versions of it?  Or should the
code be as pluggable as possible to make it easier to port it?

Thanks,

Paolo

Paolo Bonzini (4):
  libcflat: add a few more runtime functions
  chaos: add generic stress test
  chaos: add timer interrupt to the workload
  chaos: add edu device interrupt to the workload

 lib/alloc.c         |   9 +-
 lib/alloc.h         |   1 +
 lib/libcflat.h      |   4 +-
 lib/string.c        |  59 +++++++++-
 lib/string.h        |   4 +
 lib/x86/processor.h |   2 +-
 x86/Makefile.x86_64 |   1 +
 x86/chaos.c         | 263 ++++++++++++++++++++++++++++++++++++++++++++
 8 files changed, 337 insertions(+), 6 deletions(-)
 create mode 100644 x86/chaos.c

-- 
2.29.2

[PATCH kvm-unit-tests 1/4] libcflat: add a few more runtime functions

[Paolo Bonzini]

These functions will be used to parse the chaos test's command line.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 lib/alloc.c    |  9 +++++++-
 lib/alloc.h    |  1 +
 lib/libcflat.h |  4 +++-
 lib/string.c   | 59 +++++++++++++++++++++++++++++++++++++++++++++++---
 lib/string.h   |  3 +++
 5 files changed, 71 insertions(+), 5 deletions(-)

diff --git a/lib/alloc.c b/lib/alloc.c
index a46f464..a56f664 100644
--- a/lib/alloc.c
+++ b/lib/alloc.c
@@ -1,7 +1,7 @@
 #include "alloc.h"
 #include "bitops.h"
 #include "asm/page.h"
-#include "bitops.h"
+#include "string.h"
 
 void *malloc(size_t size)
 {
@@ -50,6 +50,13 @@ void *calloc(size_t nmemb, size_t size)
 	return ptr;
 }
 
+char *strdup(const char *s)
+{
+	size_t len = strlen(s) + 1;
+	char *d = malloc(len);
+	return strcpy(d, s);
+}
+
 void free(void *ptr)
 {
 	if (alloc_ops->free)
diff --git a/lib/alloc.h b/lib/alloc.h
index 9b4b634..4139465 100644
--- a/lib/alloc.h
+++ b/lib/alloc.h
@@ -34,5 +34,6 @@ void *malloc(size_t size);
 void *calloc(size_t nmemb, size_t size);
 void free(void *ptr);
 void *memalign(size_t alignment, size_t size);
+char *strdup(const char *s);
 
 #endif /* _ALLOC_H_ */
diff --git a/lib/libcflat.h b/lib/libcflat.h
index 460a123..7d5d02e 100644
--- a/lib/libcflat.h
+++ b/lib/libcflat.h
@@ -80,9 +80,11 @@ extern int __getchar(void);
 extern int getchar(void);
 extern void exit(int code) __attribute__((noreturn));
 extern void abort(void) __attribute__((noreturn));
-extern long atol(const char *ptr);
 extern char *getenv(const char *name);
 
+extern long atol(const char *ptr);
+extern int parse_long(const char *s, long *num);
+
 extern int printf(const char *fmt, ...)
 					__attribute__((format(printf, 1, 2)));
 extern int snprintf(char *buf, int size, const char *fmt, ...)
diff --git a/lib/string.c b/lib/string.c
index 75257f5..1ebefcb 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -135,10 +135,9 @@ void *memchr(const void *s, int c, size_t n)
     return NULL;
 }
 
-long atol(const char *ptr)
+int parse_long(const char *s, long *num)
 {
     long acc = 0;
-    const char *s = ptr;
     int neg, c;
 
     while (*s == ' ' || *s == '\t')
@@ -163,7 +162,15 @@ long atol(const char *ptr)
     if (neg)
         acc = -acc;
 
-    return acc;
+    *num = acc;
+    return !*s;
+}
+
+long atol(const char *ptr)
+{
+	long num;
+	parse_long(ptr, &num);
+	return num;
 }
 
 extern char **environ;
@@ -224,3 +231,49 @@ bool simple_glob(const char *text, const char *pattern)
 
 	return !strcmp(text, copy);
 }
+
+/* Based on musl libc.  */
+#define BITOP(a,b,op) \
+ ((a)[(size_t)(b)/(8*sizeof *(a))] op (size_t)1<<((size_t)(b)%(8*sizeof *(a))))
+
+size_t strcspn(const char *s, const char *c)
+{
+	const char *a = s;
+	size_t byteset[32/sizeof(size_t)] = { 0 };
+
+	if (!c[0])
+		return 0;
+	if (!c[1]) {
+		while (*s != *c)
+			s++;
+	} else {
+		while (*c) {
+			BITOP(byteset, *(unsigned char *)c, |=);
+			c++;
+		}
+		while (*s && !BITOP(byteset, *(unsigned char *)s, &))
+			s++;
+	}
+	return s - a;
+}
+
+/*
+ * Slightly more flexible strsep.  The pointer to the token
+ * must be stashed by the caller, the delimiter is the return value.
+ */
+char strdelim(char **p, const char *sep)
+{
+	char *e;
+	char *s = *p;
+	char delim;
+
+	e = s + strcspn(s, sep);
+	delim = *e;
+	if (delim) {
+		*e = 0;
+		*p = ++e;
+	} else {
+		*p = e;
+	}
+	return delim;
+}
diff --git a/lib/string.h b/lib/string.h
index 493d51b..da31668 100644
--- a/lib/string.h
+++ b/lib/string.h
@@ -20,4 +20,7 @@ extern int memcmp(const void *s1, const void *s2, size_t n);
 extern void *memmove(void *dest, const void *src, size_t n);
 extern void *memchr(const void *s, int c, size_t n);
 
+size_t strcspn(const char *s, const char *c);
+char strdelim(char **p, const char *sep);
+
 #endif /* _STRING_H */
-- 
2.29.2

Re: [PATCH kvm-unit-tests 1/4] libcflat: add a few more runtime functions

[Thomas Huth]

On 23/12/2020 02.08, Paolo Bonzini wrote:
> These functions will be used to parse the chaos test's command line.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>   lib/alloc.c    |  9 +++++++-
>   lib/alloc.h    |  1 +
>   lib/libcflat.h |  4 +++-
>   lib/string.c   | 59 +++++++++++++++++++++++++++++++++++++++++++++++---
>   lib/string.h   |  3 +++
>   5 files changed, 71 insertions(+), 5 deletions(-)
> 
> diff --git a/lib/alloc.c b/lib/alloc.c
> index a46f464..a56f664 100644
> --- a/lib/alloc.c
> +++ b/lib/alloc.c
> @@ -1,7 +1,7 @@
>   #include "alloc.h"
>   #include "bitops.h"
>   #include "asm/page.h"
> -#include "bitops.h"
> +#include "string.h"
>   
>   void *malloc(size_t size)
>   {
> @@ -50,6 +50,13 @@ void *calloc(size_t nmemb, size_t size)
>   	return ptr;
>   }
>   
> +char *strdup(const char *s)
> +{
> +	size_t len = strlen(s) + 1;
> +	char *d = malloc(len);
> +	return strcpy(d, s);
> +}
> +
>   void free(void *ptr)
>   {
>   	if (alloc_ops->free)
> diff --git a/lib/alloc.h b/lib/alloc.h
> index 9b4b634..4139465 100644
> --- a/lib/alloc.h
> +++ b/lib/alloc.h
> @@ -34,5 +34,6 @@ void *malloc(size_t size);
>   void *calloc(size_t nmemb, size_t size);
>   void free(void *ptr);
>   void *memalign(size_t alignment, size_t size);
> +char *strdup(const char *s);

Why did you put the prototype in alloc.h and not in string.h?

  Thomas

[PATCH kvm-unit-tests 2/4] chaos: add generic stress test

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 lib/x86/processor.h |   2 +-
 x86/Makefile.x86_64 |   1 +
 x86/chaos.c         | 114 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 116 insertions(+), 1 deletion(-)
 create mode 100644 x86/chaos.c

diff --git a/lib/x86/processor.h b/lib/x86/processor.h
index 291d24b..a53654a 100644
--- a/lib/x86/processor.h
+++ b/lib/x86/processor.h
@@ -546,7 +546,7 @@ static inline void irq_enable(void)
     asm volatile("sti");
 }
 
-static inline void invlpg(volatile void *va)
+static inline void invlpg(const volatile void *va)
 {
 	asm volatile("invlpg (%0)" ::"r" (va) : "memory");
 }
diff --git a/x86/Makefile.x86_64 b/x86/Makefile.x86_64
index af61d85..761a1d9 100644
--- a/x86/Makefile.x86_64
+++ b/x86/Makefile.x86_64
@@ -20,6 +20,7 @@ tests += $(TEST_DIR)/tscdeadline_latency.flat
 tests += $(TEST_DIR)/intel-iommu.flat
 tests += $(TEST_DIR)/vmware_backdoors.flat
 tests += $(TEST_DIR)/rdpru.flat
+tests += $(TEST_DIR)/chaos.flat
 
 include $(SRCDIR)/$(TEST_DIR)/Makefile.common
 
diff --git a/x86/chaos.c b/x86/chaos.c
new file mode 100644
index 0000000..e723a3b
--- /dev/null
+++ b/x86/chaos.c
@@ -0,0 +1,114 @@
+#include "libcflat.h"
+#include "smp.h"
+#include "bitops.h"
+#include "string.h"
+#include "alloc.h"
+#include "alloc_page.h"
+#include "asm/page.h"
+#include "processor.h"
+
+#define MAX_NR_CPUS 256
+
+struct chaos_args {
+	long npages;		/* 0 for CPU workload. */
+	const char *mem;
+	int invtlb;
+};
+
+int ncpus;
+struct chaos_args all_args[MAX_NR_CPUS];
+
+static void parse_arg(struct chaos_args *args, const char *arg)
+{
+	char *s = strdup(arg);
+	char *p = s;
+
+	while (*p) {
+		char *word = p;
+		char delim = strdelim(&p, ",=");
+		long i = 0;
+		bool have_arg = false;
+		if (delim == '=') {
+			char *num = p;
+			strdelim(&p, ",");
+			if (!parse_long(num, &i))
+				printf("invalid argument for %s\n", word);
+			else
+				have_arg = true;
+		}
+
+		if (!strcmp(word, "mem")) {
+			if (!have_arg)
+				i = 12;
+			else if (i >= BITS_PER_LONG - 1 - PAGE_SHIFT) {
+				printf("mem argument too large, using 12\n");
+				i = 12;
+			}
+			args->npages = 1 << i;
+			args->mem = alloc_pages(i);
+			if (!args->mem)
+				printf("could not allocate memory\n");
+			printf("CPU %d: mem=%ld @ %p\n", smp_id(), i, args->mem);
+		} else if (!strcmp(word, "invtlb")) {
+			if (!have_arg)
+				i = 1;
+			else if (i != 0 && i != 1) {
+				printf("invtlb argument must be 0 or 1\n");
+				i = 1;
+			}
+			args->invtlb = i;
+			printf("CPU %d: invtlb=%ld\n", smp_id(), i);
+		} else {
+			printf("invalid argument %s\n", word);
+		}
+	}
+	free(s);
+}
+
+static void __attribute__((noreturn)) stress(void *data)
+{
+    const char *arg = data;
+    struct chaos_args *args = &all_args[smp_id()];
+
+    printf("starting CPU %d workload: %s\n", smp_id(), arg);
+    parse_arg(args, arg);
+
+    for (;;) {
+	    if (args->mem) {
+		    const char *s = args->mem;
+		    const char *e = s + (args->npages << PAGE_SHIFT);
+		    long i;
+		    for (i = args->npages; args->invtlb && i--; )
+			    invlpg(s + ((args->npages - i) << PAGE_SHIFT));
+		    while (s < e) {
+			    (*(unsigned long *)s)++;
+			    s += sizeof(unsigned long);
+		    }
+	    }
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    int i;
+
+    setup_vm();
+    if (argc <= 1) {
+        return 1;
+    }
+
+    argv++;
+    argc--;
+    ncpus = cpu_count();
+    if (ncpus > MAX_NR_CPUS)
+	    ncpus = MAX_NR_CPUS;
+
+    for (i = 1; i < ncpus; ++i) {
+        if (i >= argc) {
+            break;
+        }
+        on_cpu_async(i, stress, argv[i]);
+    }
+
+    stress(argv[0]);
+}
-- 
2.29.2

[PATCH kvm-unit-tests 3/4] chaos: add timer interrupt to the workload

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 x86/chaos.c  | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 64 insertions(+)

diff --git a/x86/chaos.c b/x86/chaos.c
index e723a3b..0b1e29c 100644
--- a/x86/chaos.c
+++ b/x86/chaos.c
@@ -1,5 +1,7 @@
 #include "libcflat.h"
 #include "smp.h"
+#include "isr.h"
+#include "apic.h"
 #include "bitops.h"
 #include "string.h"
 #include "alloc.h"
@@ -9,14 +11,24 @@
 
 #define MAX_NR_CPUS 256
 
+#define TIMER_IRQ 0x44
+
 struct chaos_args {
 	long npages;		/* 0 for CPU workload. */
 	const char *mem;
 	int invtlb;
+
+	int hz;
+	bool hlt;
+};
+
+struct counters {
+	int ticks_left;
 };
 
 int ncpus;
 struct chaos_args all_args[MAX_NR_CPUS];
+struct counters cnt[MAX_NR_CPUS];
 
 static void parse_arg(struct chaos_args *args, const char *arg)
 {
@@ -58,6 +70,20 @@ static void parse_arg(struct chaos_args *args, const char *arg)
 			}
 			args->invtlb = i;
 			printf("CPU %d: invtlb=%ld\n", smp_id(), i);
+		} else if (!strcmp(word, "hz")) {
+			if (!have_arg)
+				i = 1000;
+			args->hz = i;
+			printf("CPU %d: hz=%ld\n", smp_id(), i);
+		} else if (!strcmp(word, "hlt")) {
+			if (!have_arg)
+				i = 1;
+			else if (i != 0 && i != 1) {
+				printf("hlt argument must be 0 or 1\n");
+				i = 1;
+			}
+			args->hlt = i;
+			printf("CPU %d: hlt=%ld\n", smp_id(), i);
 		} else {
 			printf("invalid argument %s\n", word);
 		}
@@ -65,6 +91,31 @@ static void parse_arg(struct chaos_args *args, const char *arg)
 	free(s);
 }
 
+static void do_timer(void)
+{
+	int cpu = smp_id();
+	struct counters *c = &cnt[cpu];
+	char out[4];
+	if (c->ticks_left > 0) {
+		c->ticks_left--;
+		return;
+	}
+
+	c->ticks_left = all_args[cpu].hz;
+
+	/* Print current CPU number.  */
+	out[2] = (cpu % 10) + '0'; cpu /= 10;
+	out[1] = (cpu % 10) + '0'; cpu /= 10;
+	out[0] = (cpu % 10) + '0'; cpu /= 10;
+	puts(out + (ncpus < 100) + (ncpus < 10));
+}
+
+static void timer(isr_regs_t *regs)
+{
+	do_timer();
+        eoi();
+}
+
 static void __attribute__((noreturn)) stress(void *data)
 {
     const char *arg = data;
@@ -73,6 +124,15 @@ static void __attribute__((noreturn)) stress(void *data)
     printf("starting CPU %d workload: %s\n", smp_id(), arg);
     parse_arg(args, arg);
 
+    apic_write(APIC_TDCR, 0x0000000b);
+    if (args->hz) {
+	    /* FIXME: assumes that the LAPIC timer counts in nanoseconds.  */
+
+	    apic_write(APIC_TMICT, 1000000000 / args->hz);
+	    apic_write(APIC_LVTT, TIMER_IRQ | APIC_LVT_TIMER_PERIODIC);
+    }
+
+    irq_enable();
     for (;;) {
 	    if (args->mem) {
 		    const char *s = args->mem;
@@ -85,6 +145,8 @@ static void __attribute__((noreturn)) stress(void *data)
 			    s += sizeof(unsigned long);
 		    }
 	    }
+	    if (args->hlt)
+		    asm volatile("hlt");
     }
 }
 
@@ -103,6 +165,8 @@ int main(int argc, char *argv[])
     if (ncpus > MAX_NR_CPUS)
 	    ncpus = MAX_NR_CPUS;
 
+    handle_irq(TIMER_IRQ, timer);
+
     for (i = 1; i < ncpus; ++i) {
         if (i >= argc) {
             break;
-- 
2.29.2

[PATCH kvm-unit-tests 4/4] chaos: add edu device interrupt to the workload

[Paolo Bonzini]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 x86/chaos.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/x86/chaos.c b/x86/chaos.c
index 0b1e29c..8d421b5 100644
--- a/x86/chaos.c
+++ b/x86/chaos.c
@@ -8,9 +8,11 @@
 #include "alloc_page.h"
 #include "asm/page.h"
 #include "processor.h"
+#include "pci-edu.h"
 
 #define MAX_NR_CPUS 256
 
+#define EDU_MSI 0xc4
 #define TIMER_IRQ 0x44
 
 struct chaos_args {
@@ -20,21 +22,32 @@ struct chaos_args {
 
 	int hz;
 	bool hlt;
+
+	bool edu;
+	int edu_hz;
 };
 
 struct counters {
 	int ticks_left;
+	int edu_fraction;
+	int edu_msi;
 };
 
 int ncpus;
 struct chaos_args all_args[MAX_NR_CPUS];
 struct counters cnt[MAX_NR_CPUS];
 
+bool have_edu;
+struct pci_edu_dev edu_dev;
+
 static void parse_arg(struct chaos_args *args, const char *arg)
 {
 	char *s = strdup(arg);
 	char *p = s;
 
+	/* By default generate 17 MSIs per second (if enabled).  */
+	args->edu_hz = 17;
+
 	while (*p) {
 		char *word = p;
 		char delim = strdelim(&p, ",=");
@@ -84,10 +97,33 @@ static void parse_arg(struct chaos_args *args, const char *arg)
 			}
 			args->hlt = i;
 			printf("CPU %d: hlt=%ld\n", smp_id(), i);
+		} else if (!strcmp(word, "edu")) {
+			if (!have_arg)
+				i = 1;
+			else if (i != 0 && i != 1) {
+				printf("edu argument must be 0 or 1\n");
+				i = 1;
+			}
+			if (i != 0 && !have_edu) {
+				printf("edu device not found\n");
+				i = 0;
+			}
+			args->edu = i;
+			printf("CPU %d: edu=%ld\n", smp_id(), i);
+		} else if (!strcmp(word, "edu_hz")) {
+			if (!have_arg || !i)
+				i = 100;
+			args->edu_hz = i;
+			printf("CPU %d: edu_hz=%ld\n", smp_id(), i);
 		} else {
 			printf("invalid argument %s\n", word);
 		}
 	}
+	if (args->edu && args->edu_hz > args->hz) {
+		printf("MSI rate limited to the CPU's hz value\n");
+		args->edu_hz = args->hz;
+	}
+
 	free(s);
 }
 
@@ -97,12 +133,27 @@ static void do_timer(void)
 	struct counters *c = &cnt[cpu];
 	char out[4];
 	if (c->ticks_left > 0) {
+		/*
+		 * Bresenham algorithm, generate edu_hz MSIs interrupts
+		 * every hz timer ticks.  See the other half in the
+		 * stress function.
+		 */
+		if (all_args[cpu].edu)
+			c->edu_fraction += all_args[cpu].edu_hz;
+
 		c->ticks_left--;
 		return;
 	}
 
 	c->ticks_left = all_args[cpu].hz;
 
+	if (all_args[cpu].edu) {
+		if (!c->edu_msi) {
+			puts("!!! no MSI received for edu device\n");
+		}
+		c->edu_msi = 0;
+	}
+
 	/* Print current CPU number.  */
 	out[2] = (cpu % 10) + '0'; cpu /= 10;
 	out[1] = (cpu % 10) + '0'; cpu /= 10;
@@ -116,14 +167,33 @@ static void timer(isr_regs_t *regs)
         eoi();
 }
 
+static void edu(isr_regs_t *regs)
+{
+	int cpu = smp_id();
+	struct counters *c = &cnt[cpu];
+	c->edu_msi++;
+        eoi();
+}
+
+static void x86_setup_msi(struct pci_dev *pci_dev, int dest)
+{
+	u64 address = 0xFEE00000 + (dest << 12);
+	u32 data = EDU_MSI;
+	pci_setup_msi(pci_dev, address, data);
+}
+
 static void __attribute__((noreturn)) stress(void *data)
 {
     const char *arg = data;
     struct chaos_args *args = &all_args[smp_id()];
+    struct counters *c = &cnt[smp_id()];
 
     printf("starting CPU %d workload: %s\n", smp_id(), arg);
     parse_arg(args, arg);
 
+    /* Do not print errors the first time through.  */
+    c->edu_msi = 1;
+
     apic_write(APIC_TDCR, 0x0000000b);
     if (args->hz) {
 	    /* FIXME: assumes that the LAPIC timer counts in nanoseconds.  */
@@ -132,6 +202,11 @@ static void __attribute__((noreturn)) stress(void *data)
 	    apic_write(APIC_LVTT, TIMER_IRQ | APIC_LVT_TIMER_PERIODIC);
     }
 
+    if (args->edu) {
+	    printf("starting edu device\n");
+	    x86_setup_msi(&edu_dev.pci_dev, apic_id());
+    }
+
     irq_enable();
     for (;;) {
 	    if (args->mem) {
@@ -147,6 +222,13 @@ static void __attribute__((noreturn)) stress(void *data)
 	    }
 	    if (args->hlt)
 		    asm volatile("hlt");
+
+	    if (c->edu_fraction > args->hz) {
+		    c->edu_fraction -= args->hz;
+		    edu_reg_writel(&edu_dev, EDU_REG_INTR_RAISE, 1);
+		    while (!c->edu_msi)
+			    cpu_relax();
+	    }
     }
 }
 
@@ -159,12 +241,15 @@ int main(int argc, char *argv[])
         return 1;
     }
 
+    have_edu = edu_init(&edu_dev);
+
     argv++;
     argc--;
     ncpus = cpu_count();
     if (ncpus > MAX_NR_CPUS)
 	    ncpus = MAX_NR_CPUS;
 
+    handle_irq(EDU_MSI, edu);
     handle_irq(TIMER_IRQ, timer);
 
     for (i = 1; i < ncpus; ++i) {
-- 
2.29.2

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Sean Christopherson]

On Wed, Dec 23, 2020, Paolo Bonzini wrote:
> This short series adds a generic stress test to KVM unit tests that runs a
> series of

Unintentional cliffhanger?

> The test could grow a lot more features, including:
> 
> - wrapping the stress test with a VMX or SVM veneer which would forward
>   or inject interrupts periodically
> 
> - test perf events
> 
> - do some work in the MSI handler, so that they have a chance
>   of overlapping
> 
> - use PV EOI
> 
> - play with TPR and self IPIs, similar to Windows DPCs.
> 
> The configuration of the test is set individually for each VCPU on
> the command line, for example:
> 
>    ./x86/run x86/chaos.flat -smp 2 \
>       -append 'invtlb=1,mem=12,hz=100  hz=250,edu=1,edu_hz=53,hlt' -device edu
> 
> runs a continuous INVLPG+write test on 1<<12 pages on CPU 0, interrupted
> by a 100 Hz timer tick; and keeps CPU 1 mostly idle except for 250 timer
> ticks and 53 edu device interrupts per second.

Maybe take the target cpu as part of the command line instead of implicitly
defining it via group position?  The "duplicate" hz=??? is confusing.  E.g.

    ./x86/run x86/chaos.flat -smp 2 \
      -append 'cpu=0,invtlb=1,mem=12,hz=100 cpu=1,hz=250,edu=1,edu_hz=53,hlt' -device edu

> For now, the test runs for an infinite time so it's not included in
> unittests.cfg.  Do you think this is worth including in kvm-unit-tests,

What's the motivation for this type of test?  What class of bugs can it find
that won't be found by existing kvm-unit-tests or simple boot tests?

> and if so are you interested in non-x86 versions of it?  Or should the
> code be as pluggable as possible to make it easier to port it?

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Paolo Bonzini]

On 28/12/20 23:25, Sean Christopherson wrote:
> On Wed, Dec 23, 2020, Paolo Bonzini wrote:
>> This short series adds a generic stress test to KVM unit tests that runs a
>> series of
> 
> Unintentional cliffhanger?

... event injections, timer cycles, memory updates and TLB invalidations.

>> The configuration of the test is set individually for each VCPU on
>> the command line, for example:
>>
>>     ./x86/run x86/chaos.flat -smp 2 \
>>        -append 'invtlb=1,mem=12,hz=100  hz=250,edu=1,edu_hz=53,hlt' -device edu
>>
>> runs a continuous INVLPG+write test on 1<<12 pages on CPU 0, interrupted
>> by a 100 Hz timer tick; and keeps CPU 1 mostly idle except for 250 timer
>> ticks and 53 edu device interrupts per second.
> 
> Maybe take the target cpu as part of the command line instead of implicitly
> defining it via group position?

Sure, the command line syntax can be adjusted.

   The "duplicate" hz=??? is confusing.  E.g.
> 
>      ./x86/run x86/chaos.flat -smp 2 \
>        -append 'cpu=0,invtlb=1,mem=12,hz=100 cpu=1,hz=250,edu=1,edu_hz=53,hlt' -device edu
> 
>> For now, the test runs for an infinite time so it's not included in
>> unittests.cfg.  Do you think this is worth including in kvm-unit-tests,
> 
> What's the motivation for this type of test?  What class of bugs can it find
> that won't be found by existing kvm-unit-tests or simple boot tests?

Mostly live migration tests.  For example, Maxim found a corner case in 
KVM_GET_VCPU_EVENTS that affects both nVMX and nSVM live migration 
(patches coming), and it is quite hard to turn it into a selftest 
because it requires the ioctl to be invoked exactly when 
nested_run_pending==1.  Such a test would allow stress-testing live 
migration without having to set up L1 and L2 virtual machine images.

Paolo

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Sean Christopherson]

On Sat, Jan 02, 2021, Paolo Bonzini wrote:
> On 28/12/20 23:25, Sean Christopherson wrote:
> > On Wed, Dec 23, 2020, Paolo Bonzini wrote:
> > > This short series adds a generic stress test to KVM unit tests that runs a
> > > series of
> > 
> > Unintentional cliffhanger?
> 
> ... event injections, timer cycles, memory updates and TLB invalidations.
> 
> > > The configuration of the test is set individually for each VCPU on
> > > the command line, for example:
> > > 
> > >     ./x86/run x86/chaos.flat -smp 2 \
> > >        -append 'invtlb=1,mem=12,hz=100  hz=250,edu=1,edu_hz=53,hlt' -device edu
> > > 
> > > runs a continuous INVLPG+write test on 1<<12 pages on CPU 0, interrupted
> > > by a 100 Hz timer tick; and keeps CPU 1 mostly idle except for 250 timer
> > > ticks and 53 edu device interrupts per second.
> > 
> > Maybe take the target cpu as part of the command line instead of implicitly
> > defining it via group position?
> 
> Sure, the command line syntax can be adjusted.
> 
>   The "duplicate" hz=??? is confusing.  E.g.
> > 
> >      ./x86/run x86/chaos.flat -smp 2 \
> >        -append 'cpu=0,invtlb=1,mem=12,hz=100 cpu=1,hz=250,edu=1,edu_hz=53,hlt' -device edu
> > 
> > > For now, the test runs for an infinite time so it's not included in
> > > unittests.cfg.  Do you think this is worth including in kvm-unit-tests,
> > 
> > What's the motivation for this type of test?  What class of bugs can it find
> > that won't be found by existing kvm-unit-tests or simple boot tests?
> 
> Mostly live migration tests.  For example, Maxim found a corner case in
> KVM_GET_VCPU_EVENTS that affects both nVMX and nSVM live migration (patches
> coming), and it is quite hard to turn it into a selftest because it requires
> the ioctl to be invoked exactly when nested_run_pending==1.  Such a test
> would allow stress-testing live migration without having to set up L1 and L2
> virtual machine images.

Ah, so you run the stress test in L1 and then migrate L1?

What's the biggest hurdle for doing this completely within the unit test
framework?  Is teaching the framework to migrate a unit test the biggest pain?
Writing a "unit test" that puts an L2 guest into a busy loop doesn't seem _that_
bad.

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Paolo Bonzini]

On 12/01/21 23:28, Sean Christopherson wrote:
>>> What's the motivation for this type of test?  What class of bugs can it find
>>> that won't be found by existing kvm-unit-tests or simple boot tests?
>>
>> Mostly live migration tests.  For example, Maxim found a corner case in
>> KVM_GET_VCPU_EVENTS that affects both nVMX and nSVM live migration (patches
>> coming), and it is quite hard to turn it into a selftest because it requires
>> the ioctl to be invoked exactly when nested_run_pending==1.  Such a test
>> would allow stress-testing live migration without having to set up L1 and L2
>> virtual machine images.
> 
> Ah, so you run the stress test in L1 and then migrate L1?

Yes.  I can't exclude that it would find bugs without migration, but I 
hope we'd have stomped them by now.

> What's the biggest hurdle for doing this completely within the unit test
> framework?  Is teaching the framework to migrate a unit test the biggest pain?

Yes, pretty much.  The shell script framework would show its limits.

That said, I've always treated run_tests.sh as a utility more than an 
integral part of kvm-unit-tests.  There's nothing that prevents a more 
capable framework from parsing unittests.cfg.

Paolo

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Sean Christopherson]

On Wed, Jan 13, 2021, Paolo Bonzini wrote:
> On 12/01/21 23:28, Sean Christopherson wrote:
> > What's the biggest hurdle for doing this completely within the unit test
> > framework?  Is teaching the framework to migrate a unit test the biggest pain?
> 
> Yes, pretty much.  The shell script framework would show its limits.
> 
> That said, I've always treated run_tests.sh as a utility more than an
> integral part of kvm-unit-tests.  There's nothing that prevents a more
> capable framework from parsing unittests.cfg.

Heh, got anyone you can "volunteer" to create a new framework?  One-button
migration testing would be very nice to have.  I suspect I'm not the only
contributor that doesn't do migration testing as part of their standard workflow.

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Paolo Bonzini]

On 14/01/21 21:13, Sean Christopherson wrote:
> On Wed, Jan 13, 2021, Paolo Bonzini wrote:
>> On 12/01/21 23:28, Sean Christopherson wrote:
>>> What's the biggest hurdle for doing this completely within the unit test
>>> framework?  Is teaching the framework to migrate a unit test the biggest pain?
>>
>> Yes, pretty much.  The shell script framework would show its limits.
>>
>> That said, I've always treated run_tests.sh as a utility more than an
>> integral part of kvm-unit-tests.  There's nothing that prevents a more
>> capable framework from parsing unittests.cfg.
> 
> Heh, got anyone you can "volunteer" to create a new framework?  One-button
> migration testing would be very nice to have.  I suspect I'm not the only
> contributor that doesn't do migration testing as part of their standard workflow.

avocado-vt is the one I use for installation tests.  It can do a lot 
more, including migration, but it is a bit hard to set up.

avocado-qemu (python/qemu and tests/acceptance in the QEMU tree) is a 
lot simpler, but it does not have a lot of tests and in particular it is 
not integrated with kvm-unit-tests.

Maxim also wrote a script to automate his tests which has quite a few 
features, but I've never used it myself.

Paolo

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Sean Christopherson]

On Thu, Jan 14, 2021, Paolo Bonzini wrote:
> On 14/01/21 21:13, Sean Christopherson wrote:
> > On Wed, Jan 13, 2021, Paolo Bonzini wrote:
> > > On 12/01/21 23:28, Sean Christopherson wrote:
> > > > What's the biggest hurdle for doing this completely within the unit test
> > > > framework?  Is teaching the framework to migrate a unit test the biggest pain?
> > > 
> > > Yes, pretty much.  The shell script framework would show its limits.
> > > 
> > > That said, I've always treated run_tests.sh as a utility more than an
> > > integral part of kvm-unit-tests.  There's nothing that prevents a more
> > > capable framework from parsing unittests.cfg.
> > 
> > Heh, got anyone you can "volunteer" to create a new framework?  One-button
> > migration testing would be very nice to have.  I suspect I'm not the only
> > contributor that doesn't do migration testing as part of their standard workflow.
> 
> avocado-vt is the one I use for installation tests.  It can do a lot more,
> including migration, but it is a bit hard to set up.

Is avocado-vt the test stuff you were talking about at the KVM Forum BoF?

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Paolo Bonzini]

On 14/01/21 23:13, Sean Christopherson wrote:
> On Thu, Jan 14, 2021, Paolo Bonzini wrote:
>> On 14/01/21 21:13, Sean Christopherson wrote:
>>> On Wed, Jan 13, 2021, Paolo Bonzini wrote:
>>>> On 12/01/21 23:28, Sean Christopherson wrote:
>>>>> What's the biggest hurdle for doing this completely within the unit test
>>>>> framework?  Is teaching the framework to migrate a unit test the biggest pain?
>>>>
>>>> Yes, pretty much.  The shell script framework would show its limits.
>>>>
>>>> That said, I've always treated run_tests.sh as a utility more than an
>>>> integral part of kvm-unit-tests.  There's nothing that prevents a more
>>>> capable framework from parsing unittests.cfg.
>>>
>>> Heh, got anyone you can "volunteer" to create a new framework?  One-button
>>> migration testing would be very nice to have.  I suspect I'm not the only
>>> contributor that doesn't do migration testing as part of their standard workflow.
>>
>> avocado-vt is the one I use for installation tests.  It can do a lot more,
>> including migration, but it is a bit hard to set up.
> 
> Is avocado-vt the test stuff you were talking about at the KVM Forum BoF?

Yes, it is.

Paolo

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Andrew Jones]

On Thu, Jan 14, 2021 at 12:13:22PM -0800, Sean Christopherson wrote:
> On Wed, Jan 13, 2021, Paolo Bonzini wrote:
> > On 12/01/21 23:28, Sean Christopherson wrote:
> > > What's the biggest hurdle for doing this completely within the unit test
> > > framework?  Is teaching the framework to migrate a unit test the biggest pain?
> > 
> > Yes, pretty much.  The shell script framework would show its limits.
> > 
> > That said, I've always treated run_tests.sh as a utility more than an
> > integral part of kvm-unit-tests.  There's nothing that prevents a more
> > capable framework from parsing unittests.cfg.
> 
> Heh, got anyone you can "volunteer" to create a new framework?  One-button
> migration testing would be very nice to have.  I suspect I'm not the only
> contributor that doesn't do migration testing as part of their standard workflow.
>

We have one-button migration tests already with kvm-unit-tests. Just
compile the tests that use the migration framework as standalone
tests and then run them directly.

I agree, though, that Bash is a pain for some of the stuff we're trying
to do. However, we do have requests to keep the framework written in Bash,
because KVM testing is regularly done with simulators and even in embedded
environments. It's not desirable, or even possible, to have e.g. Python
everywhere we want kvm-unit-tests.

Thanks,
drew

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Sean Christopherson]

On Mon, Jan 18, 2021, Andrew Jones wrote:
> On Thu, Jan 14, 2021 at 12:13:22PM -0800, Sean Christopherson wrote:
> > On Wed, Jan 13, 2021, Paolo Bonzini wrote:
> > > On 12/01/21 23:28, Sean Christopherson wrote:
> > > > What's the biggest hurdle for doing this completely within the unit test
> > > > framework?  Is teaching the framework to migrate a unit test the biggest pain?
> > > 
> > > Yes, pretty much.  The shell script framework would show its limits.
> > > 
> > > That said, I've always treated run_tests.sh as a utility more than an
> > > integral part of kvm-unit-tests.  There's nothing that prevents a more
> > > capable framework from parsing unittests.cfg.
> > 
> > Heh, got anyone you can "volunteer" to create a new framework?  One-button
> > migration testing would be very nice to have.  I suspect I'm not the only
> > contributor that doesn't do migration testing as part of their standard workflow.
> >
> 
> We have one-button migration tests already with kvm-unit-tests. Just
> compile the tests that use the migration framework as standalone
> tests and then run them directly.

Do those exist/work for x86?  I see migration stuff for Arm and PPC, but nothing
for x86.

> I agree, though, that Bash is a pain for some of the stuff we're trying
> to do. However, we do have requests to keep the framework written in Bash,
> because KVM testing is regularly done with simulators and even in embedded
> environments. It's not desirable, or even possible, to have e.g. Python
> everywhere we want kvm-unit-tests.

True, I would probably be one of the people complaining if the tests started
requiring some newfangled language :-)

Re: [RFC PATCH kvm-unit-tests 0/4] add generic stress test

[Andrew Jones]

On Tue, Jan 19, 2021 at 09:37:19AM -0800, Sean Christopherson wrote:
> On Mon, Jan 18, 2021, Andrew Jones wrote:
> > On Thu, Jan 14, 2021 at 12:13:22PM -0800, Sean Christopherson wrote:
> > > On Wed, Jan 13, 2021, Paolo Bonzini wrote:
> > > > On 12/01/21 23:28, Sean Christopherson wrote:
> > > > > What's the biggest hurdle for doing this completely within the unit test
> > > > > framework?  Is teaching the framework to migrate a unit test the biggest pain?
> > > > 
> > > > Yes, pretty much.  The shell script framework would show its limits.
> > > > 
> > > > That said, I've always treated run_tests.sh as a utility more than an
> > > > integral part of kvm-unit-tests.  There's nothing that prevents a more
> > > > capable framework from parsing unittests.cfg.
> > > 
> > > Heh, got anyone you can "volunteer" to create a new framework?  One-button
> > > migration testing would be very nice to have.  I suspect I'm not the only
> > > contributor that doesn't do migration testing as part of their standard workflow.
> > >
> > 
> > We have one-button migration tests already with kvm-unit-tests. Just
> > compile the tests that use the migration framework as standalone
> > tests and then run them directly.
> 
> Do those exist/work for x86?  I see migration stuff for Arm and PPC, but nothing
> for x86.

Right, we don't have migration tests yet for x86. Of course that's just a
matter of programming... We'll also need to add an x86 __getchar() first.

Thanks,
drew