[RFC PATCH 0/2] kvm selftests and MAX_VCPU_ID

[RFC PATCH 0/2] kvm selftests and MAX_VCPU_ID

[Fabiano Rosas]

I've been experimenting with the kvm selftests to see if I can enable
them for powerpc and get some useful coverage going.

This series is just the initial boilerplate to get the simplest of the
tests to run. The test is arch agnostic and is already in the tree. It
just tries to start a vm with the maximum number of vcpus. It fails on
a P9:

$ cd tools/selftests/kvm
$ make ARCH=powerpc
$ ulimit -n
4096
$ ./kvm_create_max_vcpus
KVM_CAP_MAX_VCPU_ID: 16384
KVM_CAP_MAX_VCPUS: 2048
Testing creating 2048 vCPUs, with IDs 0...2047.
Testing creating 2048 vCPUs, with IDs 14336...16383.
==== Test Assertion Failure ====
  lib/kvm_util.c:983: vcpu->fd >= 0
  pid=74066 tid=74066 - Invalid argument
     1  0x0000000010002813: vm_vcpu_add at kvm_util.c:982
     2  0x000000001000176f: test_vcpu_creation at kvm_create_max_vcpus.c:34 (discriminator 3)
     3  0x00000000100014e7: main at kvm_create_max_vcpus.c:62
     4  0x00007fff89454077: ?? ??:0
     5  0x00007fff89454263: ?? ??:0
  KVM_CREATE_VCPU failed, rc: -1 errno: 22

The second patch is my attempt at improving the situation which,
admittedly, is mostly harmless, caused by the fact that
KVM_MAX_VCPU_IDS is set to a large value that is not always supported
by P9 so KVM HV aborts with -EINVAL.

I intended to get this first patch/test to work before implementing
the rest of the selftest infrastructure and adding more tests so I
thought it would be best to get some feedback before I delve in too
deep.

Fabiano Rosas (2):
  KVM: selftests: Add max vcpus test for ppc64le
  KVM: PPC: Book3S HV: Provide a more accurate MAX_VCPU_ID in P9

 arch/powerpc/kvm/powerpc.c                    |  5 ++-
 tools/testing/selftests/kvm/Makefile          |  7 +++
 .../testing/selftests/kvm/include/kvm_util.h  |  7 +++
 .../selftests/kvm/include/powerpc/processor.h |  7 +++
 tools/testing/selftests/kvm/lib/kvm_util.c    |  5 +++
 .../selftests/kvm/lib/powerpc/processor.c     | 44 +++++++++++++++++++
 6 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 tools/testing/selftests/kvm/include/powerpc/processor.h
 create mode 100644 tools/testing/selftests/kvm/lib/powerpc/processor.c

--
2.29.2

[RFC PATCH 1/2] KVM: selftests: Add max vcpus test for ppc64le

[Fabiano Rosas]

$ cd tools/selftests/kvm
$ make ARCH=powerpc
$ ulimit -n
4096
$ ./kvm_create_max_vcpus

Note the test currently fails in P9 with:

KVM_CAP_MAX_VCPU_ID: 16384
KVM_CAP_MAX_VCPUS: 2048
Testing creating 2048 vCPUs, with IDs 0...2047.
Testing creating 2048 vCPUs, with IDs 14336...16383.
==== Test Assertion Failure ====
  lib/kvm_util.c:983: vcpu->fd >= 0
  pid=74066 tid=74066 - Invalid argument
     1  0x0000000010002813: vm_vcpu_add at kvm_util.c:982
     2 0x000000001000176f: test_vcpu_creation at
     kvm_create_max_vcpus.c:34 (discriminator 3)
     3  0x00000000100014e7: main at kvm_create_max_vcpus.c:62
     4  0x00007fff89454077: ?? ??:0
     5  0x00007fff89454263: ?? ??:0
  KVM_CREATE_VCPU failed, rc: -1 errno: 22

Signed-off-by Fabiano Rosas <farosas@linux.ibm.com>
---
 tools/testing/selftests/kvm/Makefile          |  7 +++
 .../testing/selftests/kvm/include/kvm_util.h  |  7 +++
 .../selftests/kvm/include/powerpc/processor.h |  7 +++
 tools/testing/selftests/kvm/lib/kvm_util.c    |  5 +++
 .../selftests/kvm/lib/powerpc/processor.c     | 44 +++++++++++++++++++
 5 files changed, 70 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/include/powerpc/processor.h
 create mode 100644 tools/testing/selftests/kvm/lib/powerpc/processor.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile
index 67eebb53235f..f1778b3ed093 100644
--- a/tools/testing/selftests/kvm/Makefile
+++ b/tools/testing/selftests/kvm/Makefile
@@ -33,10 +33,15 @@ ifeq ($(ARCH),s390)
 	UNAME_M := s390x
 endif
 
+ifeq ($(ARCH),powerpc)
+	UNAME_M := powerpc
+endif
+
 LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c lib/test_util.c lib/guest_modes.c lib/perf_test_util.c
 LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/svm.c lib/x86_64/ucall.c lib/x86_64/handlers.S
 LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c
 LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c lib/s390x/diag318_test_handler.c
+LIBKVM_powerpc = lib/powerpc/processor.c
 
 TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test
 TEST_GEN_PROGS_x86_64 += x86_64/get_msr_index_features
@@ -93,6 +98,8 @@ TEST_GEN_PROGS_s390x += dirty_log_test
 TEST_GEN_PROGS_s390x += kvm_create_max_vcpus
 TEST_GEN_PROGS_s390x += set_memory_region_test
 
+TEST_GEN_PROGS_powerpc += kvm_create_max_vcpus
+
 TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M))
 LIBKVM += $(LIBKVM_$(UNAME_M))
 
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index 0f4258eaa629..d4f6e079592b 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -43,6 +43,7 @@ enum vm_guest_mode {
 	VM_MODE_P40V48_4K,
 	VM_MODE_P40V48_64K,
 	VM_MODE_PXXV48_4K,	/* For 48bits VA but ANY bits PA */
+	VM_MODE_P51V52_64K,	/* ???: include/asm/book3s/64/pgtable.h says P53 */
 	NUM_VM_MODES,
 };
 
@@ -64,6 +65,12 @@ enum vm_guest_mode {
 #define MIN_PAGE_SHIFT			12U
 #define ptes_per_page(page_size)	((page_size) / 16)
 
+#elif defined(__powerpc__)
+
+#define VM_MODE_DEFAULT			VM_MODE_P51V52_64K
+#define MIN_PAGE_SHIFT			16U
+#define ptes_per_page(page_size)	((page_size) / 8)
+
 #endif
 
 #define MIN_PAGE_SIZE		(1U << MIN_PAGE_SHIFT)
diff --git a/tools/testing/selftests/kvm/include/powerpc/processor.h b/tools/testing/selftests/kvm/include/powerpc/processor.h
new file mode 100644
index 000000000000..c75197b349a8
--- /dev/null
+++ b/tools/testing/selftests/kvm/include/powerpc/processor.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * powerpc processor specific defines
+ */
+#ifndef SELFTEST_KVM_PROCESSOR_H
+#define SELFTEST_KVM_PROCESSOR_H
+#endif
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index b8849a1aca79..2e9dafc03a12 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -151,6 +151,7 @@ const char * const vm_guest_mode_string[] = {
 	"PA-bits:40,  VA-bits:48,  4K pages",
 	"PA-bits:40,  VA-bits:48, 64K pages",
 	"PA-bits:ANY, VA-bits:48,  4K pages",
+	"PA-bits:51,  VA-bits:52, 64K pages",
 };
 _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES,
 	       "Missing new mode strings?");
@@ -163,6 +164,7 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = {
 	{ 40, 48,  0x1000, 12 },
 	{ 40, 48, 0x10000, 16 },
 	{  0,  0,  0x1000, 12 },
+	{ 51, 52, 0x10000, 16 },
 };
 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
 	       "Missing new mode params?");
@@ -246,6 +248,9 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
 		TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
 #endif
 		break;
+	case VM_MODE_P51V52_64K:
+		vm->pgtable_levels = 4;
+		break;
 	default:
 		TEST_FAIL("Unknown guest mode, mode: 0x%x", mode);
 	}
diff --git a/tools/testing/selftests/kvm/lib/powerpc/processor.c b/tools/testing/selftests/kvm/lib/powerpc/processor.c
new file mode 100644
index 000000000000..e86b8516863b
--- /dev/null
+++ b/tools/testing/selftests/kvm/lib/powerpc/processor.c
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * powerpc specific code
+ */
+#include "kvm_util.h"
+#include "../kvm_util_internal.h"
+#include "processor.h"
+
+
+void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa, uint32_t memslot)
+{
+	TEST_FAIL("%s not implemented", __func__);
+}
+
+vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
+{
+	TEST_FAIL("%s not implemented", __func__);
+	return 0;
+}
+
+void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot)
+{
+	TEST_FAIL("%s not implemented", __func__);
+}
+
+void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
+{
+	TEST_FAIL("%s not implemented", __func__);
+}
+
+void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
+{
+	TEST_FAIL("%s not implemented", __func__);
+}
+
+void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
+{
+	TEST_FAIL("%s not implemented", __func__);
+}
+
+void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)
+{
+	TEST_ASSERT(false, "Unhandled exception");
+}
-- 
2.29.2

[RFC PATCH 2/2] KVM: PPC: Book3S HV: Provide a more accurate MAX_VCPU_ID in P9

[Fabiano Rosas]

The KVM_CAP_MAX_VCPU_ID capability was added by commit 0b1b1dfd52a6
("kvm: introduce KVM_MAX_VCPU_ID") to allow for vcpu ids larger than
KVM_MAX_VCPU in powerpc.

For a P9 host we depend on the guest VSMT value to know what is the
maximum number of vcpu id we can support:

kvmppc_core_vcpu_create_hv:
    (...)
    if (cpu_has_feature(CPU_FTR_ARCH_300)) {
-->         if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
                    pr_devel("KVM: VCPU ID too high\n");
                    core = KVM_MAX_VCORES;
            } else {
                    BUG_ON(kvm->arch.smt_mode != 1);
                    core = kvmppc_pack_vcpu_id(kvm, id);
            }
    } else {
            core = id / kvm->arch.smt_mode;
    }

which means that the value being returned by the capability today for
a given guest is potentially way larger than what we actually support:

\#define KVM_MAX_VCPU_ID (MAX_SMT_THREADS * KVM_MAX_VCORES)

If the capability is queried before userspace enables the
KVM_CAP_PPC_SMT ioctl there is not much we can do, but if the emulated
smt mode is already known we could provide a more accurate value.

The only practical effect of this change today is to make the
kvm_create_max_vcpus test pass for powerpc. The QEMU spapr machine has
a lower max vcpu than what KVM allows so even KVM_MAX_VCPU is not
reached.

Signed-off-by: Fabiano Rosas <farosas@linux.ibm.com>

---
I see that for ppc, QEMU uses the capability after enabling
KVM_CAP_PPC_SMT, so we could change QEMU to issue the check extension
on the vm fd so that it would get the more accurate value.
---
 arch/powerpc/kvm/powerpc.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index a2a68a958fa0..95c9f47cc1b3 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -649,7 +649,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 		r = KVM_MAX_VCPUS;
 		break;
 	case KVM_CAP_MAX_VCPU_ID:
-		r = KVM_MAX_VCPU_ID;
+		if (hv_enabled && cpu_has_feature(CPU_FTR_ARCH_300))
+			r = KVM_MAX_VCPUS * ((kvm) ? kvm->arch.emul_smt_mode : 1);
+		else
+			r = KVM_MAX_VCPU_ID;
 		break;
 #ifdef CONFIG_PPC_BOOK3S_64
 	case KVM_CAP_PPC_GET_SMMU_INFO:
-- 
2.29.2

Re: [RFC PATCH 2/2] KVM: PPC: Book3S HV: Provide a more accurate MAX_VCPU_ID in P9

[Christophe Leroy]

Le 13/04/2021 à 00:26, Fabiano Rosas a écrit :
> The KVM_CAP_MAX_VCPU_ID capability was added by commit 0b1b1dfd52a6
> ("kvm: introduce KVM_MAX_VCPU_ID") to allow for vcpu ids larger than
> KVM_MAX_VCPU in powerpc.
> 
> For a P9 host we depend on the guest VSMT value to know what is the
> maximum number of vcpu id we can support:
> 
> kvmppc_core_vcpu_create_hv:
>      (...)
>      if (cpu_has_feature(CPU_FTR_ARCH_300)) {
> -->         if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
>                      pr_devel("KVM: VCPU ID too high\n");
>                      core = KVM_MAX_VCORES;
>              } else {
>                      BUG_ON(kvm->arch.smt_mode != 1);
>                      core = kvmppc_pack_vcpu_id(kvm, id);
>              }
>      } else {
>              core = id / kvm->arch.smt_mode;
>      }
> 
> which means that the value being returned by the capability today for
> a given guest is potentially way larger than what we actually support:
> 
> \#define KVM_MAX_VCPU_ID (MAX_SMT_THREADS * KVM_MAX_VCORES)
> 
> If the capability is queried before userspace enables the
> KVM_CAP_PPC_SMT ioctl there is not much we can do, but if the emulated
> smt mode is already known we could provide a more accurate value.
> 
> The only practical effect of this change today is to make the
> kvm_create_max_vcpus test pass for powerpc. The QEMU spapr machine has
> a lower max vcpu than what KVM allows so even KVM_MAX_VCPU is not
> reached.
> 
> Signed-off-by: Fabiano Rosas <farosas@linux.ibm.com>

This patch won't apply after commit a1c42ddedf35 ("kvm: rename 
KVM_MAX_VCPU_ID to KVM_MAX_VCPU_IDS")

Feel free to resubmit if still applicable.

Christsophe


> 
> ---
> I see that for ppc, QEMU uses the capability after enabling
> KVM_CAP_PPC_SMT, so we could change QEMU to issue the check extension
> on the vm fd so that it would get the more accurate value.
> ---
>   arch/powerpc/kvm/powerpc.c | 5 ++++-
>   1 file changed, 4 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
> index a2a68a958fa0..95c9f47cc1b3 100644
> --- a/arch/powerpc/kvm/powerpc.c
> +++ b/arch/powerpc/kvm/powerpc.c
> @@ -649,7 +649,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>   		r = KVM_MAX_VCPUS;
>   		break;
>   	case KVM_CAP_MAX_VCPU_ID:
> -		r = KVM_MAX_VCPU_ID;
> +		if (hv_enabled && cpu_has_feature(CPU_FTR_ARCH_300))
> +			r = KVM_MAX_VCPUS * ((kvm) ? kvm->arch.emul_smt_mode : 1);
> +		else
> +			r = KVM_MAX_VCPU_ID;
>   		break;
>   #ifdef CONFIG_PPC_BOOK3S_64
>   	case KVM_CAP_PPC_GET_SMMU_INFO:

Re: [RFC PATCH 2/2] KVM: PPC: Book3S HV: Provide a more accurate MAX_VCPU_ID in P9

[Fabiano Rosas]

Christophe Leroy <christophe.leroy@csgroup.eu> writes:

> Le 13/04/2021 à 00:26, Fabiano Rosas a écrit :
>> The KVM_CAP_MAX_VCPU_ID capability was added by commit 0b1b1dfd52a6
>> ("kvm: introduce KVM_MAX_VCPU_ID") to allow for vcpu ids larger than
>> KVM_MAX_VCPU in powerpc.
>> 
>> For a P9 host we depend on the guest VSMT value to know what is the
>> maximum number of vcpu id we can support:
>> 
>> kvmppc_core_vcpu_create_hv:
>>      (...)
>>      if (cpu_has_feature(CPU_FTR_ARCH_300)) {
>> -->         if (id >= (KVM_MAX_VCPUS * kvm->arch.emul_smt_mode)) {
>>                      pr_devel("KVM: VCPU ID too high\n");
>>                      core = KVM_MAX_VCORES;
>>              } else {
>>                      BUG_ON(kvm->arch.smt_mode != 1);
>>                      core = kvmppc_pack_vcpu_id(kvm, id);
>>              }
>>      } else {
>>              core = id / kvm->arch.smt_mode;
>>      }
>> 
>> which means that the value being returned by the capability today for
>> a given guest is potentially way larger than what we actually support:
>> 
>> \#define KVM_MAX_VCPU_ID (MAX_SMT_THREADS * KVM_MAX_VCORES)
>> 
>> If the capability is queried before userspace enables the
>> KVM_CAP_PPC_SMT ioctl there is not much we can do, but if the emulated
>> smt mode is already known we could provide a more accurate value.
>> 
>> The only practical effect of this change today is to make the
>> kvm_create_max_vcpus test pass for powerpc. The QEMU spapr machine has
>> a lower max vcpu than what KVM allows so even KVM_MAX_VCPU is not
>> reached.
>> 
>> Signed-off-by: Fabiano Rosas <farosas@linux.ibm.com>
>
> This patch won't apply after commit a1c42ddedf35 ("kvm: rename 
> KVM_MAX_VCPU_ID to KVM_MAX_VCPU_IDS")
>
> Feel free to resubmit if still applicable.

Thanks for the reminder, Christophe.

I was focusing on enabling the rest of the kvm-selftests:
https://lore.kernel.org/r/20220120170109.948681-1-farosas@linux.ibm.com

I'm preparing a v2 for that series and will try to include these patches
as well.