[for-5.0 PATCH v2 0/4] spapr: Use less XIVE HW resources in KVM

[for-5.0 PATCH v2 0/4] spapr: Use less XIVE HW resources in KVM

[Greg Kurz]

On POWER9 systems, the XICS-on-XIVE and XIVE KVM devices currently
allocate a bunch of VPs in the XIVE HW to accomodate the highest
VCPU id that may be possibly used in a VM. This limits the number
of VMs that can run with an in-kernel interrupt controller to 63
per POWER9 chip, irrespectively of its number of HW threads, eg.
up to 96 on a POWER9 Nimbus socket. This is an unfortunate waste
of scarce HW resources since a typical VM doesn't need that much
VPs to run.

This series exploits new attributes of the XICS-on-XIVE and XIVE
KVM devices that allow userspace to tune the numbers of VPs it
really needs. Support for this attributes has just reached linux
mainline.

In combination with recent commit 29cb4187497d "spapr: Set VSMT to
smp_threads by default" which makes VCPU ids contiguous, it is
possible to start more than 200 VMs instead of 127 on an AC922
system (two sockets, 144 HW threads).

Changes in v2:
- add an nr_servers argument to the activate() handler instead of
  a QOM property for XIVE and a XICSFabric method for XICS-on-XIVE.
- even if the last two patches haven't changed much, the code base
  itself has, so I've deliberately dropped Cedric's R-b tags

--
Greg

---

Greg Kurz (4):
      linux-headers: Update
      spapr: Pass the maximum number of vCPUs to the KVM interrupt controller
      spapr/xics: Configure number of servers in KVM
      spapr/xive: Configure number of servers in KVM


 hw/intc/spapr_xive.c                         |    6 ++++--
 hw/intc/spapr_xive_kvm.c                     |   26 +++++++++++++++++++++++---
 hw/intc/xics_kvm.c                           |   24 +++++++++++++++++++++---
 hw/intc/xics_spapr.c                         |    5 +++--
 hw/ppc/spapr_irq.c                           |    8 +++++---
 include/hw/ppc/spapr_irq.h                   |   10 ++++++++--
 include/hw/ppc/spapr_xive.h                  |    3 ++-
 include/hw/ppc/xics_spapr.h                  |    3 ++-
 include/standard-headers/linux/ethtool.h     |    6 ++++++
 include/standard-headers/linux/virtio_ring.h |    2 +-
 linux-headers/asm-arm/kvm.h                  |    3 ++-
 linux-headers/asm-arm64/kvm.h                |    5 ++++-
 linux-headers/asm-mips/unistd_n32.h          |    1 +
 linux-headers/asm-mips/unistd_n64.h          |    1 +
 linux-headers/asm-mips/unistd_o32.h          |    1 +
 linux-headers/asm-powerpc/kvm.h              |    3 +++
 linux-headers/linux/kvm.h                    |   11 +++++++++++
 linux-headers/linux/psp-sev.h                |    3 +++
 18 files changed, 101 insertions(+), 20 deletions(-)

[for-5.0 PATCH v2 1/4] linux-headers: Update

[Greg Kurz]

Update to mainline commit be2eca94d144 ("Merge tag 'for-linus-5.5-1'`
of git://github.com/cminyard/linux-ipmi")

Signed-off-by: Greg Kurz <groug@kaod.org>
---
 include/standard-headers/linux/ethtool.h     |    6 ++++++
 include/standard-headers/linux/virtio_ring.h |    2 +-
 linux-headers/asm-arm/kvm.h                  |    3 ++-
 linux-headers/asm-arm64/kvm.h                |    5 ++++-
 linux-headers/asm-mips/unistd_n32.h          |    1 +
 linux-headers/asm-mips/unistd_n64.h          |    1 +
 linux-headers/asm-mips/unistd_o32.h          |    1 +
 linux-headers/asm-powerpc/kvm.h              |    3 +++
 linux-headers/linux/kvm.h                    |   11 +++++++++++
 linux-headers/linux/psp-sev.h                |    3 +++
 10 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/include/standard-headers/linux/ethtool.h b/include/standard-headers/linux/ethtool.h
index 4ff422b635db..6e8a10ee1075 100644
--- a/include/standard-headers/linux/ethtool.h
+++ b/include/standard-headers/linux/ethtool.h
@@ -1507,6 +1507,11 @@ enum ethtool_link_mode_bit_indices {
 	ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT	 = 66,
 	ETHTOOL_LINK_MODE_100baseT1_Full_BIT		 = 67,
 	ETHTOOL_LINK_MODE_1000baseT1_Full_BIT		 = 68,
+	ETHTOOL_LINK_MODE_400000baseKR8_Full_BIT	 = 69,
+	ETHTOOL_LINK_MODE_400000baseSR8_Full_BIT	 = 70,
+	ETHTOOL_LINK_MODE_400000baseLR8_ER8_FR8_Full_BIT = 71,
+	ETHTOOL_LINK_MODE_400000baseDR8_Full_BIT	 = 72,
+	ETHTOOL_LINK_MODE_400000baseCR8_Full_BIT	 = 73,
 
 	/* must be last entry */
 	__ETHTOOL_LINK_MODE_MASK_NBITS
@@ -1618,6 +1623,7 @@ enum ethtool_link_mode_bit_indices {
 #define SPEED_56000		56000
 #define SPEED_100000		100000
 #define SPEED_200000		200000
+#define SPEED_400000		400000
 
 #define SPEED_UNKNOWN		-1
 
diff --git a/include/standard-headers/linux/virtio_ring.h b/include/standard-headers/linux/virtio_ring.h
index 306cd41147be..f230fed47960 100644
--- a/include/standard-headers/linux/virtio_ring.h
+++ b/include/standard-headers/linux/virtio_ring.h
@@ -167,7 +167,7 @@ static inline void vring_init(struct vring *vr, unsigned int num, void *p,
 {
 	vr->num = num;
 	vr->desc = p;
-	vr->avail = p + num*sizeof(struct vring_desc);
+	vr->avail = (struct vring_avail *)((char *)p + num * sizeof(struct vring_desc));
 	vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16)
 		+ align-1) & ~(align - 1));
 }
diff --git a/linux-headers/asm-arm/kvm.h b/linux-headers/asm-arm/kvm.h
index 9d379d337298..0db5644e27af 100644
--- a/linux-headers/asm-arm/kvm.h
+++ b/linux-headers/asm-arm/kvm.h
@@ -131,8 +131,9 @@ struct kvm_vcpu_events {
 	struct {
 		__u8 serror_pending;
 		__u8 serror_has_esr;
+		__u8 ext_dabt_pending;
 		/* Align it to 8 bytes */
-		__u8 pad[6];
+		__u8 pad[5];
 		__u64 serror_esr;
 	} exception;
 	__u32 reserved[12];
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index 0ce6e49f3a19..920af01c8b90 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -164,8 +164,9 @@ struct kvm_vcpu_events {
 	struct {
 		__u8 serror_pending;
 		__u8 serror_has_esr;
+		__u8 ext_dabt_pending;
 		/* Align it to 8 bytes */
-		__u8 pad[6];
+		__u8 pad[5];
 		__u64 serror_esr;
 	} exception;
 	__u32 reserved[12];
@@ -323,6 +324,8 @@ struct kvm_vcpu_events {
 #define KVM_ARM_VCPU_TIMER_CTRL		1
 #define   KVM_ARM_VCPU_TIMER_IRQ_VTIMER		0
 #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
+#define KVM_ARM_VCPU_PVTIME_CTRL	2
+#define   KVM_ARM_VCPU_PVTIME_IPA	0
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_VCPU2_SHIFT		28
diff --git a/linux-headers/asm-mips/unistd_n32.h b/linux-headers/asm-mips/unistd_n32.h
index 7dffe8e34e63..659d5c9ade47 100644
--- a/linux-headers/asm-mips/unistd_n32.h
+++ b/linux-headers/asm-mips/unistd_n32.h
@@ -364,6 +364,7 @@
 #define __NR_fsmount	(__NR_Linux + 432)
 #define __NR_fspick	(__NR_Linux + 433)
 #define __NR_pidfd_open	(__NR_Linux + 434)
+#define __NR_clone3	(__NR_Linux + 435)
 
 
 #endif /* _ASM_MIPS_UNISTD_N32_H */
diff --git a/linux-headers/asm-mips/unistd_n64.h b/linux-headers/asm-mips/unistd_n64.h
index f4592d6fc50c..4b6310a05c23 100644
--- a/linux-headers/asm-mips/unistd_n64.h
+++ b/linux-headers/asm-mips/unistd_n64.h
@@ -340,6 +340,7 @@
 #define __NR_fsmount	(__NR_Linux + 432)
 #define __NR_fspick	(__NR_Linux + 433)
 #define __NR_pidfd_open	(__NR_Linux + 434)
+#define __NR_clone3	(__NR_Linux + 435)
 
 
 #endif /* _ASM_MIPS_UNISTD_N64_H */
diff --git a/linux-headers/asm-mips/unistd_o32.h b/linux-headers/asm-mips/unistd_o32.h
index 04c6728352a5..4ce7b4e288a5 100644
--- a/linux-headers/asm-mips/unistd_o32.h
+++ b/linux-headers/asm-mips/unistd_o32.h
@@ -410,6 +410,7 @@
 #define __NR_fsmount	(__NR_Linux + 432)
 #define __NR_fspick	(__NR_Linux + 433)
 #define __NR_pidfd_open	(__NR_Linux + 434)
+#define __NR_clone3	(__NR_Linux + 435)
 
 
 #endif /* _ASM_MIPS_UNISTD_O32_H */
diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index b0f72dea8b11..264e266a85bf 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -667,6 +667,8 @@ struct kvm_ppc_cpu_char {
 
 /* PPC64 eXternal Interrupt Controller Specification */
 #define KVM_DEV_XICS_GRP_SOURCES	1	/* 64-bit source attributes */
+#define KVM_DEV_XICS_GRP_CTRL		2
+#define   KVM_DEV_XICS_NR_SERVERS	1
 
 /* Layout of 64-bit source attribute values */
 #define  KVM_XICS_DESTINATION_SHIFT	0
@@ -683,6 +685,7 @@ struct kvm_ppc_cpu_char {
 #define KVM_DEV_XIVE_GRP_CTRL		1
 #define   KVM_DEV_XIVE_RESET		1
 #define   KVM_DEV_XIVE_EQ_SYNC		2
+#define   KVM_DEV_XIVE_NR_SERVERS	3
 #define KVM_DEV_XIVE_GRP_SOURCE		2	/* 64-bit source identifier */
 #define KVM_DEV_XIVE_GRP_SOURCE_CONFIG	3	/* 64-bit source identifier */
 #define KVM_DEV_XIVE_GRP_EQ_CONFIG	4	/* 64-bit EQ identifier */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 3d9b18f7f871..3b27a1ae85cc 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -235,6 +235,7 @@ struct kvm_hyperv_exit {
 #define KVM_EXIT_S390_STSI        25
 #define KVM_EXIT_IOAPIC_EOI       26
 #define KVM_EXIT_HYPERV           27
+#define KVM_EXIT_ARM_NISV         28
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -394,6 +395,11 @@ struct kvm_run {
 		} eoi;
 		/* KVM_EXIT_HYPERV */
 		struct kvm_hyperv_exit hyperv;
+		/* KVM_EXIT_ARM_NISV */
+		struct {
+			__u64 esr_iss;
+			__u64 fault_ipa;
+		} arm_nisv;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -1000,6 +1006,9 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_PMU_EVENT_FILTER 173
 #define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
 #define KVM_CAP_HYPERV_DIRECT_TLBFLUSH 175
+#define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
+#define KVM_CAP_ARM_NISV_TO_USER 177
+#define KVM_CAP_ARM_INJECT_EXT_DABT 178
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1227,6 +1236,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_ARM_VGIC_ITS	KVM_DEV_TYPE_ARM_VGIC_ITS
 	KVM_DEV_TYPE_XIVE,
 #define KVM_DEV_TYPE_XIVE		KVM_DEV_TYPE_XIVE
+	KVM_DEV_TYPE_ARM_PV_TIME,
+#define KVM_DEV_TYPE_ARM_PV_TIME	KVM_DEV_TYPE_ARM_PV_TIME
 	KVM_DEV_TYPE_MAX,
 };
 
diff --git a/linux-headers/linux/psp-sev.h b/linux-headers/linux/psp-sev.h
index 34c39690c09d..31f971e89659 100644
--- a/linux-headers/linux/psp-sev.h
+++ b/linux-headers/linux/psp-sev.h
@@ -58,6 +58,9 @@ typedef enum {
 	SEV_RET_HWSEV_RET_PLATFORM,
 	SEV_RET_HWSEV_RET_UNSAFE,
 	SEV_RET_UNSUPPORTED,
+	SEV_RET_INVALID_PARAM,
+	SEV_RET_RESOURCE_LIMIT,
+	SEV_RET_SECURE_DATA_INVALID,
 	SEV_RET_MAX,
 } sev_ret_code;
 

[for-5.0 PATCH v2 2/4] spapr: Pass the maximum number of vCPUs to the KVM interrupt controller

[Greg Kurz]

The XIVE and XICS-on-XIVE KVM devices on POWER9 hosts can greatly reduce
their consumption of some scarce HW resources, namely Virtual Presenter
identifiers, if they know the maximum number of vCPUs that may run in the
VM.

Prepare ground for this by passing the value down to xics_kvm_connect()
and kvmppc_xive_connect(). This is purely mechanical, no functional
change.

Signed-off-by: Greg Kurz <groug@kaod.org>
---
 hw/intc/spapr_xive.c        |    6 ++++--
 hw/intc/spapr_xive_kvm.c    |    3 ++-
 hw/intc/xics_kvm.c          |    3 ++-
 hw/intc/xics_spapr.c        |    5 +++--
 hw/ppc/spapr_irq.c          |    8 +++++---
 include/hw/ppc/spapr_irq.h  |   10 ++++++++--
 include/hw/ppc/spapr_xive.h |    3 ++-
 include/hw/ppc/xics_spapr.h |    3 ++-
 8 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/hw/intc/spapr_xive.c b/hw/intc/spapr_xive.c
index 729246e906c9..8364f61c786c 100644
--- a/hw/intc/spapr_xive.c
+++ b/hw/intc/spapr_xive.c
@@ -651,12 +651,14 @@ static void spapr_xive_dt(SpaprInterruptController *intc, uint32_t nr_servers,
                      plat_res_int_priorities, sizeof(plat_res_int_priorities)));
 }
 
-static int spapr_xive_activate(SpaprInterruptController *intc, Error **errp)
+static int spapr_xive_activate(SpaprInterruptController *intc,
+                               uint32_t nr_servers, Error **errp)
 {
     SpaprXive *xive = SPAPR_XIVE(intc);
 
     if (kvm_enabled()) {
-        int rc = spapr_irq_init_kvm(kvmppc_xive_connect, intc, errp);
+        int rc = spapr_irq_init_kvm(kvmppc_xive_connect, intc, nr_servers,
+                                    errp);
         if (rc < 0) {
             return rc;
         }
diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
index 69e73552f1ef..46c7609bd8d0 100644
--- a/hw/intc/spapr_xive_kvm.c
+++ b/hw/intc/spapr_xive_kvm.c
@@ -728,7 +728,8 @@ static void *kvmppc_xive_mmap(SpaprXive *xive, int pgoff, size_t len,
  * All the XIVE memory regions are now backed by mappings from the KVM
  * XIVE device.
  */
-int kvmppc_xive_connect(SpaprInterruptController *intc, Error **errp)
+int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
+                        Error **errp)
 {
     SpaprXive *xive = SPAPR_XIVE(intc);
     XiveSource *xsrc = &xive->source;
diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
index 954c424b3652..a1f1b7b0d37c 100644
--- a/hw/intc/xics_kvm.c
+++ b/hw/intc/xics_kvm.c
@@ -342,7 +342,8 @@ void ics_kvm_set_irq(ICSState *ics, int srcno, int val)
     }
 }
 
-int xics_kvm_connect(SpaprInterruptController *intc, Error **errp)
+int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
+                     Error **errp)
 {
     ICSState *ics = ICS_SPAPR(intc);
     int rc;
diff --git a/hw/intc/xics_spapr.c b/hw/intc/xics_spapr.c
index b3705dab0e8a..8ae4f41459c3 100644
--- a/hw/intc/xics_spapr.c
+++ b/hw/intc/xics_spapr.c
@@ -422,10 +422,11 @@ static int xics_spapr_post_load(SpaprInterruptController *intc, int version_id)
     return 0;
 }
 
-static int xics_spapr_activate(SpaprInterruptController *intc, Error **errp)
+static int xics_spapr_activate(SpaprInterruptController *intc,
+                               uint32_t nr_servers, Error **errp)
 {
     if (kvm_enabled()) {
-        return spapr_irq_init_kvm(xics_kvm_connect, intc, errp);
+        return spapr_irq_init_kvm(xics_kvm_connect, intc, nr_servers, errp);
     }
     return 0;
 }
diff --git a/hw/ppc/spapr_irq.c b/hw/ppc/spapr_irq.c
index d4a54afc8662..07e08d6544a0 100644
--- a/hw/ppc/spapr_irq.c
+++ b/hw/ppc/spapr_irq.c
@@ -70,15 +70,16 @@ void spapr_irq_msi_free(SpaprMachineState *spapr, int irq, uint32_t num)
     bitmap_clear(spapr->irq_map, irq - SPAPR_IRQ_MSI, num);
 }
 
-int spapr_irq_init_kvm(int (*fn)(SpaprInterruptController *, Error **),
+int spapr_irq_init_kvm(SpaprInterruptControllerInitKvm fn,
                        SpaprInterruptController *intc,
+                       uint32_t nr_servers,
                        Error **errp)
 {
     MachineState *machine = MACHINE(qdev_get_machine());
     Error *local_err = NULL;
 
     if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
-        if (fn(intc, &local_err) < 0) {
+        if (fn(intc, nr_servers, &local_err) < 0) {
             if (machine_kernel_irqchip_required(machine)) {
                 error_prepend(&local_err,
                               "kernel_irqchip requested but unavailable: ");
@@ -481,6 +482,7 @@ static void set_active_intc(SpaprMachineState *spapr,
                             SpaprInterruptController *new_intc)
 {
     SpaprInterruptControllerClass *sicc;
+    uint32_t nr_servers = spapr_max_server_number(spapr);
 
     assert(new_intc);
 
@@ -498,7 +500,7 @@ static void set_active_intc(SpaprMachineState *spapr,
 
     sicc = SPAPR_INTC_GET_CLASS(new_intc);
     if (sicc->activate) {
-        sicc->activate(new_intc, &error_fatal);
+        sicc->activate(new_intc, nr_servers, &error_fatal);
     }
 
     spapr->active_intc = new_intc;
diff --git a/include/hw/ppc/spapr_irq.h b/include/hw/ppc/spapr_irq.h
index ff814d13de37..ca8cb4421374 100644
--- a/include/hw/ppc/spapr_irq.h
+++ b/include/hw/ppc/spapr_irq.h
@@ -43,7 +43,8 @@ typedef struct SpaprInterruptController SpaprInterruptController;
 typedef struct SpaprInterruptControllerClass {
     InterfaceClass parent;
 
-    int (*activate)(SpaprInterruptController *intc, Error **errp);
+    int (*activate)(SpaprInterruptController *intc, uint32_t nr_servers,
+                    Error **errp);
     void (*deactivate)(SpaprInterruptController *intc);
 
     /*
@@ -98,8 +99,13 @@ qemu_irq spapr_qirq(SpaprMachineState *spapr, int irq);
 int spapr_irq_post_load(SpaprMachineState *spapr, int version_id);
 void spapr_irq_reset(SpaprMachineState *spapr, Error **errp);
 int spapr_irq_get_phandle(SpaprMachineState *spapr, void *fdt, Error **errp);
-int spapr_irq_init_kvm(int (*fn)(SpaprInterruptController *, Error **),
+
+typedef int (*SpaprInterruptControllerInitKvm)(SpaprInterruptController *,
+                                               uint32_t, Error **);
+
+int spapr_irq_init_kvm(SpaprInterruptControllerInitKvm fn,
                        SpaprInterruptController *intc,
+                       uint32_t nr_servers,
                        Error **errp);
 
 /*
diff --git a/include/hw/ppc/spapr_xive.h b/include/hw/ppc/spapr_xive.h
index 742b7e834f2a..3a103c224d44 100644
--- a/include/hw/ppc/spapr_xive.h
+++ b/include/hw/ppc/spapr_xive.h
@@ -66,7 +66,8 @@ int spapr_xive_end_to_target(uint8_t end_blk, uint32_t end_idx,
 /*
  * KVM XIVE device helpers
  */
-int kvmppc_xive_connect(SpaprInterruptController *intc, Error **errp);
+int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
+                        Error **errp);
 void kvmppc_xive_disconnect(SpaprInterruptController *intc);
 void kvmppc_xive_reset(SpaprXive *xive, Error **errp);
 void kvmppc_xive_set_source_config(SpaprXive *xive, uint32_t lisn, XiveEAS *eas,
diff --git a/include/hw/ppc/xics_spapr.h b/include/hw/ppc/xics_spapr.h
index 28b87038c89a..1c65c96e3c7a 100644
--- a/include/hw/ppc/xics_spapr.h
+++ b/include/hw/ppc/xics_spapr.h
@@ -32,7 +32,8 @@
 #define TYPE_ICS_SPAPR "ics-spapr"
 #define ICS_SPAPR(obj) OBJECT_CHECK(ICSState, (obj), TYPE_ICS_SPAPR)
 
-int xics_kvm_connect(SpaprInterruptController *intc, Error **errp);
+int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
+                     Error **errp);
 void xics_kvm_disconnect(SpaprInterruptController *intc);
 bool xics_kvm_has_broken_disconnect(SpaprMachineState *spapr);
 

[for-5.0 PATCH v2 3/4] spapr/xics: Configure number of servers in KVM

[Greg Kurz]

The XICS-on-XIVE KVM devices now has an attribute to configure the number
of interrupt servers. This allows to greatly optimize the usage of the VP
space in the XIVE HW, and thus to start a lot more VMs.

Only set this attribute if available in order to support older POWER9 KVM
and pre-POWER9 XICS KVM devices.

The XICS-on-XIVE KVM device now reports the exhaustion of VPs upon the
connection of the first VCPU. Check that in order to have a chance to
provide a hint to the user.
`
Signed-off-by: Greg Kurz <groug@kaod.org>
---
 hw/intc/xics_kvm.c |   21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
index a1f1b7b0d37c..8d6156578fbc 100644
--- a/hw/intc/xics_kvm.c
+++ b/hw/intc/xics_kvm.c
@@ -165,8 +165,15 @@ void icp_kvm_realize(DeviceState *dev, Error **errp)
 
     ret = kvm_vcpu_enable_cap(cs, KVM_CAP_IRQ_XICS, 0, kernel_xics_fd, vcpu_id);
     if (ret < 0) {
-        error_setg(errp, "Unable to connect CPU%ld to kernel XICS: %s", vcpu_id,
-                   strerror(errno));
+        Error *local_err = NULL;
+
+        error_setg(&local_err, "Unable to connect CPU%ld to kernel XICS: %s",
+                   vcpu_id, strerror(errno));
+        if (errno == ENOSPC) {
+            error_append_hint(&local_err, "Try -smp maxcpus=N with N < %u\n",
+                              MACHINE(qdev_get_machine())->smp.max_cpus);
+        }
+        error_propagate(errp, local_err);
         return;
     }
     enabled_icp = g_malloc(sizeof(*enabled_icp));
@@ -399,6 +406,16 @@ int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
         goto fail;
     }
 
+    /* Tell KVM about the # of VCPUs we may have (POWER9 and newer only) */
+    if (kvm_device_check_attr(rc, KVM_DEV_XICS_GRP_CTRL,
+                              KVM_DEV_XICS_NR_SERVERS)) {
+        if (kvm_device_access(rc, KVM_DEV_XICS_GRP_CTRL,
+                              KVM_DEV_XICS_NR_SERVERS, &nr_servers, true,
+                              &local_err)) {
+            goto fail;
+        }
+    }
+
     kernel_xics_fd = rc;
     kvm_kernel_irqchip = true;
     kvm_msi_via_irqfd_allowed = true;

[for-5.0 PATCH v2 4/4] spapr/xive: Configure number of servers in KVM

[Greg Kurz]

The XIVE KVM devices now has an attribute to configure the number of
interrupt servers. This allows to greatly optimize the usage of the VP
space in the XIVE HW, and thus to start a lot more VMs.

Only set this attribute if available in order to support older POWER9
KVM.

The XIVE KVM device now reports the exhaustion of VPs upon the
connection of the first VCPU. Check that in order to have a chance
to provide a hint to the user.

Signed-off-by: Greg Kurz <groug@kaod.org>
---
 hw/intc/spapr_xive_kvm.c |   23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
index 46c7609bd8d0..32b2809210a0 100644
--- a/hw/intc/spapr_xive_kvm.c
+++ b/hw/intc/spapr_xive_kvm.c
@@ -152,7 +152,8 @@ void kvmppc_xive_cpu_synchronize_state(XiveTCTX *tctx, Error **errp)
 
 void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)
 {
-    SpaprXive *xive = SPAPR_MACHINE(qdev_get_machine())->xive;
+    MachineState *ms = MACHINE(qdev_get_machine());
+    SpaprXive *xive = SPAPR_MACHINE(ms)->xive;
     unsigned long vcpu_id;
     int ret;
 
@@ -171,8 +172,16 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)
     ret = kvm_vcpu_enable_cap(tctx->cs, KVM_CAP_PPC_IRQ_XIVE, 0, xive->fd,
                               vcpu_id, 0);
     if (ret < 0) {
-        error_setg(errp, "XIVE: unable to connect CPU%ld to KVM device: %s",
+        Error *local_err = NULL;
+
+        error_setg(&local_err,
+                   "XIVE: unable to connect CPU%ld to KVM device: %s",
                    vcpu_id, strerror(errno));
+        if (errno == ENOSPC) {
+            error_append_hint(&local_err, "Try -smp maxcpus=N with N < %u\n",
+                              ms->smp.max_cpus);
+        }
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -758,6 +767,16 @@ int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
         return -1;
     }
 
+    /* Tell KVM about the # of VCPUs we may have */
+    if (kvm_device_check_attr(xive->fd, KVM_DEV_XIVE_GRP_CTRL,
+                              KVM_DEV_XIVE_NR_SERVERS)) {
+        if (kvm_device_access(xive->fd, KVM_DEV_XIVE_GRP_CTRL,
+                              KVM_DEV_XIVE_NR_SERVERS, &nr_servers, true,
+                              &local_err)) {
+            goto fail;
+        }
+    }
+
     /*
      * 1. Source ESB pages - KVM mapping
      */

Re: [for-5.0 PATCH v2 2/4] spapr: Pass the maximum number of vCPUs to the KVM interrupt controller

[Cédric Le Goater]

On 26/11/2019 17:46, Greg Kurz wrote:
> The XIVE and XICS-on-XIVE KVM devices on POWER9 hosts can greatly reduce
> their consumption of some scarce HW resources, namely Virtual Presenter
> identifiers, if they know the maximum number of vCPUs that may run in the
> VM.
> 
> Prepare ground for this by passing the value down to xics_kvm_connect()
> and kvmppc_xive_connect(). This is purely mechanical, no functional
> change.
> 
> Signed-off-by: Greg Kurz <groug@kaod.org>

Reviewed-by: Cédric Le Goater <clg@kaod.org>

> ---
>  hw/intc/spapr_xive.c        |    6 ++++--
>  hw/intc/spapr_xive_kvm.c    |    3 ++-
>  hw/intc/xics_kvm.c          |    3 ++-
>  hw/intc/xics_spapr.c        |    5 +++--
>  hw/ppc/spapr_irq.c          |    8 +++++---
>  include/hw/ppc/spapr_irq.h  |   10 ++++++++--
>  include/hw/ppc/spapr_xive.h |    3 ++-
>  include/hw/ppc/xics_spapr.h |    3 ++-
>  8 files changed, 28 insertions(+), 13 deletions(-)
> 
> diff --git a/hw/intc/spapr_xive.c b/hw/intc/spapr_xive.c
> index 729246e906c9..8364f61c786c 100644
> --- a/hw/intc/spapr_xive.c
> +++ b/hw/intc/spapr_xive.c
> @@ -651,12 +651,14 @@ static void spapr_xive_dt(SpaprInterruptController *intc, uint32_t nr_servers,
>                       plat_res_int_priorities, sizeof(plat_res_int_priorities)));
>  }
>  
> -static int spapr_xive_activate(SpaprInterruptController *intc, Error **errp)
> +static int spapr_xive_activate(SpaprInterruptController *intc,
> +                               uint32_t nr_servers, Error **errp)
>  {
>      SpaprXive *xive = SPAPR_XIVE(intc);
>  
>      if (kvm_enabled()) {
> -        int rc = spapr_irq_init_kvm(kvmppc_xive_connect, intc, errp);
> +        int rc = spapr_irq_init_kvm(kvmppc_xive_connect, intc, nr_servers,
> +                                    errp);
>          if (rc < 0) {
>              return rc;
>          }
> diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
> index 69e73552f1ef..46c7609bd8d0 100644
> --- a/hw/intc/spapr_xive_kvm.c
> +++ b/hw/intc/spapr_xive_kvm.c
> @@ -728,7 +728,8 @@ static void *kvmppc_xive_mmap(SpaprXive *xive, int pgoff, size_t len,
>   * All the XIVE memory regions are now backed by mappings from the KVM
>   * XIVE device.
>   */
> -int kvmppc_xive_connect(SpaprInterruptController *intc, Error **errp)
> +int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
> +                        Error **errp)
>  {
>      SpaprXive *xive = SPAPR_XIVE(intc);
>      XiveSource *xsrc = &xive->source;
> diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
> index 954c424b3652..a1f1b7b0d37c 100644
> --- a/hw/intc/xics_kvm.c
> +++ b/hw/intc/xics_kvm.c
> @@ -342,7 +342,8 @@ void ics_kvm_set_irq(ICSState *ics, int srcno, int val)
>      }
>  }
>  
> -int xics_kvm_connect(SpaprInterruptController *intc, Error **errp)
> +int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
> +                     Error **errp)
>  {
>      ICSState *ics = ICS_SPAPR(intc);
>      int rc;
> diff --git a/hw/intc/xics_spapr.c b/hw/intc/xics_spapr.c
> index b3705dab0e8a..8ae4f41459c3 100644
> --- a/hw/intc/xics_spapr.c
> +++ b/hw/intc/xics_spapr.c
> @@ -422,10 +422,11 @@ static int xics_spapr_post_load(SpaprInterruptController *intc, int version_id)
>      return 0;
>  }
>  
> -static int xics_spapr_activate(SpaprInterruptController *intc, Error **errp)
> +static int xics_spapr_activate(SpaprInterruptController *intc,
> +                               uint32_t nr_servers, Error **errp)
>  {
>      if (kvm_enabled()) {
> -        return spapr_irq_init_kvm(xics_kvm_connect, intc, errp);
> +        return spapr_irq_init_kvm(xics_kvm_connect, intc, nr_servers, errp);
>      }
>      return 0;
>  }
> diff --git a/hw/ppc/spapr_irq.c b/hw/ppc/spapr_irq.c
> index d4a54afc8662..07e08d6544a0 100644
> --- a/hw/ppc/spapr_irq.c
> +++ b/hw/ppc/spapr_irq.c
> @@ -70,15 +70,16 @@ void spapr_irq_msi_free(SpaprMachineState *spapr, int irq, uint32_t num)
>      bitmap_clear(spapr->irq_map, irq - SPAPR_IRQ_MSI, num);
>  }
>  
> -int spapr_irq_init_kvm(int (*fn)(SpaprInterruptController *, Error **),
> +int spapr_irq_init_kvm(SpaprInterruptControllerInitKvm fn,
>                         SpaprInterruptController *intc,
> +                       uint32_t nr_servers,
>                         Error **errp)
>  {
>      MachineState *machine = MACHINE(qdev_get_machine());
>      Error *local_err = NULL;
>  
>      if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
> -        if (fn(intc, &local_err) < 0) {
> +        if (fn(intc, nr_servers, &local_err) < 0) {
>              if (machine_kernel_irqchip_required(machine)) {
>                  error_prepend(&local_err,
>                                "kernel_irqchip requested but unavailable: ");
> @@ -481,6 +482,7 @@ static void set_active_intc(SpaprMachineState *spapr,
>                              SpaprInterruptController *new_intc)
>  {
>      SpaprInterruptControllerClass *sicc;
> +    uint32_t nr_servers = spapr_max_server_number(spapr);
>  
>      assert(new_intc);
>  
> @@ -498,7 +500,7 @@ static void set_active_intc(SpaprMachineState *spapr,
>  
>      sicc = SPAPR_INTC_GET_CLASS(new_intc);
>      if (sicc->activate) {
> -        sicc->activate(new_intc, &error_fatal);
> +        sicc->activate(new_intc, nr_servers, &error_fatal);
>      }
>  
>      spapr->active_intc = new_intc;
> diff --git a/include/hw/ppc/spapr_irq.h b/include/hw/ppc/spapr_irq.h
> index ff814d13de37..ca8cb4421374 100644
> --- a/include/hw/ppc/spapr_irq.h
> +++ b/include/hw/ppc/spapr_irq.h
> @@ -43,7 +43,8 @@ typedef struct SpaprInterruptController SpaprInterruptController;
>  typedef struct SpaprInterruptControllerClass {
>      InterfaceClass parent;
>  
> -    int (*activate)(SpaprInterruptController *intc, Error **errp);
> +    int (*activate)(SpaprInterruptController *intc, uint32_t nr_servers,
> +                    Error **errp);
>      void (*deactivate)(SpaprInterruptController *intc);
>  
>      /*
> @@ -98,8 +99,13 @@ qemu_irq spapr_qirq(SpaprMachineState *spapr, int irq);
>  int spapr_irq_post_load(SpaprMachineState *spapr, int version_id);
>  void spapr_irq_reset(SpaprMachineState *spapr, Error **errp);
>  int spapr_irq_get_phandle(SpaprMachineState *spapr, void *fdt, Error **errp);
> -int spapr_irq_init_kvm(int (*fn)(SpaprInterruptController *, Error **),
> +
> +typedef int (*SpaprInterruptControllerInitKvm)(SpaprInterruptController *,
> +                                               uint32_t, Error **);
> +
> +int spapr_irq_init_kvm(SpaprInterruptControllerInitKvm fn,
>                         SpaprInterruptController *intc,
> +                       uint32_t nr_servers,
>                         Error **errp);
>  
>  /*
> diff --git a/include/hw/ppc/spapr_xive.h b/include/hw/ppc/spapr_xive.h
> index 742b7e834f2a..3a103c224d44 100644
> --- a/include/hw/ppc/spapr_xive.h
> +++ b/include/hw/ppc/spapr_xive.h
> @@ -66,7 +66,8 @@ int spapr_xive_end_to_target(uint8_t end_blk, uint32_t end_idx,
>  /*
>   * KVM XIVE device helpers
>   */
> -int kvmppc_xive_connect(SpaprInterruptController *intc, Error **errp);
> +int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
> +                        Error **errp);
>  void kvmppc_xive_disconnect(SpaprInterruptController *intc);
>  void kvmppc_xive_reset(SpaprXive *xive, Error **errp);
>  void kvmppc_xive_set_source_config(SpaprXive *xive, uint32_t lisn, XiveEAS *eas,
> diff --git a/include/hw/ppc/xics_spapr.h b/include/hw/ppc/xics_spapr.h
> index 28b87038c89a..1c65c96e3c7a 100644
> --- a/include/hw/ppc/xics_spapr.h
> +++ b/include/hw/ppc/xics_spapr.h
> @@ -32,7 +32,8 @@
>  #define TYPE_ICS_SPAPR "ics-spapr"
>  #define ICS_SPAPR(obj) OBJECT_CHECK(ICSState, (obj), TYPE_ICS_SPAPR)
>  
> -int xics_kvm_connect(SpaprInterruptController *intc, Error **errp);
> +int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
> +                     Error **errp);
>  void xics_kvm_disconnect(SpaprInterruptController *intc);
>  bool xics_kvm_has_broken_disconnect(SpaprMachineState *spapr);
>  
> 

Re: [for-5.0 PATCH v2 4/4] spapr/xive: Configure number of servers in KVM

[Cédric Le Goater]

On 26/11/2019 17:46, Greg Kurz wrote:
> The XIVE KVM devices now has an attribute to configure the number of
> interrupt servers. This allows to greatly optimize the usage of the VP
> space in the XIVE HW, and thus to start a lot more VMs.
> 
> Only set this attribute if available in order to support older POWER9
> KVM.
> 
> The XIVE KVM device now reports the exhaustion of VPs upon the
> connection of the first VCPU. Check that in order to have a chance
> to provide a hint to the user.
> 
> Signed-off-by: Greg Kurz <groug@kaod.org>

Reviewed-by: Cédric Le Goater <clg@kaod.org>

> ---
>  hw/intc/spapr_xive_kvm.c |   23 +++++++++++++++++++++--
>  1 file changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
> index 46c7609bd8d0..32b2809210a0 100644
> --- a/hw/intc/spapr_xive_kvm.c
> +++ b/hw/intc/spapr_xive_kvm.c
> @@ -152,7 +152,8 @@ void kvmppc_xive_cpu_synchronize_state(XiveTCTX *tctx, Error **errp)
>  
>  void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)
>  {
> -    SpaprXive *xive = SPAPR_MACHINE(qdev_get_machine())->xive;
> +    MachineState *ms = MACHINE(qdev_get_machine());
> +    SpaprXive *xive = SPAPR_MACHINE(ms)->xive;
>      unsigned long vcpu_id;
>      int ret;
>  
> @@ -171,8 +172,16 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)
>      ret = kvm_vcpu_enable_cap(tctx->cs, KVM_CAP_PPC_IRQ_XIVE, 0, xive->fd,
>                                vcpu_id, 0);
>      if (ret < 0) {
> -        error_setg(errp, "XIVE: unable to connect CPU%ld to KVM device: %s",
> +        Error *local_err = NULL;
> +
> +        error_setg(&local_err,
> +                   "XIVE: unable to connect CPU%ld to KVM device: %s",
>                     vcpu_id, strerror(errno));
> +        if (errno == ENOSPC) {
> +            error_append_hint(&local_err, "Try -smp maxcpus=N with N < %u\n",
> +                              ms->smp.max_cpus);
> +        }
> +        error_propagate(errp, local_err);
>          return;
>      }
>  
> @@ -758,6 +767,16 @@ int kvmppc_xive_connect(SpaprInterruptController *intc, uint32_t nr_servers,
>          return -1;
>      }
>  
> +    /* Tell KVM about the # of VCPUs we may have */
> +    if (kvm_device_check_attr(xive->fd, KVM_DEV_XIVE_GRP_CTRL,
> +                              KVM_DEV_XIVE_NR_SERVERS)) {
> +        if (kvm_device_access(xive->fd, KVM_DEV_XIVE_GRP_CTRL,
> +                              KVM_DEV_XIVE_NR_SERVERS, &nr_servers, true,
> +                              &local_err)) {
> +            goto fail;
> +        }
> +    }
> +
>      /*
>       * 1. Source ESB pages - KVM mapping
>       */
> 

Re: [for-5.0 PATCH v2 3/4] spapr/xics: Configure number of servers in KVM

[Cédric Le Goater]

On 26/11/2019 17:46, Greg Kurz wrote:
> The XICS-on-XIVE KVM devices now has an attribute to configure the number
> of interrupt servers. This allows to greatly optimize the usage of the VP
> space in the XIVE HW, and thus to start a lot more VMs.
> 
> Only set this attribute if available in order to support older POWER9 KVM
> and pre-POWER9 XICS KVM devices.
> 
> The XICS-on-XIVE KVM device now reports the exhaustion of VPs upon the
> connection of the first VCPU. Check that in order to have a chance to
> provide a hint to the user.
> `
> Signed-off-by: Greg Kurz <groug@kaod.org>


Reviewed-by: Cédric Le Goater <clg@kaod.org>

> ---
>  hw/intc/xics_kvm.c |   21 +++++++++++++++++++--
>  1 file changed, 19 insertions(+), 2 deletions(-)
> 
> diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
> index a1f1b7b0d37c..8d6156578fbc 100644
> --- a/hw/intc/xics_kvm.c
> +++ b/hw/intc/xics_kvm.c
> @@ -165,8 +165,15 @@ void icp_kvm_realize(DeviceState *dev, Error **errp)
>  
>      ret = kvm_vcpu_enable_cap(cs, KVM_CAP_IRQ_XICS, 0, kernel_xics_fd, vcpu_id);
>      if (ret < 0) {
> -        error_setg(errp, "Unable to connect CPU%ld to kernel XICS: %s", vcpu_id,
> -                   strerror(errno));
> +        Error *local_err = NULL;
> +
> +        error_setg(&local_err, "Unable to connect CPU%ld to kernel XICS: %s",
> +                   vcpu_id, strerror(errno));
> +        if (errno == ENOSPC) {
> +            error_append_hint(&local_err, "Try -smp maxcpus=N with N < %u\n",
> +                              MACHINE(qdev_get_machine())->smp.max_cpus);
> +        }
> +        error_propagate(errp, local_err);
>          return;
>      }
>      enabled_icp = g_malloc(sizeof(*enabled_icp));
> @@ -399,6 +406,16 @@ int xics_kvm_connect(SpaprInterruptController *intc, uint32_t nr_servers,
>          goto fail;
>      }
>  
> +    /* Tell KVM about the # of VCPUs we may have (POWER9 and newer only) */
> +    if (kvm_device_check_attr(rc, KVM_DEV_XICS_GRP_CTRL,
> +                              KVM_DEV_XICS_NR_SERVERS)) {
> +        if (kvm_device_access(rc, KVM_DEV_XICS_GRP_CTRL,
> +                              KVM_DEV_XICS_NR_SERVERS, &nr_servers, true,
> +                              &local_err)) {
> +            goto fail;
> +        }
> +    }
> +
>      kernel_xics_fd = rc;
>      kvm_kernel_irqchip = true;
>      kvm_msi_via_irqfd_allowed = true;
> 

Re: [for-5.0 PATCH v2 0/4] spapr: Use less XIVE HW resources in KVM

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 1547 bytes --]

On Tue, Nov 26, 2019 at 05:46:12PM +0100, Greg Kurz wrote:
> On POWER9 systems, the XICS-on-XIVE and XIVE KVM devices currently
> allocate a bunch of VPs in the XIVE HW to accomodate the highest
> VCPU id that may be possibly used in a VM. This limits the number
> of VMs that can run with an in-kernel interrupt controller to 63
> per POWER9 chip, irrespectively of its number of HW threads, eg.
> up to 96 on a POWER9 Nimbus socket. This is an unfortunate waste
> of scarce HW resources since a typical VM doesn't need that much
> VPs to run.
> 
> This series exploits new attributes of the XICS-on-XIVE and XIVE
> KVM devices that allow userspace to tune the numbers of VPs it
> really needs. Support for this attributes has just reached linux
> mainline.
> 
> In combination with recent commit 29cb4187497d "spapr: Set VSMT to
> smp_threads by default" which makes VCPU ids contiguous, it is
> possible to start more than 200 VMs instead of 127 on an AC922
> system (two sockets, 144 HW threads).
> 
> Changes in v2:
> - add an nr_servers argument to the activate() handler instead of
>   a QOM property for XIVE and a XICSFabric method for XICS-on-XIVE.
> - even if the last two patches haven't changed much, the code base
>   itself has, so I've deliberately dropped Cedric's R-b tags

Applied to ppc-for-5.0, thanks.

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]