[PATCH 0/4] RfC: microvm: add second ioapic

[PATCH 0/4] RfC: microvm: add second ioapic

[Gerd Hoffmann]

Add a second ioapic to microvm.  Gives us more IRQ lines we can
use for virtio-mmio devices.  Bump number of possible virtio-mmio
devices from 8 to 24.

Gerd Hoffmann (4):
  microvm: make number of virtio transports runtime configurable
  microvm: make pcie irq base runtime configurable
  microvm: add second ioapic
  microvm: reconfigure irqs if second ioapic is available

 include/hw/i386/ioapic_internal.h |  2 +-
 include/hw/i386/microvm.h         |  4 +--
 include/hw/i386/x86.h             |  1 +
 hw/i386/acpi-common.c             | 12 ++++++-
 hw/i386/microvm.c                 | 54 +++++++++++++++++++++++++------
 5 files changed, 60 insertions(+), 13 deletions(-)

-- 
2.27.0

[PATCH 1/4] microvm: make number of virtio transports runtime configurable

[Gerd Hoffmann]

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 include/hw/i386/microvm.h | 2 +-
 hw/i386/microvm.c         | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/include/hw/i386/microvm.h b/include/hw/i386/microvm.h
index 91b064575d55..0154ad5bd707 100644
--- a/include/hw/i386/microvm.h
+++ b/include/hw/i386/microvm.h
@@ -52,7 +52,6 @@
 
 /* Platform virtio definitions */
 #define VIRTIO_MMIO_BASE      0xfeb00000
-#define VIRTIO_NUM_TRANSPORTS 8
 #define VIRTIO_CMDLINE_MAXLEN 64
 
 #define GED_MMIO_BASE         0xfea00000
@@ -95,6 +94,7 @@ struct MicrovmMachineState {
 
     /* Machine state */
     uint32_t virtio_irq_base;
+    uint32_t virtio_num_transports;
     bool kernel_cmdline_fixed;
     Notifier machine_done;
     Notifier powerdown_req;
diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
index 9dd74458aca4..eaf5da31f7e1 100644
--- a/hw/i386/microvm.c
+++ b/hw/i386/microvm.c
@@ -177,8 +177,13 @@ static void microvm_devices_init(MicrovmMachineState *mms)
 
     kvmclock_create(true);
 
-    mms->virtio_irq_base = x86_machine_is_acpi_enabled(x86ms) ? 16 : 5;
-    for (i = 0; i < VIRTIO_NUM_TRANSPORTS; i++) {
+    mms->virtio_irq_base = 5;
+    mms->virtio_num_transports = 8;
+    if (x86_machine_is_acpi_enabled(x86ms)) {
+        mms->virtio_irq_base = 16;
+    }
+
+    for (i = 0; i < mms->virtio_num_transports; i++) {
         sysbus_create_simple("virtio-mmio",
                              VIRTIO_MMIO_BASE + i * 512,
                              x86ms->gsi[mms->virtio_irq_base + i]);
-- 
2.27.0

[PATCH 2/4] microvm: make pcie irq base runtime configurable

[Gerd Hoffmann]

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 include/hw/i386/microvm.h |  2 +-
 hw/i386/microvm.c         | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/include/hw/i386/microvm.h b/include/hw/i386/microvm.h
index 0154ad5bd707..ede9625756b8 100644
--- a/include/hw/i386/microvm.h
+++ b/include/hw/i386/microvm.h
@@ -63,7 +63,6 @@
 #define PCIE_MMIO_SIZE        0x20000000
 #define PCIE_ECAM_BASE        0xe0000000
 #define PCIE_ECAM_SIZE        0x10000000
-#define PCIE_IRQ_BASE         12
 
 /* Machine type options */
 #define MICROVM_MACHINE_PIT                 "pit"
@@ -93,6 +92,7 @@ struct MicrovmMachineState {
     bool auto_kernel_cmdline;
 
     /* Machine state */
+    uint32_t pcie_irq_base;
     uint32_t virtio_irq_base;
     uint32_t virtio_num_transports;
     bool kernel_cmdline_fixed;
diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
index eaf5da31f7e1..638e95c39e8c 100644
--- a/hw/i386/microvm.c
+++ b/hw/i386/microvm.c
@@ -180,6 +180,7 @@ static void microvm_devices_init(MicrovmMachineState *mms)
     mms->virtio_irq_base = 5;
     mms->virtio_num_transports = 8;
     if (x86_machine_is_acpi_enabled(x86ms)) {
+        mms->pcie_irq_base = 12;
         mms->virtio_irq_base = 16;
     }
 
@@ -213,12 +214,12 @@ static void microvm_devices_init(MicrovmMachineState *mms)
         mms->gpex.mmio32.size = PCIE_MMIO_SIZE;
         mms->gpex.ecam.base   = PCIE_ECAM_BASE;
         mms->gpex.ecam.size   = PCIE_ECAM_SIZE;
-        mms->gpex.irq         = PCIE_IRQ_BASE;
+        mms->gpex.irq         = mms->pcie_irq_base;
         create_gpex(mms);
-        x86ms->pci_irq_mask = ((1 << (PCIE_IRQ_BASE + 0)) |
-                               (1 << (PCIE_IRQ_BASE + 1)) |
-                               (1 << (PCIE_IRQ_BASE + 2)) |
-                               (1 << (PCIE_IRQ_BASE + 3)));
+        x86ms->pci_irq_mask = ((1 << (mms->pcie_irq_base + 0)) |
+                               (1 << (mms->pcie_irq_base + 1)) |
+                               (1 << (mms->pcie_irq_base + 2)) |
+                               (1 << (mms->pcie_irq_base + 3)));
     } else {
         x86ms->pci_irq_mask = 0;
     }
-- 
2.27.0

[PATCH 3/4] microvm: add second ioapic

[Gerd Hoffmann]

Add more IRQ lines.  Depends on ACPI.
Also enable this only with userspace ioapic,
not sure whenever the kernel can handle two ioapics.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 include/hw/i386/ioapic_internal.h |  2 +-
 include/hw/i386/x86.h             |  1 +
 hw/i386/acpi-common.c             | 10 ++++++++++
 hw/i386/microvm.c                 | 30 ++++++++++++++++++++++++++++--
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/include/hw/i386/ioapic_internal.h b/include/hw/i386/ioapic_internal.h
index 0ac9e2400d6b..4cebd2e32c9f 100644
--- a/include/hw/i386/ioapic_internal.h
+++ b/include/hw/i386/ioapic_internal.h
@@ -27,7 +27,7 @@
 #include "qemu/notify.h"
 #include "qom/object.h"
 
-#define MAX_IOAPICS                     1
+#define MAX_IOAPICS                     2
 
 #define IOAPIC_LVT_DEST_SHIFT           56
 #define IOAPIC_LVT_DEST_IDX_SHIFT       48
diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h
index bfa9cb2a258b..6da57033a875 100644
--- a/include/hw/i386/x86.h
+++ b/include/hw/i386/x86.h
@@ -120,6 +120,7 @@ bool x86_machine_is_acpi_enabled(const X86MachineState *x86ms);
 typedef struct GSIState {
     qemu_irq i8259_irq[ISA_NUM_IRQS];
     qemu_irq ioapic_irq[IOAPIC_NUM_PINS];
+    qemu_irq ioapic2_irq[IOAPIC_NUM_PINS];
 } GSIState;
 
 qemu_irq x86_allocate_cpu_irq(void);
diff --git a/hw/i386/acpi-common.c b/hw/i386/acpi-common.c
index 8a769654060e..f0689392a39f 100644
--- a/hw/i386/acpi-common.c
+++ b/hw/i386/acpi-common.c
@@ -103,6 +103,16 @@ void acpi_build_madt(GArray *table_data, BIOSLinker *linker,
     io_apic->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS);
     io_apic->interrupt = cpu_to_le32(0);
 
+    if (object_property_find(OBJECT(x86ms), "ioapic2")) {
+        AcpiMadtIoApic *io_apic2;
+        io_apic2 = acpi_data_push(table_data, sizeof *io_apic);
+        io_apic2->type = ACPI_APIC_IO;
+        io_apic2->length = sizeof(*io_apic);
+        io_apic2->io_apic_id = ACPI_BUILD_IOAPIC_ID + 1;
+        io_apic2->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS + 0x10000);
+        io_apic2->interrupt = cpu_to_le32(24);
+    }
+
     if (x86ms->apic_xrupt_override) {
         intsrcovr = acpi_data_push(table_data, sizeof *intsrcovr);
         intsrcovr->type   = ACPI_APIC_XRUPT_OVERRIDE;
diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
index 638e95c39e8c..15c3e078a4aa 100644
--- a/hw/i386/microvm.c
+++ b/hw/i386/microvm.c
@@ -99,7 +99,11 @@ static void microvm_gsi_handler(void *opaque, int n, int level)
 {
     GSIState *s = opaque;
 
-    qemu_set_irq(s->ioapic_irq[n], level);
+    if (n >= 24) {
+        qemu_set_irq(s->ioapic2_irq[n - 24], level);
+    } else {
+        qemu_set_irq(s->ioapic_irq[n], level);
+    }
 }
 
 static void create_gpex(MicrovmMachineState *mms)
@@ -157,6 +161,7 @@ static void microvm_devices_init(MicrovmMachineState *mms)
     ISABus *isa_bus;
     ISADevice *rtc_state;
     GSIState *gsi_state;
+    bool ioapic2 = false;
     int i;
 
     /* Core components */
@@ -165,8 +170,13 @@ static void microvm_devices_init(MicrovmMachineState *mms)
     if (mms->pic == ON_OFF_AUTO_ON || mms->pic == ON_OFF_AUTO_AUTO) {
         x86ms->gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
     } else {
+        int pins = GSI_NUM_PINS;
+        if (!kvm_ioapic_in_kernel() && x86_machine_is_acpi_enabled(x86ms)) {
+            ioapic2 = true;
+            pins += 24;
+        }
         x86ms->gsi = qemu_allocate_irqs(microvm_gsi_handler,
-                                        gsi_state, GSI_NUM_PINS);
+                                        gsi_state, pins);
     }
 
     isa_bus = isa_bus_new(NULL, get_system_memory(), get_system_io(),
@@ -175,6 +185,22 @@ static void microvm_devices_init(MicrovmMachineState *mms)
 
     ioapic_init_gsi(gsi_state, "machine");
 
+    if (ioapic2) {
+        DeviceState *dev;
+        SysBusDevice *d;
+        unsigned int i;
+
+        dev = qdev_new(TYPE_IOAPIC);
+        object_property_add_child(OBJECT(mms), "ioapic2", OBJECT(dev));
+        d = SYS_BUS_DEVICE(dev);
+        sysbus_realize_and_unref(d, &error_fatal);
+        sysbus_mmio_map(d, 0, IO_APIC_DEFAULT_ADDRESS + 0x10000);
+
+        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+            gsi_state->ioapic2_irq[i] = qdev_get_gpio_in(dev, i);
+        }
+    }
+
     kvmclock_create(true);
 
     mms->virtio_irq_base = 5;
-- 
2.27.0

[PATCH 4/4] microvm: reconfigure irqs if second ioapic is available

[Gerd Hoffmann]

Use GSI 16+ for PCIe (needs acpi_build_madt() tweak).
Use GSI 24+ (second ioapic) for virtio-mmio.
Use all irq lines of the second ioapic
and allow up to 24 virtio-mmio devices.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/i386/acpi-common.c | 2 +-
 hw/i386/microvm.c     | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/hw/i386/acpi-common.c b/hw/i386/acpi-common.c
index f0689392a39f..1653a8315248 100644
--- a/hw/i386/acpi-common.c
+++ b/hw/i386/acpi-common.c
@@ -122,7 +122,7 @@ void acpi_build_madt(GArray *table_data, BIOSLinker *linker,
         intsrcovr->flags  = cpu_to_le16(0); /* conforms to bus specifications */
     }
 
-    for (i = 1; i < 16; i++) {
+    for (i = 1; i < 24; i++) {
         if (!(x86ms->pci_irq_mask & (1 << i))) {
             /* No need for a INT source override structure. */
             continue;
diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
index 15c3e078a4aa..70bb8a4e3954 100644
--- a/hw/i386/microvm.c
+++ b/hw/i386/microvm.c
@@ -205,7 +205,11 @@ static void microvm_devices_init(MicrovmMachineState *mms)
 
     mms->virtio_irq_base = 5;
     mms->virtio_num_transports = 8;
-    if (x86_machine_is_acpi_enabled(x86ms)) {
+    if (ioapic2) {
+        mms->pcie_irq_base = 16;
+        mms->virtio_irq_base = 24;
+        mms->virtio_num_transports = 24;
+    } else if (x86_machine_is_acpi_enabled(x86ms)) {
         mms->pcie_irq_base = 12;
         mms->virtio_irq_base = 16;
     }
-- 
2.27.0

Re: [PATCH 0/4] RfC: microvm: add second ioapic

[Philippe Mathieu-Daudé]

On 10/16/20 1:43 PM, Gerd Hoffmann wrote:
> Add a second ioapic to microvm.  Gives us more IRQ lines we can
> use for virtio-mmio devices.  Bump number of possible virtio-mmio
> devices from 8 to 24.
> 
> Gerd Hoffmann (4):
>    microvm: make number of virtio transports runtime configurable
>    microvm: make pcie irq base runtime configurable
>    microvm: add second ioapic
>    microvm: reconfigure irqs if second ioapic is available

After looking at Laurent's m68k virt machine,
I wonder if it is possible to use the Goldfish-PIC
with the MicroVM instead (or another Goldfish machine
type).

Re: [PATCH 0/4] RfC: microvm: add second ioapic

[Gerd Hoffmann]

On Fri, Oct 16, 2020 at 03:16:20PM +0200, Philippe Mathieu-Daudé wrote:
> On 10/16/20 1:43 PM, Gerd Hoffmann wrote:
> > Add a second ioapic to microvm.  Gives us more IRQ lines we can
> > use for virtio-mmio devices.  Bump number of possible virtio-mmio
> > devices from 8 to 24.
> > 
> > Gerd Hoffmann (4):
> >    microvm: make number of virtio transports runtime configurable
> >    microvm: make pcie irq base runtime configurable
> >    microvm: add second ioapic
> >    microvm: reconfigure irqs if second ioapic is available
> 
> After looking at Laurent's m68k virt machine,
> I wonder if it is possible to use the Goldfish-PIC
> with the MicroVM instead (or another Goldfish machine
> type).

/me updates the WIP microvm blog post ;)

Problem with goldfish is that it tries to drag over stuff from the arm
world to x86.  Specifically device trees.  On arm this works reasonable
well meanwhile.  Firmware and bootloaders have support for using device
trees and passing them on in the boot chain, so the linux kernel has a
device tree which it can use to figure which hardware is present on the
system.

On x86 this doesn't work at all, so you'll end up building a custom
kernel for the goldfish platform.  A stock distro kernel is not going
to work.

I'm to driving microvm the opposite direction:  Add acpi support.  That
is the standard x86 way to do hardware discovery (for hardware you can't
detect in other ways like pci devices).  Which in turn allows us to drop
microvm-specific quirks (like adding virtio-mmio devices to the kernel
command line) when booting linux.

So, for microvm this is not going to happen.  But feel free to add a
goldfish machine type if you want play with that.  The microvm merge
also cleaned up the x86 code base which in turn should make this
relatively easy now.

take care,
  Gerd

Re: [PATCH 0/4] RfC: microvm: add second ioapic

[Philippe Mathieu-Daudé]

On 10/19/20 9:07 AM, Gerd Hoffmann wrote:
> On Fri, Oct 16, 2020 at 03:16:20PM +0200, Philippe Mathieu-Daudé wrote:
>> On 10/16/20 1:43 PM, Gerd Hoffmann wrote:
>>> Add a second ioapic to microvm.  Gives us more IRQ lines we can
>>> use for virtio-mmio devices.  Bump number of possible virtio-mmio
>>> devices from 8 to 24.
>>>
>>> Gerd Hoffmann (4):
>>>     microvm: make number of virtio transports runtime configurable
>>>     microvm: make pcie irq base runtime configurable
>>>     microvm: add second ioapic
>>>     microvm: reconfigure irqs if second ioapic is available
>>
>> After looking at Laurent's m68k virt machine,
>> I wonder if it is possible to use the Goldfish-PIC
>> with the MicroVM instead (or another Goldfish machine
>> type).
> 
> /me updates the WIP microvm blog post ;)
> 
> Problem with goldfish is that it tries to drag over stuff from the arm
> world to x86.  Specifically device trees.  On arm this works reasonable
> well meanwhile.  Firmware and bootloaders have support for using device
> trees and passing them on in the boot chain, so the linux kernel has a
> device tree which it can use to figure which hardware is present on the
> system.
> 
> On x86 this doesn't work at all, so you'll end up building a custom
> kernel for the goldfish platform.  A stock distro kernel is not going
> to work.
> 
> I'm to driving microvm the opposite direction:  Add acpi support.  That
> is the standard x86 way to do hardware discovery (for hardware you can't
> detect in other ways like pci devices).  Which in turn allows us to drop
> microvm-specific quirks (like adding virtio-mmio devices to the kernel
> command line) when booting linux.

My x86 knowledge is limited to asm, not to the architecture.
While this might be obvious for some, it helped me to better
understand, so thank you for the explanation :)

> So, for microvm this is not going to happen.  But feel free to add a
> goldfish machine type if you want play with that.  The microvm merge
> also cleaned up the x86 code base which in turn should make this
> relatively easy now.
> 
> take care,
>    Gerd
> 

Re: [PATCH 0/4] RfC: microvm: add second ioapic

[Gerd Hoffmann]

  Hi,

> /me updates the WIP microvm blog post ;)

Online now.
https://www.kraxel.org/blog/2020/10/qemu-microvm-acpi/

take care,
  Gerd

Re: [PATCH 4/4] microvm: reconfigure irqs if second ioapic is available

[Igor Mammedov]

On Fri, 16 Oct 2020 13:43:28 +0200
Gerd Hoffmann <kraxel@redhat.com> wrote:

> Use GSI 16+ for PCIe (needs acpi_build_madt() tweak).
> Use GSI 24+ (second ioapic) for virtio-mmio.
> Use all irq lines of the second ioapic
> and allow up to 24 virtio-mmio devices.

description is  missing answer to question why it's done.


> 
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> ---
>  hw/i386/acpi-common.c | 2 +-
>  hw/i386/microvm.c     | 6 +++++-
>  2 files changed, 6 insertions(+), 2 deletions(-)
> 
> diff --git a/hw/i386/acpi-common.c b/hw/i386/acpi-common.c
> index f0689392a39f..1653a8315248 100644
> --- a/hw/i386/acpi-common.c
> +++ b/hw/i386/acpi-common.c
> @@ -122,7 +122,7 @@ void acpi_build_madt(GArray *table_data, BIOSLinker *linker,
>          intsrcovr->flags  = cpu_to_le16(0); /* conforms to bus specifications */
>      }
>  
> -    for (i = 1; i < 16; i++) {
> +    for (i = 1; i < 24; i++) {
>          if (!(x86ms->pci_irq_mask & (1 << i))) {
>              /* No need for a INT source override structure. */
>              continue;
> diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
> index 15c3e078a4aa..70bb8a4e3954 100644
> --- a/hw/i386/microvm.c
> +++ b/hw/i386/microvm.c
> @@ -205,7 +205,11 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>  
>      mms->virtio_irq_base = 5;
>      mms->virtio_num_transports = 8;
> -    if (x86_machine_is_acpi_enabled(x86ms)) {
> +    if (ioapic2) {
> +        mms->pcie_irq_base = 16;
> +        mms->virtio_irq_base = 24;
> +        mms->virtio_num_transports = 24;
> +    } else if (x86_machine_is_acpi_enabled(x86ms)) {
>          mms->pcie_irq_base = 12;
>          mms->virtio_irq_base = 16;
>      }

Re: [PATCH 3/4] microvm: add second ioapic

[Igor Mammedov]

On Fri, 16 Oct 2020 13:43:27 +0200
Gerd Hoffmann <kraxel@redhat.com> wrote:

> Add more IRQ lines.  Depends on ACPI.
> Also enable this only with userspace ioapic,
> not sure whenever the kernel can handle two ioapics.
> 
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

For ACPI part:
Reviewed-by: Igor Mammedov <imammedo@redhat.com>

> ---
>  include/hw/i386/ioapic_internal.h |  2 +-
>  include/hw/i386/x86.h             |  1 +
>  hw/i386/acpi-common.c             | 10 ++++++++++
>  hw/i386/microvm.c                 | 30 ++++++++++++++++++++++++++++--
>  4 files changed, 40 insertions(+), 3 deletions(-)
> 
> diff --git a/include/hw/i386/ioapic_internal.h b/include/hw/i386/ioapic_internal.h
> index 0ac9e2400d6b..4cebd2e32c9f 100644
> --- a/include/hw/i386/ioapic_internal.h
> +++ b/include/hw/i386/ioapic_internal.h
> @@ -27,7 +27,7 @@
>  #include "qemu/notify.h"
>  #include "qom/object.h"
>  
> -#define MAX_IOAPICS                     1
> +#define MAX_IOAPICS                     2
>  
>  #define IOAPIC_LVT_DEST_SHIFT           56
>  #define IOAPIC_LVT_DEST_IDX_SHIFT       48
> diff --git a/include/hw/i386/x86.h b/include/hw/i386/x86.h
> index bfa9cb2a258b..6da57033a875 100644
> --- a/include/hw/i386/x86.h
> +++ b/include/hw/i386/x86.h
> @@ -120,6 +120,7 @@ bool x86_machine_is_acpi_enabled(const X86MachineState *x86ms);
>  typedef struct GSIState {
>      qemu_irq i8259_irq[ISA_NUM_IRQS];
>      qemu_irq ioapic_irq[IOAPIC_NUM_PINS];
> +    qemu_irq ioapic2_irq[IOAPIC_NUM_PINS];
>  } GSIState;
>  
>  qemu_irq x86_allocate_cpu_irq(void);
> diff --git a/hw/i386/acpi-common.c b/hw/i386/acpi-common.c
> index 8a769654060e..f0689392a39f 100644
> --- a/hw/i386/acpi-common.c
> +++ b/hw/i386/acpi-common.c
> @@ -103,6 +103,16 @@ void acpi_build_madt(GArray *table_data, BIOSLinker *linker,
>      io_apic->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS);
>      io_apic->interrupt = cpu_to_le32(0);
>  
> +    if (object_property_find(OBJECT(x86ms), "ioapic2")) {
> +        AcpiMadtIoApic *io_apic2;
> +        io_apic2 = acpi_data_push(table_data, sizeof *io_apic);
> +        io_apic2->type = ACPI_APIC_IO;
> +        io_apic2->length = sizeof(*io_apic);
> +        io_apic2->io_apic_id = ACPI_BUILD_IOAPIC_ID + 1;
> +        io_apic2->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS + 0x10000);
> +        io_apic2->interrupt = cpu_to_le32(24);
> +    }
> +
>      if (x86ms->apic_xrupt_override) {
>          intsrcovr = acpi_data_push(table_data, sizeof *intsrcovr);
>          intsrcovr->type   = ACPI_APIC_XRUPT_OVERRIDE;
> diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
> index 638e95c39e8c..15c3e078a4aa 100644
> --- a/hw/i386/microvm.c
> +++ b/hw/i386/microvm.c
> @@ -99,7 +99,11 @@ static void microvm_gsi_handler(void *opaque, int n, int level)
>  {
>      GSIState *s = opaque;
>  
> -    qemu_set_irq(s->ioapic_irq[n], level);
> +    if (n >= 24) {
> +        qemu_set_irq(s->ioapic2_irq[n - 24], level);
> +    } else {
> +        qemu_set_irq(s->ioapic_irq[n], level);
> +    }
>  }
>  
>  static void create_gpex(MicrovmMachineState *mms)
> @@ -157,6 +161,7 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>      ISABus *isa_bus;
>      ISADevice *rtc_state;
>      GSIState *gsi_state;
> +    bool ioapic2 = false;
>      int i;
>  
>      /* Core components */
> @@ -165,8 +170,13 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>      if (mms->pic == ON_OFF_AUTO_ON || mms->pic == ON_OFF_AUTO_AUTO) {
>          x86ms->gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
>      } else {
> +        int pins = GSI_NUM_PINS;
> +        if (!kvm_ioapic_in_kernel() && x86_machine_is_acpi_enabled(x86ms)) {
> +            ioapic2 = true;
> +            pins += 24;
> +        }
>          x86ms->gsi = qemu_allocate_irqs(microvm_gsi_handler,
> -                                        gsi_state, GSI_NUM_PINS);
> +                                        gsi_state, pins);
>      }
>  
>      isa_bus = isa_bus_new(NULL, get_system_memory(), get_system_io(),
> @@ -175,6 +185,22 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>  
>      ioapic_init_gsi(gsi_state, "machine");
>  
> +    if (ioapic2) {
> +        DeviceState *dev;
> +        SysBusDevice *d;
> +        unsigned int i;
> +
> +        dev = qdev_new(TYPE_IOAPIC);
> +        object_property_add_child(OBJECT(mms), "ioapic2", OBJECT(dev));
> +        d = SYS_BUS_DEVICE(dev);
> +        sysbus_realize_and_unref(d, &error_fatal);
> +        sysbus_mmio_map(d, 0, IO_APIC_DEFAULT_ADDRESS + 0x10000);
> +
> +        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
> +            gsi_state->ioapic2_irq[i] = qdev_get_gpio_in(dev, i);
> +        }
> +    }
> +
>      kvmclock_create(true);
>  
>      mms->virtio_irq_base = 5;

Re: [PATCH 2/4] microvm: make pcie irq base runtime configurable

[Igor Mammedov]

On Fri, 16 Oct 2020 13:43:26 +0200
Gerd Hoffmann <kraxel@redhat.com> wrote:

> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

Reviewed-by: Igor Mammedov <imammedo@redhat.com>

> ---
>  include/hw/i386/microvm.h |  2 +-
>  hw/i386/microvm.c         | 11 ++++++-----
>  2 files changed, 7 insertions(+), 6 deletions(-)
> 
> diff --git a/include/hw/i386/microvm.h b/include/hw/i386/microvm.h
> index 0154ad5bd707..ede9625756b8 100644
> --- a/include/hw/i386/microvm.h
> +++ b/include/hw/i386/microvm.h
> @@ -63,7 +63,6 @@
>  #define PCIE_MMIO_SIZE        0x20000000
>  #define PCIE_ECAM_BASE        0xe0000000
>  #define PCIE_ECAM_SIZE        0x10000000
> -#define PCIE_IRQ_BASE         12
>  
>  /* Machine type options */
>  #define MICROVM_MACHINE_PIT                 "pit"
> @@ -93,6 +92,7 @@ struct MicrovmMachineState {
>      bool auto_kernel_cmdline;
>  
>      /* Machine state */
> +    uint32_t pcie_irq_base;
>      uint32_t virtio_irq_base;
>      uint32_t virtio_num_transports;
>      bool kernel_cmdline_fixed;
> diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
> index eaf5da31f7e1..638e95c39e8c 100644
> --- a/hw/i386/microvm.c
> +++ b/hw/i386/microvm.c
> @@ -180,6 +180,7 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>      mms->virtio_irq_base = 5;
>      mms->virtio_num_transports = 8;
>      if (x86_machine_is_acpi_enabled(x86ms)) {
> +        mms->pcie_irq_base = 12;
>          mms->virtio_irq_base = 16;
>      }
>  
> @@ -213,12 +214,12 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>          mms->gpex.mmio32.size = PCIE_MMIO_SIZE;
>          mms->gpex.ecam.base   = PCIE_ECAM_BASE;
>          mms->gpex.ecam.size   = PCIE_ECAM_SIZE;
> -        mms->gpex.irq         = PCIE_IRQ_BASE;
> +        mms->gpex.irq         = mms->pcie_irq_base;
>          create_gpex(mms);
> -        x86ms->pci_irq_mask = ((1 << (PCIE_IRQ_BASE + 0)) |
> -                               (1 << (PCIE_IRQ_BASE + 1)) |
> -                               (1 << (PCIE_IRQ_BASE + 2)) |
> -                               (1 << (PCIE_IRQ_BASE + 3)));
> +        x86ms->pci_irq_mask = ((1 << (mms->pcie_irq_base + 0)) |
> +                               (1 << (mms->pcie_irq_base + 1)) |
> +                               (1 << (mms->pcie_irq_base + 2)) |
> +                               (1 << (mms->pcie_irq_base + 3)));
>      } else {
>          x86ms->pci_irq_mask = 0;
>      }

Re: [PATCH 1/4] microvm: make number of virtio transports runtime configurable

[Igor Mammedov]

On Fri, 16 Oct 2020 13:43:25 +0200
Gerd Hoffmann <kraxel@redhat.com> wrote:

> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> ---
>  include/hw/i386/microvm.h | 2 +-
>  hw/i386/microvm.c         | 9 +++++++--
>  2 files changed, 8 insertions(+), 3 deletions(-)
> 
> diff --git a/include/hw/i386/microvm.h b/include/hw/i386/microvm.h
> index 91b064575d55..0154ad5bd707 100644
> --- a/include/hw/i386/microvm.h
> +++ b/include/hw/i386/microvm.h
> @@ -52,7 +52,6 @@
>  
>  /* Platform virtio definitions */
>  #define VIRTIO_MMIO_BASE      0xfeb00000
> -#define VIRTIO_NUM_TRANSPORTS 8
>  #define VIRTIO_CMDLINE_MAXLEN 64
>  
>  #define GED_MMIO_BASE         0xfea00000
> @@ -95,6 +94,7 @@ struct MicrovmMachineState {
>  
>      /* Machine state */
>      uint32_t virtio_irq_base;
> +    uint32_t virtio_num_transports;
>      bool kernel_cmdline_fixed;
>      Notifier machine_done;
>      Notifier powerdown_req;
> diff --git a/hw/i386/microvm.c b/hw/i386/microvm.c
> index 9dd74458aca4..eaf5da31f7e1 100644
> --- a/hw/i386/microvm.c
> +++ b/hw/i386/microvm.c
> @@ -177,8 +177,13 @@ static void microvm_devices_init(MicrovmMachineState *mms)
>  
>      kvmclock_create(true);
>  
> -    mms->virtio_irq_base = x86_machine_is_acpi_enabled(x86ms) ? 16 : 5;
> -    for (i = 0; i < VIRTIO_NUM_TRANSPORTS; i++) {
> +    mms->virtio_irq_base = 5;
> +    mms->virtio_num_transports = 8;
> +    if (x86_machine_is_acpi_enabled(x86ms)) {
> +        mms->virtio_irq_base = 16;
> +    }

can we unify and use the same base in both cases?

> +
> +    for (i = 0; i < mms->virtio_num_transports; i++) {
>          sysbus_create_simple("virtio-mmio",
>                               VIRTIO_MMIO_BASE + i * 512,
>                               x86ms->gsi[mms->virtio_irq_base + i]);

Re: [PATCH 1/4] microvm: make number of virtio transports runtime configurable

[Gerd Hoffmann]

  Hi,

> > +    mms->virtio_irq_base = 5;
> > +    mms->virtio_num_transports = 8;
> > +    if (x86_machine_is_acpi_enabled(x86ms)) {
> > +        mms->virtio_irq_base = 16;
> > +    }
> 
> can we unify and use the same base in both cases?

Well, without ACPI the IO-APIC detection seems to not work reliable,
which in turn makes using IRQs 16+ problematic because the classic PIC
can't handle those ...

take care,
  Gerd

Re: [PATCH 4/4] microvm: reconfigure irqs if second ioapic is available

[Gerd Hoffmann]

On Fri, Oct 23, 2020 at 08:52:09PM +0200, Igor Mammedov wrote:
> On Fri, 16 Oct 2020 13:43:28 +0200
> Gerd Hoffmann <kraxel@redhat.com> wrote:
> 
> > Use GSI 16+ for PCIe (needs acpi_build_madt() tweak).
> > Use GSI 24+ (second ioapic) for virtio-mmio.
> > Use all irq lines of the second ioapic
> > and allow up to 24 virtio-mmio devices.
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> description is  missing answer to question why it's done.

It's right there.  We can support more virtio devices then ;)

take care,
  Gerd