[PATCH v3 0/6] Add RISC-V KVM AIA Support

[PATCH v3 0/6] Add RISC-V KVM AIA Support

[Yong-Xuan Wang]

This series adds support for KVM AIA in RISC-V architecture.

In order to test these patches, we require Linux with KVM AIA support which can
be found in the qemu_kvm_aia branch at https://github.com/yong-xuan/linux.git
This kernel branch is based on the riscv_aia_v1 branch available at
https://github.com/avpatel/linux.git, and it also includes two additional
patches that fix a KVM AIA bug and reply to the query of KVM_CAP_IRQCHIP.

---
v3:
- fix typo
- tag the linux-header patch as placeholder

v2:
- rebase to riscv-to-apply.next
- update the linux header by the scripts/update-linux-headers.sh

Yong-Xuan Wang (6):
  update-linux-headers: sync-up header with Linux for KVM AIA support
    placeholder
  target/riscv: support the AIA device emulation with KVM enabled
  target/riscv: check the in-kernel irqchip support
  target/riscv: Create an KVM AIA irqchip
  target/riscv: update APLIC and IMSIC to support KVM AIA
  target/riscv: select KVM AIA in riscv virt machine

 hw/intc/riscv_aplic.c     |  19 +++-
 hw/intc/riscv_imsic.c     |  16 ++-
 hw/riscv/virt.c           | 209 +++++++++++++++++++++-----------------
 linux-headers/linux/kvm.h |   2 +
 target/riscv/kvm.c        |  96 ++++++++++++++++-
 target/riscv/kvm_riscv.h  |  36 +++++++
 6 files changed, 274 insertions(+), 104 deletions(-)

-- 
2.17.1

[PATCH v3 1/6] update-linux-headers: sync-up header with Linux for KVM AIA support placeholder

[Yong-Xuan Wang]

Sync-up Linux header to get latest KVM RISC-V headers having AIA support.

Note: This is a placeholder commit and could be replaced when all referenced Linux patchsets are mainlined.

The linux-headers changes are from 2 different patchsets.
[1] https://lore.kernel.org/lkml/20230404153452.2405681-1-apatel@ventanamicro.com/
[2] https://www.spinics.net/lists/kernel/msg4791872.html

Currently, patchset 1 is already merged into mainline kernel in v6.4-rc1 and patchset 2 is not.

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 linux-headers/linux/kvm.h |  2 ++
 target/riscv/kvm_riscv.h  | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 599de3c6e3..a9a4f5791d 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1434,6 +1434,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_XIVE		KVM_DEV_TYPE_XIVE
 	KVM_DEV_TYPE_ARM_PV_TIME,
 #define KVM_DEV_TYPE_ARM_PV_TIME	KVM_DEV_TYPE_ARM_PV_TIME
+	KVM_DEV_TYPE_RISCV_AIA,
+#define KVM_DEV_TYPE_RISCV_AIA		KVM_DEV_TYPE_RISCV_AIA
 	KVM_DEV_TYPE_MAX,
 };
 
diff --git a/target/riscv/kvm_riscv.h b/target/riscv/kvm_riscv.h
index ed281bdce0..606968a4b7 100644
--- a/target/riscv/kvm_riscv.h
+++ b/target/riscv/kvm_riscv.h
@@ -22,4 +22,37 @@
 void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
 void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
 
+#define KVM_DEV_RISCV_AIA_GRP_CONFIG            0
+#define KVM_DEV_RISCV_AIA_CONFIG_MODE           0
+#define KVM_DEV_RISCV_AIA_CONFIG_IDS            1
+#define KVM_DEV_RISCV_AIA_CONFIG_SRCS           2
+#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS     3
+#define KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT    4
+#define KVM_DEV_RISCV_AIA_CONFIG_HART_BITS      5
+#define KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS     6
+#define KVM_DEV_RISCV_AIA_MODE_EMUL             0
+#define KVM_DEV_RISCV_AIA_MODE_HWACCEL          1
+#define KVM_DEV_RISCV_AIA_MODE_AUTO             2
+#define KVM_DEV_RISCV_AIA_IDS_MIN               63
+#define KVM_DEV_RISCV_AIA_IDS_MAX               2048
+#define KVM_DEV_RISCV_AIA_SRCS_MAX              1024
+#define KVM_DEV_RISCV_AIA_GROUP_BITS_MAX        8
+#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN       24
+#define KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX       56
+#define KVM_DEV_RISCV_AIA_HART_BITS_MAX         16
+#define KVM_DEV_RISCV_AIA_GUEST_BITS_MAX        8
+
+#define KVM_DEV_RISCV_AIA_GRP_ADDR              1
+#define KVM_DEV_RISCV_AIA_ADDR_APLIC            0
+#define KVM_DEV_RISCV_AIA_ADDR_IMSIC(__vcpu)    (1 + (__vcpu))
+#define KVM_DEV_RISCV_AIA_ADDR_MAX              \
+        (1 + KVM_DEV_RISCV_APLIC_MAX_HARTS)
+
+#define KVM_DEV_RISCV_AIA_GRP_CTRL              2
+#define KVM_DEV_RISCV_AIA_CTRL_INIT             0
+
+#define KVM_DEV_RISCV_AIA_GRP_APLIC             3
+
+#define KVM_DEV_RISCV_AIA_GRP_IMSIC             4
+
 #endif
-- 
2.17.1

[PATCH v3 2/6] target/riscv: support the AIA device emulation with KVM enabled

[Yong-Xuan Wang]

Remove M mode AIA devices when using KVM acceleration

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 hw/riscv/virt.c | 199 +++++++++++++++++++++++++-----------------------
 1 file changed, 105 insertions(+), 94 deletions(-)

diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index 4e3efbee16..18b94888ab 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -531,52 +531,54 @@ static void create_fdt_imsic(RISCVVirtState *s, const MemMapEntry *memmap,
     imsic_cells = g_new0(uint32_t, ms->smp.cpus * 2);
     imsic_regs = g_new0(uint32_t, socket_count * 4);
 
-    /* M-level IMSIC node */
-    for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
-        imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
-        imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
-    }
-    imsic_max_hart_per_socket = 0;
-    for (socket = 0; socket < socket_count; socket++) {
-        imsic_addr = memmap[VIRT_IMSIC_M].base +
-                     socket * VIRT_IMSIC_GROUP_MAX_SIZE;
-        imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
-        imsic_regs[socket * 4 + 0] = 0;
-        imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
-        imsic_regs[socket * 4 + 2] = 0;
-        imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
-        if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
-            imsic_max_hart_per_socket = s->soc[socket].num_harts;
+    if (!kvm_enabled()) {
+        /* M-level IMSIC node */
+        for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
+            imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
+            imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
         }
-    }
-    imsic_name = g_strdup_printf("/soc/imsics@%lx",
-        (unsigned long)memmap[VIRT_IMSIC_M].base);
-    qemu_fdt_add_subnode(ms->fdt, imsic_name);
-    qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
-        "riscv,imsics");
-    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
-        FDT_IMSIC_INT_CELLS);
-    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
-        NULL, 0);
-    qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
-        NULL, 0);
-    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
-        imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
-    qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
-        socket_count * sizeof(uint32_t) * 4);
-    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
-        VIRT_IRQCHIP_NUM_MSIS);
-    if (socket_count > 1) {
-        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
-            imsic_num_bits(imsic_max_hart_per_socket));
-        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
-            imsic_num_bits(socket_count));
-        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-shift",
-            IMSIC_MMIO_GROUP_MIN_SHIFT);
-    }
-    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
+        imsic_max_hart_per_socket = 0;
+        for (socket = 0; socket < socket_count; socket++) {
+            imsic_addr = memmap[VIRT_IMSIC_M].base +
+                         socket * VIRT_IMSIC_GROUP_MAX_SIZE;
+            imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
+            imsic_regs[socket * 4 + 0] = 0;
+            imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
+            imsic_regs[socket * 4 + 2] = 0;
+            imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
+            if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
+                imsic_max_hart_per_socket = s->soc[socket].num_harts;
+            }
+        }
+        imsic_name = g_strdup_printf("/soc/imsics@%lx",
+            (unsigned long)memmap[VIRT_IMSIC_M].base);
+        qemu_fdt_add_subnode(ms->fdt, imsic_name);
+        qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
+            "riscv,imsics");
+        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
+            FDT_IMSIC_INT_CELLS);
+        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
+            NULL, 0);
+        qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
+            NULL, 0);
+        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
+            imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
+        qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
+            socket_count * sizeof(uint32_t) * 4);
+        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
+            VIRT_IRQCHIP_NUM_MSIS);
+        if (socket_count > 1) {
+            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
+                imsic_num_bits(imsic_max_hart_per_socket));
+            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
+                imsic_num_bits(socket_count));
+            qemu_fdt_setprop_cell(ms->fdt, imsic_name,
+                "riscv,group-index-shift", IMSIC_MMIO_GROUP_MIN_SHIFT);
+        }
+        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
 
-    g_free(imsic_name);
+        g_free(imsic_name);
+    }
 
     /* S-level IMSIC node */
     for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
@@ -653,37 +655,40 @@ static void create_fdt_socket_aplic(RISCVVirtState *s,
     aplic_s_phandle = (*phandle)++;
     aplic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 2);
 
-    /* M-level APLIC node */
-    for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
-        aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
-        aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
-    }
-    aplic_addr = memmap[VIRT_APLIC_M].base +
-                 (memmap[VIRT_APLIC_M].size * socket);
-    aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
-    qemu_fdt_add_subnode(ms->fdt, aplic_name);
-    qemu_fdt_setprop_string(ms->fdt, aplic_name, "compatible", "riscv,aplic");
-    qemu_fdt_setprop_cell(ms->fdt, aplic_name,
-        "#interrupt-cells", FDT_APLIC_INT_CELLS);
-    qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
-    if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
-        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
-            aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
-    } else {
-        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
-            msi_m_phandle);
+    if (!kvm_enabled()) {
+        /* M-level APLIC node */
+        for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
+            aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
+            aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
+        }
+        aplic_addr = memmap[VIRT_APLIC_M].base +
+                     (memmap[VIRT_APLIC_M].size * socket);
+        aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
+        qemu_fdt_add_subnode(ms->fdt, aplic_name);
+        qemu_fdt_setprop_string(ms->fdt, aplic_name,
+            "compatible", "riscv,aplic");
+        qemu_fdt_setprop_cell(ms->fdt, aplic_name,
+            "#interrupt-cells", FDT_APLIC_INT_CELLS);
+        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
+        if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
+            qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
+                aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
+        } else {
+            qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
+                msi_m_phandle);
+        }
+        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
+            0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
+        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
+            VIRT_IRQCHIP_NUM_SOURCES);
+        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
+            aplic_s_phandle);
+        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
+            aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
+        riscv_socket_fdt_write_id(ms, aplic_name, socket);
+        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
+        g_free(aplic_name);
     }
-    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
-        0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
-    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
-        VIRT_IRQCHIP_NUM_SOURCES);
-    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
-        aplic_s_phandle);
-    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
-        aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
-    riscv_socket_fdt_write_id(ms, aplic_name, socket);
-    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
-    g_free(aplic_name);
 
     /* S-level APLIC node */
     for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
@@ -1162,16 +1167,20 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
     int i;
     hwaddr addr;
     uint32_t guest_bits;
-    DeviceState *aplic_m;
+    DeviceState *aplic_s = NULL;
+    DeviceState *aplic_m = NULL;
     bool msimode = (aia_type == VIRT_AIA_TYPE_APLIC_IMSIC) ? true : false;
 
     if (msimode) {
-        /* Per-socket M-level IMSICs */
-        addr = memmap[VIRT_IMSIC_M].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE;
-        for (i = 0; i < hart_count; i++) {
-            riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
-                               base_hartid + i, true, 1,
-                               VIRT_IRQCHIP_NUM_MSIS);
+        if (!kvm_enabled()) {
+            /* Per-socket M-level IMSICs */
+            addr = memmap[VIRT_IMSIC_M].base +
+                   socket * VIRT_IMSIC_GROUP_MAX_SIZE;
+            for (i = 0; i < hart_count; i++) {
+                riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
+                        base_hartid + i, true, 1,
+                        VIRT_IRQCHIP_NUM_MSIS);
+            }
         }
 
         /* Per-socket S-level IMSICs */
@@ -1184,19 +1193,21 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
         }
     }
 
-    /* Per-socket M-level APLIC */
-    aplic_m = riscv_aplic_create(
-        memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
-        memmap[VIRT_APLIC_M].size,
-        (msimode) ? 0 : base_hartid,
-        (msimode) ? 0 : hart_count,
-        VIRT_IRQCHIP_NUM_SOURCES,
-        VIRT_IRQCHIP_NUM_PRIO_BITS,
-        msimode, true, NULL);
-
-    if (aplic_m) {
+    if (!kvm_enabled()) {
+        /* Per-socket M-level APLIC */
+        aplic_m = riscv_aplic_create(
+                memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
+                memmap[VIRT_APLIC_M].size,
+                (msimode) ? 0 : base_hartid,
+                (msimode) ? 0 : hart_count,
+                VIRT_IRQCHIP_NUM_SOURCES,
+                VIRT_IRQCHIP_NUM_PRIO_BITS,
+                msimode, true, NULL);
+    }
+
+    if (aplic_m || kvm_enabled()) {
         /* Per-socket S-level APLIC */
-        riscv_aplic_create(
+        aplic_s = riscv_aplic_create(
             memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
             memmap[VIRT_APLIC_S].size,
             (msimode) ? 0 : base_hartid,
@@ -1206,7 +1217,7 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
             msimode, false, aplic_m);
     }
 
-    return aplic_m;
+    return kvm_enabled() ? aplic_s : aplic_m;
 }
 
 static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)
-- 
2.17.1

Re: [PATCH v3 2/6] target/riscv: support the AIA device emulation with KVM enabled

[Daniel Henrique Barboza]

On 5/26/23 03:25, Yong-Xuan Wang wrote:
> Remove M mode AIA devices when using KVM acceleration
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---
>   hw/riscv/virt.c | 199 +++++++++++++++++++++++++-----------------------
>   1 file changed, 105 insertions(+), 94 deletions(-)
> 
> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> index 4e3efbee16..18b94888ab 100644
> --- a/hw/riscv/virt.c
> +++ b/hw/riscv/virt.c
> @@ -531,52 +531,54 @@ static void create_fdt_imsic(RISCVVirtState *s, const MemMapEntry *memmap,
>       imsic_cells = g_new0(uint32_t, ms->smp.cpus * 2);
>       imsic_regs = g_new0(uint32_t, socket_count * 4);
>   
> -    /* M-level IMSIC node */
> -    for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> -        imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> -        imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> -    }
> -    imsic_max_hart_per_socket = 0;
> -    for (socket = 0; socket < socket_count; socket++) {
> -        imsic_addr = memmap[VIRT_IMSIC_M].base +
> -                     socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> -        imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
> -        imsic_regs[socket * 4 + 0] = 0;
> -        imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
> -        imsic_regs[socket * 4 + 2] = 0;
> -        imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
> -        if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
> -            imsic_max_hart_per_socket = s->soc[socket].num_harts;
> +    if (!kvm_enabled()) {
> +        /* M-level IMSIC node */
> +        for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> +            imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> +            imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
>           }
> -    }
> -    imsic_name = g_strdup_printf("/soc/imsics@%lx",
> -        (unsigned long)memmap[VIRT_IMSIC_M].base);
> -    qemu_fdt_add_subnode(ms->fdt, imsic_name);
> -    qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
> -        "riscv,imsics");
> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
> -        FDT_IMSIC_INT_CELLS);
> -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
> -        NULL, 0);
> -    qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
> -        NULL, 0);
> -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
> -        imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
> -    qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
> -        socket_count * sizeof(uint32_t) * 4);
> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
> -        VIRT_IRQCHIP_NUM_MSIS);
> -    if (socket_count > 1) {
> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
> -            imsic_num_bits(imsic_max_hart_per_socket));
> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
> -            imsic_num_bits(socket_count));
> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-shift",
> -            IMSIC_MMIO_GROUP_MIN_SHIFT);
> -    }
> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
> +        imsic_max_hart_per_socket = 0;
> +        for (socket = 0; socket < socket_count; socket++) {
> +            imsic_addr = memmap[VIRT_IMSIC_M].base +
> +                         socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> +            imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
> +            imsic_regs[socket * 4 + 0] = 0;
> +            imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
> +            imsic_regs[socket * 4 + 2] = 0;
> +            imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
> +            if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
> +                imsic_max_hart_per_socket = s->soc[socket].num_harts;
> +            }
> +        }
> +        imsic_name = g_strdup_printf("/soc/imsics@%lx",
> +            (unsigned long)memmap[VIRT_IMSIC_M].base);
> +        qemu_fdt_add_subnode(ms->fdt, imsic_name);
> +        qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
> +            "riscv,imsics");
> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
> +            FDT_IMSIC_INT_CELLS);
> +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
> +            NULL, 0);
> +        qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
> +            NULL, 0);
> +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
> +            imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
> +        qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
> +            socket_count * sizeof(uint32_t) * 4);
> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
> +            VIRT_IRQCHIP_NUM_MSIS);
> +        if (socket_count > 1) {
> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
> +                imsic_num_bits(imsic_max_hart_per_socket));
> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
> +                imsic_num_bits(socket_count));
> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name,
> +                "riscv,group-index-shift", IMSIC_MMIO_GROUP_MIN_SHIFT);
> +        }
> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
>   
> -    g_free(imsic_name);
> +        g_free(imsic_name);
> +    }
>   
>       /* S-level IMSIC node */
>       for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> @@ -653,37 +655,40 @@ static void create_fdt_socket_aplic(RISCVVirtState *s,
>       aplic_s_phandle = (*phandle)++;
>       aplic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 2);
>   
> -    /* M-level APLIC node */
> -    for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> -        aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> -        aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> -    }
> -    aplic_addr = memmap[VIRT_APLIC_M].base +
> -                 (memmap[VIRT_APLIC_M].size * socket);
> -    aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
> -    qemu_fdt_add_subnode(ms->fdt, aplic_name);
> -    qemu_fdt_setprop_string(ms->fdt, aplic_name, "compatible", "riscv,aplic");
> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name,
> -        "#interrupt-cells", FDT_APLIC_INT_CELLS);
> -    qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
> -    if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
> -        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
> -            aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
> -    } else {
> -        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
> -            msi_m_phandle);
> +    if (!kvm_enabled()) {
> +        /* M-level APLIC node */
> +        for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> +            aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> +            aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> +        }
> +        aplic_addr = memmap[VIRT_APLIC_M].base +
> +                     (memmap[VIRT_APLIC_M].size * socket);
> +        aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
> +        qemu_fdt_add_subnode(ms->fdt, aplic_name);
> +        qemu_fdt_setprop_string(ms->fdt, aplic_name,
> +            "compatible", "riscv,aplic");
> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name,
> +            "#interrupt-cells", FDT_APLIC_INT_CELLS);
> +        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
> +        if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
> +            qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
> +                aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
> +        } else {
> +            qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
> +                msi_m_phandle);
> +        }
> +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
> +            0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
> +            VIRT_IRQCHIP_NUM_SOURCES);
> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
> +            aplic_s_phandle);
> +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
> +            aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
> +        riscv_socket_fdt_write_id(ms, aplic_name, socket);
> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
> +        g_free(aplic_name);
>       }
> -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
> -        0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
> -        VIRT_IRQCHIP_NUM_SOURCES);
> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
> -        aplic_s_phandle);
> -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
> -        aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
> -    riscv_socket_fdt_write_id(ms, aplic_name, socket);
> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
> -    g_free(aplic_name);
>   
>       /* S-level APLIC node */
>       for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> @@ -1162,16 +1167,20 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>       int i;
>       hwaddr addr;
>       uint32_t guest_bits;
> -    DeviceState *aplic_m;
> +    DeviceState *aplic_s = NULL;
> +    DeviceState *aplic_m = NULL;
>       bool msimode = (aia_type == VIRT_AIA_TYPE_APLIC_IMSIC) ? true : false;
>   
>       if (msimode) {
> -        /* Per-socket M-level IMSICs */
> -        addr = memmap[VIRT_IMSIC_M].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> -        for (i = 0; i < hart_count; i++) {
> -            riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
> -                               base_hartid + i, true, 1,
> -                               VIRT_IRQCHIP_NUM_MSIS);
> +        if (!kvm_enabled()) {
> +            /* Per-socket M-level IMSICs */
> +            addr = memmap[VIRT_IMSIC_M].base +
> +                   socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> +            for (i = 0; i < hart_count; i++) {
> +                riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
> +                        base_hartid + i, true, 1,
> +                        VIRT_IRQCHIP_NUM_MSIS);
> +            }
>           }
>   
>           /* Per-socket S-level IMSICs */
> @@ -1184,19 +1193,21 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>           }
>       }
>   
> -    /* Per-socket M-level APLIC */
> -    aplic_m = riscv_aplic_create(
> -        memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
> -        memmap[VIRT_APLIC_M].size,
> -        (msimode) ? 0 : base_hartid,
> -        (msimode) ? 0 : hart_count,
> -        VIRT_IRQCHIP_NUM_SOURCES,
> -        VIRT_IRQCHIP_NUM_PRIO_BITS,
> -        msimode, true, NULL);
> -
> -    if (aplic_m) {
> +    if (!kvm_enabled()) {
> +        /* Per-socket M-level APLIC */
> +        aplic_m = riscv_aplic_create(
> +                memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
> +                memmap[VIRT_APLIC_M].size,
> +                (msimode) ? 0 : base_hartid,
> +                (msimode) ? 0 : hart_count,
> +                VIRT_IRQCHIP_NUM_SOURCES,
> +                VIRT_IRQCHIP_NUM_PRIO_BITS,
> +                msimode, true, NULL);
> +    }
> +
> +    if (aplic_m || kvm_enabled()) {

I am struggling to understand why we're checking for "aplic_m" here given that
riscv_aplic_create() will either returning something not NULL or error out via
&error_fatal.

This behavior predates this patch though.



Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

>           /* Per-socket S-level APLIC */
> -        riscv_aplic_create(
> +        aplic_s = riscv_aplic_create(
>               memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
>               memmap[VIRT_APLIC_S].size,
>               (msimode) ? 0 : base_hartid,
> @@ -1206,7 +1217,7 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>               msimode, false, aplic_m);
>       }
>   
> -    return aplic_m;
> +    return kvm_enabled() ? aplic_s : aplic_m;
>   }
>   
>   static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)

Re: [PATCH v3 2/6] target/riscv: support the AIA device emulation with KVM enabled

[Yong-Xuan Wang]

Hi Daniel,

I think this checking can be removed too. Would you send a patch to
fix it? Or I can remove it in this patch.

Regards,
Yong-Xuan

On Tue, Jun 6, 2023 at 2:45 AM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
>
>
> On 5/26/23 03:25, Yong-Xuan Wang wrote:
> > Remove M mode AIA devices when using KVM acceleration
> >
> > Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> > Reviewed-by: Jim Shu <jim.shu@sifive.com>
> > ---
> >   hw/riscv/virt.c | 199 +++++++++++++++++++++++++-----------------------
> >   1 file changed, 105 insertions(+), 94 deletions(-)
> >
> > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> > index 4e3efbee16..18b94888ab 100644
> > --- a/hw/riscv/virt.c
> > +++ b/hw/riscv/virt.c
> > @@ -531,52 +531,54 @@ static void create_fdt_imsic(RISCVVirtState *s, const MemMapEntry *memmap,
> >       imsic_cells = g_new0(uint32_t, ms->smp.cpus * 2);
> >       imsic_regs = g_new0(uint32_t, socket_count * 4);
> >
> > -    /* M-level IMSIC node */
> > -    for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> > -        imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> > -        imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> > -    }
> > -    imsic_max_hart_per_socket = 0;
> > -    for (socket = 0; socket < socket_count; socket++) {
> > -        imsic_addr = memmap[VIRT_IMSIC_M].base +
> > -                     socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> > -        imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
> > -        imsic_regs[socket * 4 + 0] = 0;
> > -        imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
> > -        imsic_regs[socket * 4 + 2] = 0;
> > -        imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
> > -        if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
> > -            imsic_max_hart_per_socket = s->soc[socket].num_harts;
> > +    if (!kvm_enabled()) {
> > +        /* M-level IMSIC node */
> > +        for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> > +            imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> > +            imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> >           }
> > -    }
> > -    imsic_name = g_strdup_printf("/soc/imsics@%lx",
> > -        (unsigned long)memmap[VIRT_IMSIC_M].base);
> > -    qemu_fdt_add_subnode(ms->fdt, imsic_name);
> > -    qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
> > -        "riscv,imsics");
> > -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
> > -        FDT_IMSIC_INT_CELLS);
> > -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
> > -        NULL, 0);
> > -    qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
> > -        NULL, 0);
> > -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
> > -        imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
> > -    qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
> > -        socket_count * sizeof(uint32_t) * 4);
> > -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
> > -        VIRT_IRQCHIP_NUM_MSIS);
> > -    if (socket_count > 1) {
> > -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
> > -            imsic_num_bits(imsic_max_hart_per_socket));
> > -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
> > -            imsic_num_bits(socket_count));
> > -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-shift",
> > -            IMSIC_MMIO_GROUP_MIN_SHIFT);
> > -    }
> > -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
> > +        imsic_max_hart_per_socket = 0;
> > +        for (socket = 0; socket < socket_count; socket++) {
> > +            imsic_addr = memmap[VIRT_IMSIC_M].base +
> > +                         socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> > +            imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
> > +            imsic_regs[socket * 4 + 0] = 0;
> > +            imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
> > +            imsic_regs[socket * 4 + 2] = 0;
> > +            imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
> > +            if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
> > +                imsic_max_hart_per_socket = s->soc[socket].num_harts;
> > +            }
> > +        }
> > +        imsic_name = g_strdup_printf("/soc/imsics@%lx",
> > +            (unsigned long)memmap[VIRT_IMSIC_M].base);
> > +        qemu_fdt_add_subnode(ms->fdt, imsic_name);
> > +        qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
> > +            "riscv,imsics");
> > +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
> > +            FDT_IMSIC_INT_CELLS);
> > +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
> > +            NULL, 0);
> > +        qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
> > +            NULL, 0);
> > +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
> > +            imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
> > +        qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
> > +            socket_count * sizeof(uint32_t) * 4);
> > +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
> > +            VIRT_IRQCHIP_NUM_MSIS);
> > +        if (socket_count > 1) {
> > +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
> > +                imsic_num_bits(imsic_max_hart_per_socket));
> > +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
> > +                imsic_num_bits(socket_count));
> > +            qemu_fdt_setprop_cell(ms->fdt, imsic_name,
> > +                "riscv,group-index-shift", IMSIC_MMIO_GROUP_MIN_SHIFT);
> > +        }
> > +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
> >
> > -    g_free(imsic_name);
> > +        g_free(imsic_name);
> > +    }
> >
> >       /* S-level IMSIC node */
> >       for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
> > @@ -653,37 +655,40 @@ static void create_fdt_socket_aplic(RISCVVirtState *s,
> >       aplic_s_phandle = (*phandle)++;
> >       aplic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 2);
> >
> > -    /* M-level APLIC node */
> > -    for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> > -        aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> > -        aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> > -    }
> > -    aplic_addr = memmap[VIRT_APLIC_M].base +
> > -                 (memmap[VIRT_APLIC_M].size * socket);
> > -    aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
> > -    qemu_fdt_add_subnode(ms->fdt, aplic_name);
> > -    qemu_fdt_setprop_string(ms->fdt, aplic_name, "compatible", "riscv,aplic");
> > -    qemu_fdt_setprop_cell(ms->fdt, aplic_name,
> > -        "#interrupt-cells", FDT_APLIC_INT_CELLS);
> > -    qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
> > -    if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
> > -        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
> > -            aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
> > -    } else {
> > -        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
> > -            msi_m_phandle);
> > +    if (!kvm_enabled()) {
> > +        /* M-level APLIC node */
> > +        for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> > +            aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
> > +            aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
> > +        }
> > +        aplic_addr = memmap[VIRT_APLIC_M].base +
> > +                     (memmap[VIRT_APLIC_M].size * socket);
> > +        aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
> > +        qemu_fdt_add_subnode(ms->fdt, aplic_name);
> > +        qemu_fdt_setprop_string(ms->fdt, aplic_name,
> > +            "compatible", "riscv,aplic");
> > +        qemu_fdt_setprop_cell(ms->fdt, aplic_name,
> > +            "#interrupt-cells", FDT_APLIC_INT_CELLS);
> > +        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
> > +        if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
> > +            qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
> > +                aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
> > +        } else {
> > +            qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
> > +                msi_m_phandle);
> > +        }
> > +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
> > +            0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
> > +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
> > +            VIRT_IRQCHIP_NUM_SOURCES);
> > +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
> > +            aplic_s_phandle);
> > +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
> > +            aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
> > +        riscv_socket_fdt_write_id(ms, aplic_name, socket);
> > +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
> > +        g_free(aplic_name);
> >       }
> > -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
> > -        0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
> > -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
> > -        VIRT_IRQCHIP_NUM_SOURCES);
> > -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
> > -        aplic_s_phandle);
> > -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
> > -        aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
> > -    riscv_socket_fdt_write_id(ms, aplic_name, socket);
> > -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
> > -    g_free(aplic_name);
> >
> >       /* S-level APLIC node */
> >       for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
> > @@ -1162,16 +1167,20 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
> >       int i;
> >       hwaddr addr;
> >       uint32_t guest_bits;
> > -    DeviceState *aplic_m;
> > +    DeviceState *aplic_s = NULL;
> > +    DeviceState *aplic_m = NULL;
> >       bool msimode = (aia_type == VIRT_AIA_TYPE_APLIC_IMSIC) ? true : false;
> >
> >       if (msimode) {
> > -        /* Per-socket M-level IMSICs */
> > -        addr = memmap[VIRT_IMSIC_M].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> > -        for (i = 0; i < hart_count; i++) {
> > -            riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
> > -                               base_hartid + i, true, 1,
> > -                               VIRT_IRQCHIP_NUM_MSIS);
> > +        if (!kvm_enabled()) {
> > +            /* Per-socket M-level IMSICs */
> > +            addr = memmap[VIRT_IMSIC_M].base +
> > +                   socket * VIRT_IMSIC_GROUP_MAX_SIZE;
> > +            for (i = 0; i < hart_count; i++) {
> > +                riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
> > +                        base_hartid + i, true, 1,
> > +                        VIRT_IRQCHIP_NUM_MSIS);
> > +            }
> >           }
> >
> >           /* Per-socket S-level IMSICs */
> > @@ -1184,19 +1193,21 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
> >           }
> >       }
> >
> > -    /* Per-socket M-level APLIC */
> > -    aplic_m = riscv_aplic_create(
> > -        memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
> > -        memmap[VIRT_APLIC_M].size,
> > -        (msimode) ? 0 : base_hartid,
> > -        (msimode) ? 0 : hart_count,
> > -        VIRT_IRQCHIP_NUM_SOURCES,
> > -        VIRT_IRQCHIP_NUM_PRIO_BITS,
> > -        msimode, true, NULL);
> > -
> > -    if (aplic_m) {
> > +    if (!kvm_enabled()) {
> > +        /* Per-socket M-level APLIC */
> > +        aplic_m = riscv_aplic_create(
> > +                memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
> > +                memmap[VIRT_APLIC_M].size,
> > +                (msimode) ? 0 : base_hartid,
> > +                (msimode) ? 0 : hart_count,
> > +                VIRT_IRQCHIP_NUM_SOURCES,
> > +                VIRT_IRQCHIP_NUM_PRIO_BITS,
> > +                msimode, true, NULL);
> > +    }
> > +
> > +    if (aplic_m || kvm_enabled()) {
>
> I am struggling to understand why we're checking for "aplic_m" here given that
> riscv_aplic_create() will either returning something not NULL or error out via
> &error_fatal.
>
> This behavior predates this patch though.
>
>
>
> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
>
> >           /* Per-socket S-level APLIC */
> > -        riscv_aplic_create(
> > +        aplic_s = riscv_aplic_create(
> >               memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
> >               memmap[VIRT_APLIC_S].size,
> >               (msimode) ? 0 : base_hartid,
> > @@ -1206,7 +1217,7 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
> >               msimode, false, aplic_m);
> >       }
> >
> > -    return aplic_m;
> > +    return kvm_enabled() ? aplic_s : aplic_m;
> >   }
> >
> >   static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)

Re: [PATCH v3 2/6] target/riscv: support the AIA device emulation with KVM enabled

[Daniel Henrique Barboza]

Hi,

On 6/12/23 03:50, Yong-Xuan Wang wrote:
> Hi Daniel,
> 
> I think this checking can be removed too. Would you send a patch to
> fix it? Or I can remove it in this patch.

Please go ahead and fix it on this patch. Thanks,


Daniel

> 
> Regards,
> Yong-Xuan
> 
> On Tue, Jun 6, 2023 at 2:45 AM Daniel Henrique Barboza
> <dbarboza@ventanamicro.com> wrote:
>>
>>
>>
>> On 5/26/23 03:25, Yong-Xuan Wang wrote:
>>> Remove M mode AIA devices when using KVM acceleration
>>>
>>> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
>>> Reviewed-by: Jim Shu <jim.shu@sifive.com>
>>> ---
>>>    hw/riscv/virt.c | 199 +++++++++++++++++++++++++-----------------------
>>>    1 file changed, 105 insertions(+), 94 deletions(-)
>>>
>>> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
>>> index 4e3efbee16..18b94888ab 100644
>>> --- a/hw/riscv/virt.c
>>> +++ b/hw/riscv/virt.c
>>> @@ -531,52 +531,54 @@ static void create_fdt_imsic(RISCVVirtState *s, const MemMapEntry *memmap,
>>>        imsic_cells = g_new0(uint32_t, ms->smp.cpus * 2);
>>>        imsic_regs = g_new0(uint32_t, socket_count * 4);
>>>
>>> -    /* M-level IMSIC node */
>>> -    for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
>>> -        imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
>>> -        imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
>>> -    }
>>> -    imsic_max_hart_per_socket = 0;
>>> -    for (socket = 0; socket < socket_count; socket++) {
>>> -        imsic_addr = memmap[VIRT_IMSIC_M].base +
>>> -                     socket * VIRT_IMSIC_GROUP_MAX_SIZE;
>>> -        imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
>>> -        imsic_regs[socket * 4 + 0] = 0;
>>> -        imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
>>> -        imsic_regs[socket * 4 + 2] = 0;
>>> -        imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
>>> -        if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
>>> -            imsic_max_hart_per_socket = s->soc[socket].num_harts;
>>> +    if (!kvm_enabled()) {
>>> +        /* M-level IMSIC node */
>>> +        for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
>>> +            imsic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
>>> +            imsic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
>>>            }
>>> -    }
>>> -    imsic_name = g_strdup_printf("/soc/imsics@%lx",
>>> -        (unsigned long)memmap[VIRT_IMSIC_M].base);
>>> -    qemu_fdt_add_subnode(ms->fdt, imsic_name);
>>> -    qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
>>> -        "riscv,imsics");
>>> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
>>> -        FDT_IMSIC_INT_CELLS);
>>> -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
>>> -        NULL, 0);
>>> -    qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
>>> -        NULL, 0);
>>> -    qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
>>> -        imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
>>> -    qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
>>> -        socket_count * sizeof(uint32_t) * 4);
>>> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
>>> -        VIRT_IRQCHIP_NUM_MSIS);
>>> -    if (socket_count > 1) {
>>> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
>>> -            imsic_num_bits(imsic_max_hart_per_socket));
>>> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
>>> -            imsic_num_bits(socket_count));
>>> -        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-shift",
>>> -            IMSIC_MMIO_GROUP_MIN_SHIFT);
>>> -    }
>>> -    qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
>>> +        imsic_max_hart_per_socket = 0;
>>> +        for (socket = 0; socket < socket_count; socket++) {
>>> +            imsic_addr = memmap[VIRT_IMSIC_M].base +
>>> +                         socket * VIRT_IMSIC_GROUP_MAX_SIZE;
>>> +            imsic_size = IMSIC_HART_SIZE(0) * s->soc[socket].num_harts;
>>> +            imsic_regs[socket * 4 + 0] = 0;
>>> +            imsic_regs[socket * 4 + 1] = cpu_to_be32(imsic_addr);
>>> +            imsic_regs[socket * 4 + 2] = 0;
>>> +            imsic_regs[socket * 4 + 3] = cpu_to_be32(imsic_size);
>>> +            if (imsic_max_hart_per_socket < s->soc[socket].num_harts) {
>>> +                imsic_max_hart_per_socket = s->soc[socket].num_harts;
>>> +            }
>>> +        }
>>> +        imsic_name = g_strdup_printf("/soc/imsics@%lx",
>>> +            (unsigned long)memmap[VIRT_IMSIC_M].base);
>>> +        qemu_fdt_add_subnode(ms->fdt, imsic_name);
>>> +        qemu_fdt_setprop_string(ms->fdt, imsic_name, "compatible",
>>> +            "riscv,imsics");
>>> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "#interrupt-cells",
>>> +            FDT_IMSIC_INT_CELLS);
>>> +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupt-controller",
>>> +            NULL, 0);
>>> +        qemu_fdt_setprop(ms->fdt, imsic_name, "msi-controller",
>>> +            NULL, 0);
>>> +        qemu_fdt_setprop(ms->fdt, imsic_name, "interrupts-extended",
>>> +            imsic_cells, ms->smp.cpus * sizeof(uint32_t) * 2);
>>> +        qemu_fdt_setprop(ms->fdt, imsic_name, "reg", imsic_regs,
>>> +            socket_count * sizeof(uint32_t) * 4);
>>> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,num-ids",
>>> +            VIRT_IRQCHIP_NUM_MSIS);
>>> +        if (socket_count > 1) {
>>> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,hart-index-bits",
>>> +                imsic_num_bits(imsic_max_hart_per_socket));
>>> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name, "riscv,group-index-bits",
>>> +                imsic_num_bits(socket_count));
>>> +            qemu_fdt_setprop_cell(ms->fdt, imsic_name,
>>> +                "riscv,group-index-shift", IMSIC_MMIO_GROUP_MIN_SHIFT);
>>> +        }
>>> +        qemu_fdt_setprop_cell(ms->fdt, imsic_name, "phandle", *msi_m_phandle);
>>>
>>> -    g_free(imsic_name);
>>> +        g_free(imsic_name);
>>> +    }
>>>
>>>        /* S-level IMSIC node */
>>>        for (cpu = 0; cpu < ms->smp.cpus; cpu++) {
>>> @@ -653,37 +655,40 @@ static void create_fdt_socket_aplic(RISCVVirtState *s,
>>>        aplic_s_phandle = (*phandle)++;
>>>        aplic_cells = g_new0(uint32_t, s->soc[socket].num_harts * 2);
>>>
>>> -    /* M-level APLIC node */
>>> -    for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
>>> -        aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
>>> -        aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
>>> -    }
>>> -    aplic_addr = memmap[VIRT_APLIC_M].base +
>>> -                 (memmap[VIRT_APLIC_M].size * socket);
>>> -    aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
>>> -    qemu_fdt_add_subnode(ms->fdt, aplic_name);
>>> -    qemu_fdt_setprop_string(ms->fdt, aplic_name, "compatible", "riscv,aplic");
>>> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name,
>>> -        "#interrupt-cells", FDT_APLIC_INT_CELLS);
>>> -    qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
>>> -    if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
>>> -        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
>>> -            aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
>>> -    } else {
>>> -        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
>>> -            msi_m_phandle);
>>> +    if (!kvm_enabled()) {
>>> +        /* M-level APLIC node */
>>> +        for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
>>> +            aplic_cells[cpu * 2 + 0] = cpu_to_be32(intc_phandles[cpu]);
>>> +            aplic_cells[cpu * 2 + 1] = cpu_to_be32(IRQ_M_EXT);
>>> +        }
>>> +        aplic_addr = memmap[VIRT_APLIC_M].base +
>>> +                     (memmap[VIRT_APLIC_M].size * socket);
>>> +        aplic_name = g_strdup_printf("/soc/aplic@%lx", aplic_addr);
>>> +        qemu_fdt_add_subnode(ms->fdt, aplic_name);
>>> +        qemu_fdt_setprop_string(ms->fdt, aplic_name,
>>> +            "compatible", "riscv,aplic");
>>> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name,
>>> +            "#interrupt-cells", FDT_APLIC_INT_CELLS);
>>> +        qemu_fdt_setprop(ms->fdt, aplic_name, "interrupt-controller", NULL, 0);
>>> +        if (s->aia_type == VIRT_AIA_TYPE_APLIC) {
>>> +            qemu_fdt_setprop(ms->fdt, aplic_name, "interrupts-extended",
>>> +                aplic_cells, s->soc[socket].num_harts * sizeof(uint32_t) * 2);
>>> +        } else {
>>> +            qemu_fdt_setprop_cell(ms->fdt, aplic_name, "msi-parent",
>>> +                msi_m_phandle);
>>> +        }
>>> +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
>>> +            0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
>>> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
>>> +            VIRT_IRQCHIP_NUM_SOURCES);
>>> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
>>> +            aplic_s_phandle);
>>> +        qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
>>> +            aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
>>> +        riscv_socket_fdt_write_id(ms, aplic_name, socket);
>>> +        qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
>>> +        g_free(aplic_name);
>>>        }
>>> -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "reg",
>>> -        0x0, aplic_addr, 0x0, memmap[VIRT_APLIC_M].size);
>>> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,num-sources",
>>> -        VIRT_IRQCHIP_NUM_SOURCES);
>>> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "riscv,children",
>>> -        aplic_s_phandle);
>>> -    qemu_fdt_setprop_cells(ms->fdt, aplic_name, "riscv,delegate",
>>> -        aplic_s_phandle, 0x1, VIRT_IRQCHIP_NUM_SOURCES);
>>> -    riscv_socket_fdt_write_id(ms, aplic_name, socket);
>>> -    qemu_fdt_setprop_cell(ms->fdt, aplic_name, "phandle", aplic_m_phandle);
>>> -    g_free(aplic_name);
>>>
>>>        /* S-level APLIC node */
>>>        for (cpu = 0; cpu < s->soc[socket].num_harts; cpu++) {
>>> @@ -1162,16 +1167,20 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>>>        int i;
>>>        hwaddr addr;
>>>        uint32_t guest_bits;
>>> -    DeviceState *aplic_m;
>>> +    DeviceState *aplic_s = NULL;
>>> +    DeviceState *aplic_m = NULL;
>>>        bool msimode = (aia_type == VIRT_AIA_TYPE_APLIC_IMSIC) ? true : false;
>>>
>>>        if (msimode) {
>>> -        /* Per-socket M-level IMSICs */
>>> -        addr = memmap[VIRT_IMSIC_M].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE;
>>> -        for (i = 0; i < hart_count; i++) {
>>> -            riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
>>> -                               base_hartid + i, true, 1,
>>> -                               VIRT_IRQCHIP_NUM_MSIS);
>>> +        if (!kvm_enabled()) {
>>> +            /* Per-socket M-level IMSICs */
>>> +            addr = memmap[VIRT_IMSIC_M].base +
>>> +                   socket * VIRT_IMSIC_GROUP_MAX_SIZE;
>>> +            for (i = 0; i < hart_count; i++) {
>>> +                riscv_imsic_create(addr + i * IMSIC_HART_SIZE(0),
>>> +                        base_hartid + i, true, 1,
>>> +                        VIRT_IRQCHIP_NUM_MSIS);
>>> +            }
>>>            }
>>>
>>>            /* Per-socket S-level IMSICs */
>>> @@ -1184,19 +1193,21 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>>>            }
>>>        }
>>>
>>> -    /* Per-socket M-level APLIC */
>>> -    aplic_m = riscv_aplic_create(
>>> -        memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
>>> -        memmap[VIRT_APLIC_M].size,
>>> -        (msimode) ? 0 : base_hartid,
>>> -        (msimode) ? 0 : hart_count,
>>> -        VIRT_IRQCHIP_NUM_SOURCES,
>>> -        VIRT_IRQCHIP_NUM_PRIO_BITS,
>>> -        msimode, true, NULL);
>>> -
>>> -    if (aplic_m) {
>>> +    if (!kvm_enabled()) {
>>> +        /* Per-socket M-level APLIC */
>>> +        aplic_m = riscv_aplic_create(
>>> +                memmap[VIRT_APLIC_M].base + socket * memmap[VIRT_APLIC_M].size,
>>> +                memmap[VIRT_APLIC_M].size,
>>> +                (msimode) ? 0 : base_hartid,
>>> +                (msimode) ? 0 : hart_count,
>>> +                VIRT_IRQCHIP_NUM_SOURCES,
>>> +                VIRT_IRQCHIP_NUM_PRIO_BITS,
>>> +                msimode, true, NULL);
>>> +    }
>>> +
>>> +    if (aplic_m || kvm_enabled()) {
>>
>> I am struggling to understand why we're checking for "aplic_m" here given that
>> riscv_aplic_create() will either returning something not NULL or error out via
>> &error_fatal.
>>
>> This behavior predates this patch though.
>>
>>
>>
>> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
>>
>>>            /* Per-socket S-level APLIC */
>>> -        riscv_aplic_create(
>>> +        aplic_s = riscv_aplic_create(
>>>                memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
>>>                memmap[VIRT_APLIC_S].size,
>>>                (msimode) ? 0 : base_hartid,
>>> @@ -1206,7 +1217,7 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>>>                msimode, false, aplic_m);
>>>        }
>>>
>>> -    return aplic_m;
>>> +    return kvm_enabled() ? aplic_s : aplic_m;
>>>    }
>>>
>>>    static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)

[PATCH v3 3/6] target/riscv: check the in-kernel irqchip support

[Yong-Xuan Wang]

We check the in-kernel irqchip support when using KVM acceleration.

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 target/riscv/kvm.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c
index 0f932a5b96..eb469e8ca5 100644
--- a/target/riscv/kvm.c
+++ b/target/riscv/kvm.c
@@ -433,7 +433,18 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
 
 int kvm_arch_irqchip_create(KVMState *s)
 {
-    return 0;
+    if (kvm_kernel_irqchip_split()) {
+        error_report("-machine kernel_irqchip=split is not supported "
+                     "on RISC-V.");
+        exit(1);
+    }
+
+    /*
+     * If we can create the VAIA using the newer device control API, we
+     * let the device do this when it initializes itself, otherwise we
+     * fall back to the old API
+     */
+    return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
 }
 
 int kvm_arch_process_async_events(CPUState *cs)
-- 
2.17.1

Re: [PATCH v3 3/6] target/riscv: check the in-kernel irqchip support

[Daniel Henrique Barboza]

On 5/26/23 03:25, Yong-Xuan Wang wrote:
> We check the in-kernel irqchip support when using KVM acceleration.
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---

Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

>   target/riscv/kvm.c | 13 ++++++++++++-
>   1 file changed, 12 insertions(+), 1 deletion(-)
> 
> diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c
> index 0f932a5b96..eb469e8ca5 100644
> --- a/target/riscv/kvm.c
> +++ b/target/riscv/kvm.c
> @@ -433,7 +433,18 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
>   
>   int kvm_arch_irqchip_create(KVMState *s)
>   {
> -    return 0;
> +    if (kvm_kernel_irqchip_split()) {
> +        error_report("-machine kernel_irqchip=split is not supported "
> +                     "on RISC-V.");
> +        exit(1);
> +    }
> +
> +    /*
> +     * If we can create the VAIA using the newer device control API, we
> +     * let the device do this when it initializes itself, otherwise we
> +     * fall back to the old API
> +     */
> +    return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
>   }
>   
>   int kvm_arch_process_async_events(CPUState *cs)

[PATCH v3 4/6] target/riscv: Create an KVM AIA irqchip

[Yong-Xuan Wang]

implement a function to create an KVM AIA chip

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 target/riscv/kvm.c       | 83 ++++++++++++++++++++++++++++++++++++++++
 target/riscv/kvm_riscv.h |  3 ++
 2 files changed, 86 insertions(+)

diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c
index eb469e8ca5..ead121154f 100644
--- a/target/riscv/kvm.c
+++ b/target/riscv/kvm.c
@@ -34,6 +34,7 @@
 #include "exec/address-spaces.h"
 #include "hw/boards.h"
 #include "hw/irq.h"
+#include "hw/intc/riscv_imsic.h"
 #include "qemu/log.h"
 #include "hw/loader.h"
 #include "kvm_riscv.h"
@@ -548,3 +549,85 @@ bool kvm_arch_cpu_check_are_resettable(void)
 void kvm_arch_accel_class_init(ObjectClass *oc)
 {
 }
+
+void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
+                          uint64_t aia_irq_num, uint64_t hart_count,
+                          uint64_t aplic_base, uint64_t imsic_base)
+{
+    int ret;
+    int aia_fd = -1;
+    uint64_t aia_mode;
+    uint64_t aia_nr_ids;
+    uint64_t aia_hart_bits = find_last_bit(&hart_count, BITS_PER_LONG) + 1;
+
+    if (!msimode) {
+        error_report("Currently KVM AIA only supports aplic_imsic mode");
+        exit(1);
+    }
+
+    aia_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_RISCV_AIA, false);
+
+    if (aia_fd < 0) {
+        error_report("Unable to create in-kernel irqchip");
+        exit(1);
+    }
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
+                            KVM_DEV_RISCV_AIA_CONFIG_MODE,
+                            &aia_mode, false, NULL);
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
+                            KVM_DEV_RISCV_AIA_CONFIG_IDS,
+                            &aia_nr_ids, false, NULL);
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
+                            KVM_DEV_RISCV_AIA_CONFIG_SRCS,
+                            &aia_irq_num, true, NULL);
+    if (ret < 0) {
+        error_report("KVM AIA: fail to set number input irq lines");
+        exit(1);
+    }
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
+                            KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
+                            &aia_hart_bits, true, NULL);
+    if (ret < 0) {
+        error_report("KVM AIA: fail to set number of harts");
+        exit(1);
+    }
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
+                            KVM_DEV_RISCV_AIA_ADDR_APLIC,
+                            &aplic_base, true, NULL);
+    if (ret < 0) {
+        error_report("KVM AIA: fail to set the base address of APLIC");
+        exit(1);
+    }
+
+    for (int i = 0; i < hart_count; i++) {
+        uint64_t imsic_addr = imsic_base + i * IMSIC_HART_SIZE(0);
+        ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
+                                KVM_DEV_RISCV_AIA_ADDR_IMSIC(i),
+                                &imsic_addr, true, NULL);
+        if (ret < 0) {
+            error_report("KVM AIA: fail to set the base address of IMSICs");
+            exit(1);
+        }
+    }
+
+    if (kvm_has_gsi_routing()) {
+        for (uint64_t idx = 0; idx < aia_irq_num + 1; ++idx) {
+            kvm_irqchip_add_irq_route(kvm_state, idx, socket, idx);
+        }
+        kvm_gsi_routing_allowed = true;
+        kvm_irqchip_commit_routes(kvm_state);
+    }
+
+    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CTRL,
+                            KVM_DEV_RISCV_AIA_CTRL_INIT,
+                            NULL, true, NULL);
+    if (ret < 0) {
+        error_report("KVM AIA: initialized fail");
+        exit(1);
+    }
+}
diff --git a/target/riscv/kvm_riscv.h b/target/riscv/kvm_riscv.h
index 606968a4b7..6067adff51 100644
--- a/target/riscv/kvm_riscv.h
+++ b/target/riscv/kvm_riscv.h
@@ -21,6 +21,9 @@
 
 void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
 void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
+void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
+                          uint64_t aia_irq_num, uint64_t hart_count,
+                          uint64_t aplic_base, uint64_t imsic_base);
 
 #define KVM_DEV_RISCV_AIA_GRP_CONFIG            0
 #define KVM_DEV_RISCV_AIA_CONFIG_MODE           0
-- 
2.17.1

Re: [PATCH v3 4/6] target/riscv: Create an KVM AIA irqchip

[Daniel Henrique Barboza]

On 5/26/23 03:25, Yong-Xuan Wang wrote:
> implement a function to create an KVM AIA chip
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---
>   target/riscv/kvm.c       | 83 ++++++++++++++++++++++++++++++++++++++++
>   target/riscv/kvm_riscv.h |  3 ++
>   2 files changed, 86 insertions(+)
> 
> diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c
> index eb469e8ca5..ead121154f 100644
> --- a/target/riscv/kvm.c
> +++ b/target/riscv/kvm.c
> @@ -34,6 +34,7 @@
>   #include "exec/address-spaces.h"
>   #include "hw/boards.h"
>   #include "hw/irq.h"
> +#include "hw/intc/riscv_imsic.h"
>   #include "qemu/log.h"
>   #include "hw/loader.h"
>   #include "kvm_riscv.h"
> @@ -548,3 +549,85 @@ bool kvm_arch_cpu_check_are_resettable(void)
>   void kvm_arch_accel_class_init(ObjectClass *oc)
>   {
>   }
> +
> +void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
> +                          uint64_t aia_irq_num, uint64_t hart_count,
> +                          uint64_t aplic_base, uint64_t imsic_base)
> +{
> +    int ret;
> +    int aia_fd = -1;
> +    uint64_t aia_mode;
> +    uint64_t aia_nr_ids;
> +    uint64_t aia_hart_bits = find_last_bit(&hart_count, BITS_PER_LONG) + 1;
> +
> +    if (!msimode) {
> +        error_report("Currently KVM AIA only supports aplic_imsic mode");
> +        exit(1);
> +    }
> +
> +    aia_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_RISCV_AIA, false);
> +
> +    if (aia_fd < 0) {
> +        error_report("Unable to create in-kernel irqchip");
> +        exit(1);
> +    }
> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> +                            KVM_DEV_RISCV_AIA_CONFIG_MODE,
> +                            &aia_mode, false, NULL);
> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> +                            KVM_DEV_RISCV_AIA_CONFIG_IDS,
> +                            &aia_nr_ids, false, NULL);
> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> +                            KVM_DEV_RISCV_AIA_CONFIG_SRCS,
> +                            &aia_irq_num, true, NULL);
> +    if (ret < 0) {
> +        error_report("KVM AIA: fail to set number input irq lines");
> +        exit(1);
> +    }

I see that you didn't check 'ret' for the first 2 calls of kvm_device_access().
Is it intentional?

Since you're setting customized error messages for every step I think it's worth
also handling the case where we fail to set aia_mode and aia_nr_ids.


Thanks,


Daniel


> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> +                            KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
> +                            &aia_hart_bits, true, NULL);
> +    if (ret < 0) {
> +        error_report("KVM AIA: fail to set number of harts");
> +        exit(1);
> +    }
> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
> +                            KVM_DEV_RISCV_AIA_ADDR_APLIC,
> +                            &aplic_base, true, NULL);
> +    if (ret < 0) {
> +        error_report("KVM AIA: fail to set the base address of APLIC");
> +        exit(1);
> +    }
> +
> +    for (int i = 0; i < hart_count; i++) {
> +        uint64_t imsic_addr = imsic_base + i * IMSIC_HART_SIZE(0);
> +        ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
> +                                KVM_DEV_RISCV_AIA_ADDR_IMSIC(i),
> +                                &imsic_addr, true, NULL);
> +        if (ret < 0) {
> +            error_report("KVM AIA: fail to set the base address of IMSICs");
> +            exit(1);
> +        }
> +    }
> +
> +    if (kvm_has_gsi_routing()) {
> +        for (uint64_t idx = 0; idx < aia_irq_num + 1; ++idx) {
> +            kvm_irqchip_add_irq_route(kvm_state, idx, socket, idx);
> +        }
> +        kvm_gsi_routing_allowed = true;
> +        kvm_irqchip_commit_routes(kvm_state);
> +    }
> +
> +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CTRL,
> +                            KVM_DEV_RISCV_AIA_CTRL_INIT,
> +                            NULL, true, NULL);
> +    if (ret < 0) {
> +        error_report("KVM AIA: initialized fail");
> +        exit(1);
> +    }
> +}
> diff --git a/target/riscv/kvm_riscv.h b/target/riscv/kvm_riscv.h
> index 606968a4b7..6067adff51 100644
> --- a/target/riscv/kvm_riscv.h
> +++ b/target/riscv/kvm_riscv.h
> @@ -21,6 +21,9 @@
>   
>   void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
>   void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
> +void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
> +                          uint64_t aia_irq_num, uint64_t hart_count,
> +                          uint64_t aplic_base, uint64_t imsic_base);
>   
>   #define KVM_DEV_RISCV_AIA_GRP_CONFIG            0
>   #define KVM_DEV_RISCV_AIA_CONFIG_MODE           0

Re: [PATCH v3 4/6] target/riscv: Create an KVM AIA irqchip

[Yong-Xuan Wang]

Hi  Daniel,

Thanks for your suggestions! I'll fix it in patch v4.

Regards,
Yong-Xuan

On Tue, Jun 6, 2023 at 9:45 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
>
>
> On 5/26/23 03:25, Yong-Xuan Wang wrote:
> > implement a function to create an KVM AIA chip
> >
> > Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> > Reviewed-by: Jim Shu <jim.shu@sifive.com>
> > ---
> >   target/riscv/kvm.c       | 83 ++++++++++++++++++++++++++++++++++++++++
> >   target/riscv/kvm_riscv.h |  3 ++
> >   2 files changed, 86 insertions(+)
> >
> > diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c
> > index eb469e8ca5..ead121154f 100644
> > --- a/target/riscv/kvm.c
> > +++ b/target/riscv/kvm.c
> > @@ -34,6 +34,7 @@
> >   #include "exec/address-spaces.h"
> >   #include "hw/boards.h"
> >   #include "hw/irq.h"
> > +#include "hw/intc/riscv_imsic.h"
> >   #include "qemu/log.h"
> >   #include "hw/loader.h"
> >   #include "kvm_riscv.h"
> > @@ -548,3 +549,85 @@ bool kvm_arch_cpu_check_are_resettable(void)
> >   void kvm_arch_accel_class_init(ObjectClass *oc)
> >   {
> >   }
> > +
> > +void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
> > +                          uint64_t aia_irq_num, uint64_t hart_count,
> > +                          uint64_t aplic_base, uint64_t imsic_base)
> > +{
> > +    int ret;
> > +    int aia_fd = -1;
> > +    uint64_t aia_mode;
> > +    uint64_t aia_nr_ids;
> > +    uint64_t aia_hart_bits = find_last_bit(&hart_count, BITS_PER_LONG) + 1;
> > +
> > +    if (!msimode) {
> > +        error_report("Currently KVM AIA only supports aplic_imsic mode");
> > +        exit(1);
> > +    }
> > +
> > +    aia_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_RISCV_AIA, false);
> > +
> > +    if (aia_fd < 0) {
> > +        error_report("Unable to create in-kernel irqchip");
> > +        exit(1);
> > +    }
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> > +                            KVM_DEV_RISCV_AIA_CONFIG_MODE,
> > +                            &aia_mode, false, NULL);
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> > +                            KVM_DEV_RISCV_AIA_CONFIG_IDS,
> > +                            &aia_nr_ids, false, NULL);
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> > +                            KVM_DEV_RISCV_AIA_CONFIG_SRCS,
> > +                            &aia_irq_num, true, NULL);
> > +    if (ret < 0) {
> > +        error_report("KVM AIA: fail to set number input irq lines");
> > +        exit(1);
> > +    }
>
> I see that you didn't check 'ret' for the first 2 calls of kvm_device_access().
> Is it intentional?
>
> Since you're setting customized error messages for every step I think it's worth
> also handling the case where we fail to set aia_mode and aia_nr_ids.
>
>
> Thanks,
>
>
> Daniel
>
>
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
> > +                            KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
> > +                            &aia_hart_bits, true, NULL);
> > +    if (ret < 0) {
> > +        error_report("KVM AIA: fail to set number of harts");
> > +        exit(1);
> > +    }
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
> > +                            KVM_DEV_RISCV_AIA_ADDR_APLIC,
> > +                            &aplic_base, true, NULL);
> > +    if (ret < 0) {
> > +        error_report("KVM AIA: fail to set the base address of APLIC");
> > +        exit(1);
> > +    }
> > +
> > +    for (int i = 0; i < hart_count; i++) {
> > +        uint64_t imsic_addr = imsic_base + i * IMSIC_HART_SIZE(0);
> > +        ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
> > +                                KVM_DEV_RISCV_AIA_ADDR_IMSIC(i),
> > +                                &imsic_addr, true, NULL);
> > +        if (ret < 0) {
> > +            error_report("KVM AIA: fail to set the base address of IMSICs");
> > +            exit(1);
> > +        }
> > +    }
> > +
> > +    if (kvm_has_gsi_routing()) {
> > +        for (uint64_t idx = 0; idx < aia_irq_num + 1; ++idx) {
> > +            kvm_irqchip_add_irq_route(kvm_state, idx, socket, idx);
> > +        }
> > +        kvm_gsi_routing_allowed = true;
> > +        kvm_irqchip_commit_routes(kvm_state);
> > +    }
> > +
> > +    ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CTRL,
> > +                            KVM_DEV_RISCV_AIA_CTRL_INIT,
> > +                            NULL, true, NULL);
> > +    if (ret < 0) {
> > +        error_report("KVM AIA: initialized fail");
> > +        exit(1);
> > +    }
> > +}
> > diff --git a/target/riscv/kvm_riscv.h b/target/riscv/kvm_riscv.h
> > index 606968a4b7..6067adff51 100644
> > --- a/target/riscv/kvm_riscv.h
> > +++ b/target/riscv/kvm_riscv.h
> > @@ -21,6 +21,9 @@
> >
> >   void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
> >   void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
> > +void kvm_riscv_aia_create(DeviceState *aplic_s, bool msimode, int socket,
> > +                          uint64_t aia_irq_num, uint64_t hart_count,
> > +                          uint64_t aplic_base, uint64_t imsic_base);
> >
> >   #define KVM_DEV_RISCV_AIA_GRP_CONFIG            0
> >   #define KVM_DEV_RISCV_AIA_CONFIG_MODE           0

[PATCH v3 5/6] target/riscv: update APLIC and IMSIC to support KVM AIA

[Yong-Xuan Wang]

- Do not set the mmio operations of APLIC and IMSIC when using KVM AIA
- Send interrupt signal to KVM AIA via KVM_IRQ_LINE API

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 hw/intc/riscv_aplic.c | 19 +++++++++++++++----
 hw/intc/riscv_imsic.c | 16 +++++++++++-----
 2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c
index afc5b54dbb..adf5427f22 100644
--- a/hw/intc/riscv_aplic.c
+++ b/hw/intc/riscv_aplic.c
@@ -31,6 +31,7 @@
 #include "hw/irq.h"
 #include "target/riscv/cpu.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "migration/vmstate.h"
 
 #define APLIC_MAX_IDC                  (1UL << 14)
@@ -479,6 +480,11 @@ static void riscv_aplic_request(void *opaque, int irq, int level)
 
     assert((0 < irq) && (irq < aplic->num_irqs));
 
+    if (kvm_irqchip_in_kernel()) {
+        kvm_set_irq(kvm_state, irq, !!level);
+        return;
+    }
+
     sourcecfg = aplic->sourcecfg[irq];
     if (sourcecfg & APLIC_SOURCECFG_D) {
         childidx = sourcecfg & APLIC_SOURCECFG_CHILDIDX_MASK;
@@ -814,9 +820,11 @@ static void riscv_aplic_realize(DeviceState *dev, Error **errp)
     aplic->iforce = g_new0(uint32_t, aplic->num_harts);
     aplic->ithreshold = g_new0(uint32_t, aplic->num_harts);
 
-    memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops, aplic,
-                          TYPE_RISCV_APLIC, aplic->aperture_size);
-    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
+    if (!kvm_irqchip_in_kernel()) {
+        memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops,
+                             aplic, TYPE_RISCV_APLIC, aplic->aperture_size);
+        sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
+    }
 
     /*
      * Only root APLICs have hardware IRQ lines. All non-root APLICs
@@ -958,7 +966,10 @@ DeviceState *riscv_aplic_create(hwaddr addr, hwaddr size,
     qdev_prop_set_bit(dev, "mmode", mmode);
 
     sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
+
+    if (!kvm_irqchip_in_kernel()) {
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
+    }
 
     if (parent) {
         riscv_aplic_add_child(parent, dev);
diff --git a/hw/intc/riscv_imsic.c b/hw/intc/riscv_imsic.c
index fea3385b51..8bfa480f7c 100644
--- a/hw/intc/riscv_imsic.c
+++ b/hw/intc/riscv_imsic.c
@@ -32,6 +32,7 @@
 #include "target/riscv/cpu.h"
 #include "target/riscv/cpu_bits.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "migration/vmstate.h"
 
 #define IMSIC_MMIO_PAGE_LE             0x00
@@ -325,10 +326,12 @@ static void riscv_imsic_realize(DeviceState *dev, Error **errp)
     imsic->eithreshold = g_new0(uint32_t, imsic->num_pages);
     imsic->eistate = g_new0(uint32_t, imsic->num_eistate);
 
-    memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
-                          imsic, TYPE_RISCV_IMSIC,
-                          IMSIC_MMIO_SIZE(imsic->num_pages));
-    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &imsic->mmio);
+    if (!kvm_irqchip_in_kernel()) {
+        memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
+                              imsic, TYPE_RISCV_IMSIC,
+                              IMSIC_MMIO_SIZE(imsic->num_pages));
+        sysbus_init_mmio(SYS_BUS_DEVICE(dev), &imsic->mmio);
+    }
 
     /* Claim the CPU interrupt to be triggered by this IMSIC */
     if (riscv_cpu_claim_interrupts(rcpu,
@@ -432,7 +435,10 @@ DeviceState *riscv_imsic_create(hwaddr addr, uint32_t hartid, bool mmode,
     qdev_prop_set_uint32(dev, "num-irqs", num_ids + 1);
 
     sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
+
+    if (!kvm_irqchip_in_kernel()) {
+        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
+    }
 
     for (i = 0; i < num_pages; i++) {
         if (!i) {
-- 
2.17.1

Re: [PATCH v3 5/6] target/riscv: update APLIC and IMSIC to support KVM AIA

[Daniel Henrique Barboza]

On 5/26/23 03:25, Yong-Xuan Wang wrote:
> - Do not set the mmio operations of APLIC and IMSIC when using KVM AIA
> - Send interrupt signal to KVM AIA via KVM_IRQ_LINE API
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---

Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

>   hw/intc/riscv_aplic.c | 19 +++++++++++++++----
>   hw/intc/riscv_imsic.c | 16 +++++++++++-----
>   2 files changed, 26 insertions(+), 9 deletions(-)
> 
> diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c
> index afc5b54dbb..adf5427f22 100644
> --- a/hw/intc/riscv_aplic.c
> +++ b/hw/intc/riscv_aplic.c
> @@ -31,6 +31,7 @@
>   #include "hw/irq.h"
>   #include "target/riscv/cpu.h"
>   #include "sysemu/sysemu.h"
> +#include "sysemu/kvm.h"
>   #include "migration/vmstate.h"
>   
>   #define APLIC_MAX_IDC                  (1UL << 14)
> @@ -479,6 +480,11 @@ static void riscv_aplic_request(void *opaque, int irq, int level)
>   
>       assert((0 < irq) && (irq < aplic->num_irqs));
>   
> +    if (kvm_irqchip_in_kernel()) {
> +        kvm_set_irq(kvm_state, irq, !!level);
> +        return;
> +    }
> +
>       sourcecfg = aplic->sourcecfg[irq];
>       if (sourcecfg & APLIC_SOURCECFG_D) {
>           childidx = sourcecfg & APLIC_SOURCECFG_CHILDIDX_MASK;
> @@ -814,9 +820,11 @@ static void riscv_aplic_realize(DeviceState *dev, Error **errp)
>       aplic->iforce = g_new0(uint32_t, aplic->num_harts);
>       aplic->ithreshold = g_new0(uint32_t, aplic->num_harts);
>   
> -    memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops, aplic,
> -                          TYPE_RISCV_APLIC, aplic->aperture_size);
> -    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
> +    if (!kvm_irqchip_in_kernel()) {
> +        memory_region_init_io(&aplic->mmio, OBJECT(dev), &riscv_aplic_ops,
> +                             aplic, TYPE_RISCV_APLIC, aplic->aperture_size);
> +        sysbus_init_mmio(SYS_BUS_DEVICE(dev), &aplic->mmio);
> +    }
>   
>       /*
>        * Only root APLICs have hardware IRQ lines. All non-root APLICs
> @@ -958,7 +966,10 @@ DeviceState *riscv_aplic_create(hwaddr addr, hwaddr size,
>       qdev_prop_set_bit(dev, "mmode", mmode);
>   
>       sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> -    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +
> +    if (!kvm_irqchip_in_kernel()) {
> +        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +    }
>   
>       if (parent) {
>           riscv_aplic_add_child(parent, dev);
> diff --git a/hw/intc/riscv_imsic.c b/hw/intc/riscv_imsic.c
> index fea3385b51..8bfa480f7c 100644
> --- a/hw/intc/riscv_imsic.c
> +++ b/hw/intc/riscv_imsic.c
> @@ -32,6 +32,7 @@
>   #include "target/riscv/cpu.h"
>   #include "target/riscv/cpu_bits.h"
>   #include "sysemu/sysemu.h"
> +#include "sysemu/kvm.h"
>   #include "migration/vmstate.h"
>   
>   #define IMSIC_MMIO_PAGE_LE             0x00
> @@ -325,10 +326,12 @@ static void riscv_imsic_realize(DeviceState *dev, Error **errp)
>       imsic->eithreshold = g_new0(uint32_t, imsic->num_pages);
>       imsic->eistate = g_new0(uint32_t, imsic->num_eistate);
>   
> -    memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
> -                          imsic, TYPE_RISCV_IMSIC,
> -                          IMSIC_MMIO_SIZE(imsic->num_pages));
> -    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &imsic->mmio);
> +    if (!kvm_irqchip_in_kernel()) {
> +        memory_region_init_io(&imsic->mmio, OBJECT(dev), &riscv_imsic_ops,
> +                              imsic, TYPE_RISCV_IMSIC,
> +                              IMSIC_MMIO_SIZE(imsic->num_pages));
> +        sysbus_init_mmio(SYS_BUS_DEVICE(dev), &imsic->mmio);
> +    }
>   
>       /* Claim the CPU interrupt to be triggered by this IMSIC */
>       if (riscv_cpu_claim_interrupts(rcpu,
> @@ -432,7 +435,10 @@ DeviceState *riscv_imsic_create(hwaddr addr, uint32_t hartid, bool mmode,
>       qdev_prop_set_uint32(dev, "num-irqs", num_ids + 1);
>   
>       sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> -    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +
> +    if (!kvm_irqchip_in_kernel()) {
> +        sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr);
> +    }
>   
>       for (i = 0; i < num_pages; i++) {
>           if (!i) {

[PATCH v3 6/6] target/riscv: select KVM AIA in riscv virt machine

[Yong-Xuan Wang]

Select KVM AIA when the host kernel has in-kernel AIA chip support.

Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
Reviewed-by: Jim Shu <jim.shu@sifive.com>
---
 hw/riscv/virt.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index 18b94888ab..57a07fa6c5 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -35,6 +35,7 @@
 #include "hw/riscv/virt.h"
 #include "hw/riscv/boot.h"
 #include "hw/riscv/numa.h"
+#include "kvm_riscv.h"
 #include "hw/intc/riscv_aclint.h"
 #include "hw/intc/riscv_aplic.h"
 #include "hw/intc/riscv_imsic.h"
@@ -1217,6 +1218,15 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
             msimode, false, aplic_m);
     }
 
+    if (kvm_irqchip_in_kernel()) {
+        kvm_riscv_aia_create(
+            aplic_s, msimode, socket,
+            VIRT_IRQCHIP_NUM_SOURCES,
+            hart_count,
+            memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
+            memmap[VIRT_IMSIC_S].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE);
+    }
+
     return kvm_enabled() ? aplic_s : aplic_m;
 }
 
-- 
2.17.1

Re: [PATCH v3 6/6] target/riscv: select KVM AIA in riscv virt machine

[Daniel Henrique Barboza]

On 5/26/23 03:25, Yong-Xuan Wang wrote:
> Select KVM AIA when the host kernel has in-kernel AIA chip support.
> 
> Signed-off-by: Yong-Xuan Wang <yongxuan.wang@sifive.com>
> Reviewed-by: Jim Shu <jim.shu@sifive.com>
> ---

Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

>   hw/riscv/virt.c | 10 ++++++++++
>   1 file changed, 10 insertions(+)
> 
> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> index 18b94888ab..57a07fa6c5 100644
> --- a/hw/riscv/virt.c
> +++ b/hw/riscv/virt.c
> @@ -35,6 +35,7 @@
>   #include "hw/riscv/virt.h"
>   #include "hw/riscv/boot.h"
>   #include "hw/riscv/numa.h"
> +#include "kvm_riscv.h"
>   #include "hw/intc/riscv_aclint.h"
>   #include "hw/intc/riscv_aplic.h"
>   #include "hw/intc/riscv_imsic.h"
> @@ -1217,6 +1218,15 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
>               msimode, false, aplic_m);
>       }
>   
> +    if (kvm_irqchip_in_kernel()) {
> +        kvm_riscv_aia_create(
> +            aplic_s, msimode, socket,
> +            VIRT_IRQCHIP_NUM_SOURCES,
> +            hart_count,
> +            memmap[VIRT_APLIC_S].base + socket * memmap[VIRT_APLIC_S].size,
> +            memmap[VIRT_IMSIC_S].base + socket * VIRT_IMSIC_GROUP_MAX_SIZE);
> +    }
> +
>       return kvm_enabled() ? aplic_s : aplic_m;
>   }
>