Re: [PATCH 5/5] hw/riscv: virt: support for RISC-V IOMMU platform device.

[Tomasz Jeznach <tjeznach@rivosinc.com>]

On Sun, Jul 23, 2023 at 7:35 PM Alistair Francis <alistair23@gmail.com> wrote:
>
> On Thu, Jul 20, 2023 at 12:35 PM Tomasz Jeznach <tjeznach@rivosinc.com> wrote:
> >
> > Adding virt machine property 'iommu' to enable/disable IOMMU
> > support, with platform RISC-V IOMMU device implementation.
> >
> > Generate device tree entry for riscv-iommu device, along with
> > mapping all PCI device identifiers to the single IOMMU device
> > instance.
> >
> > Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> > ---
> >  hw/riscv/Kconfig        |   1 +
> >  hw/riscv/virt.c         | 100 +++++++++++++++++++++++++++++++++++++++-
> >  include/hw/riscv/virt.h |   3 ++
> >  3 files changed, 103 insertions(+), 1 deletion(-)
> >
> > diff --git a/hw/riscv/Kconfig b/hw/riscv/Kconfig
> > index 617a509f1b..b1a3a9994f 100644
> > --- a/hw/riscv/Kconfig
> > +++ b/hw/riscv/Kconfig
> > @@ -41,6 +41,7 @@ config RISCV_VIRT
> >      select SERIAL
> >      select RISCV_ACLINT
> >      select RISCV_APLIC
> > +    select RISCV_IOMMU
> >      select RISCV_IMSIC
> >      select SIFIVE_PLIC
> >      select SIFIVE_TEST
> > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> > index d90286dc46..49cc7105af 100644
> > --- a/hw/riscv/virt.c
> > +++ b/hw/riscv/virt.c
> > @@ -32,6 +32,7 @@
> >  #include "hw/core/sysbus-fdt.h"
> >  #include "target/riscv/pmu.h"
> >  #include "hw/riscv/riscv_hart.h"
> > +#include "hw/riscv/iommu.h"
> >  #include "hw/riscv/virt.h"
> >  #include "hw/riscv/boot.h"
> >  #include "hw/riscv/numa.h"
> > @@ -88,7 +89,8 @@ static const MemMapEntry virt_memmap[] = {
> >      [VIRT_APLIC_M] =      {  0xc000000, APLIC_SIZE(VIRT_CPUS_MAX) },
> >      [VIRT_APLIC_S] =      {  0xd000000, APLIC_SIZE(VIRT_CPUS_MAX) },
> >      [VIRT_UART0] =        { 0x10000000,         0x100 },
> > -    [VIRT_VIRTIO] =       { 0x10001000,        0x1000 },
> > +    [VIRT_IOMMU] =        { 0x10001000,        0x1000 },
> > +    [VIRT_VIRTIO] =       { 0x10008000,        0x1000 }, /* VIRTIO_COUNT */
>
> We shouldn't change existing addresses
>

OK, I'll find another 4k window for IOMMU.

> Alistair
>
> >      [VIRT_FW_CFG] =       { 0x10100000,          0x18 },
> >      [VIRT_FLASH] =        { 0x20000000,     0x4000000 },
> >      [VIRT_IMSIC_M] =      { 0x24000000, VIRT_IMSIC_MAX_SIZE },
> > @@ -1019,6 +1021,44 @@ static void create_fdt_fw_cfg(RISCVVirtState *s, const MemMapEntry *memmap)
> >      g_free(nodename);
> >  }
> >
> > +static void create_fdt_iommu(RISCVVirtState *s, const MemMapEntry *memmap,
> > +    uint32_t irq_mmio_phandle)
> > +{
> > +    MachineState *ms = MACHINE(s);
> > +    uint32_t iommu_phandle;
> > +    const char *irq_names[] = { "cmdq", "fltq", "pm", "priq" };
> > +    char *iommu_node;
> > +    char *pci_node;
> > +
> > +    pci_node = g_strdup_printf("/soc/pci@%" PRIx64, memmap[VIRT_PCIE_ECAM].base);
> > +    iommu_node = g_strdup_printf("/soc/iommu@%" PRIx64, memmap[VIRT_IOMMU].base);
> > +
> > +    iommu_phandle = qemu_fdt_alloc_phandle(ms->fdt);
> > +    qemu_fdt_add_subnode(ms->fdt, iommu_node);
> > +    qemu_fdt_setprop_string(ms->fdt, iommu_node, "compatible", "riscv,iommu");
> > +    qemu_fdt_setprop_cell(ms->fdt, iommu_node, "#iommu-cells", 1);
> > +    qemu_fdt_setprop_cell(ms->fdt, iommu_node, "phandle", iommu_phandle);
> > +    qemu_fdt_setprop_cells(ms->fdt, iommu_node, "reg",
> > +        0x0, memmap[VIRT_IOMMU].base, 0x0, memmap[VIRT_IOMMU].size);
> > +    qemu_fdt_setprop_cell(ms->fdt, iommu_node, "interrupt-parent", irq_mmio_phandle);
> > +    qemu_fdt_setprop_string_array(ms->fdt, iommu_node, "interrupt-names",
> > +        (char **) &irq_names, ARRAY_SIZE(irq_names));
> > +    qemu_fdt_setprop_cells(ms->fdt, iommu_node, "interrupts",
> > +        IOMMU_IRQ + 0, 0x4,
> > +        IOMMU_IRQ + 1, 0x4,
> > +        IOMMU_IRQ + 2, 0x4,
> > +        IOMMU_IRQ + 3, 0x4);
> > +    qemu_fdt_setprop_cells(ms->fdt, pci_node, "iommu-map",
> > +        0x0, iommu_phandle, 0x0, 0xffff);
> > +    g_free(iommu_node);
> > +    g_free(pci_node);
> > +}
> > +
> > +static bool virt_is_iommu_enabled(RISCVVirtState *s)
> > +{
> > +    return s->iommu != ON_OFF_AUTO_OFF;
> > +}
> > +
> >  static void create_fdt(RISCVVirtState *s, const MemMapEntry *memmap)
> >  {
> >      MachineState *ms = MACHINE(s);
> > @@ -1051,6 +1091,10 @@ static void create_fdt(RISCVVirtState *s, const MemMapEntry *memmap)
> >
> >      create_fdt_pcie(s, memmap, irq_pcie_phandle, msi_pcie_phandle);
> >
> > +    if (virt_is_iommu_enabled(s)) {
> > +        create_fdt_iommu(s, memmap, irq_mmio_phandle);
> > +    }
> > +
> >      create_fdt_reset(s, memmap, &phandle);
> >
> >      create_fdt_uart(s, memmap, irq_mmio_phandle);
> > @@ -1210,6 +1254,31 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType aia_type, int aia_guests,
> >      return aplic_m;
> >  }
> >
> > +static DeviceState *virt_create_iommu(RISCVVirtState *s, DeviceState *irqchip)
> > +{
> > +    DeviceState *iommu;
> > +    int i;
> > +
> > +    iommu = qdev_new(TYPE_RISCV_IOMMU_SYS);
> > +
> > +    if (s->aia_type != VIRT_AIA_TYPE_APLIC_IMSIC) {
> > +        /* Disable MSI_FLAT [22], MSI_MRIF [23] if IMSIC is not enabled. */
> > +        qdev_prop_set_uint64(iommu, "capabilities", ~(BIT_ULL(22) | BIT_ULL(23)));
> > +    }
> > +
> > +    /* Fixed base register address */
> > +    qdev_prop_set_uint64(iommu, "addr", virt_memmap[VIRT_IOMMU].base);
> > +
> > +    sysbus_realize_and_unref(SYS_BUS_DEVICE(iommu), &error_fatal);
> > +
> > +    for (i = 0; i < 4; i++) {
> > +        sysbus_connect_irq(SYS_BUS_DEVICE(iommu), i,
> > +            qdev_get_gpio_in(irqchip, IOMMU_IRQ + i));
> > +    }
> > +
> > +    return iommu;
> > +}
> > +
> >  static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)
> >  {
> >      DeviceState *dev;
> > @@ -1506,6 +1575,10 @@ static void virt_machine_init(MachineState *machine)
> >
> >      create_platform_bus(s, mmio_irqchip);
> >
> > +    if (virt_is_iommu_enabled(s)) {
> > +        virt_create_iommu(s, mmio_irqchip);
> > +    }
> > +
> >      serial_mm_init(system_memory, memmap[VIRT_UART0].base,
> >          0, qdev_get_gpio_in(mmio_irqchip, UART0_IRQ), 399193,
> >          serial_hd(0), DEVICE_LITTLE_ENDIAN);
> > @@ -1533,6 +1606,7 @@ static void virt_machine_instance_init(Object *obj)
> >      s->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
> >      s->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
> >      s->acpi = ON_OFF_AUTO_AUTO;
> > +    s->iommu = ON_OFF_AUTO_AUTO;
> >  }
> >
> >  static char *virt_get_aia_guests(Object *obj, Error **errp)
> > @@ -1607,6 +1681,23 @@ static void virt_set_aclint(Object *obj, bool value, Error **errp)
> >      s->have_aclint = value;
> >  }
> >
> > +static void virt_get_iommu(Object *obj, Visitor *v, const char *name,
> > +                           void *opaque, Error **errp)
> > +{
> > +    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> > +    OnOffAuto iommu = s->iommu;
> > +
> > +    visit_type_OnOffAuto(v, name, &iommu, errp);
> > +}
> > +
> > +static void virt_set_iommu(Object *obj, Visitor *v, const char *name,
> > +                           void *opaque, Error **errp)
> > +{
> > +    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> > +
> > +    visit_type_OnOffAuto(v, name, &s->iommu, errp);
> > +}
> > +
> >  bool virt_is_acpi_enabled(RISCVVirtState *s)
> >  {
> >      return s->acpi != ON_OFF_AUTO_OFF;
> > @@ -1683,6 +1774,13 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
> >      machine_class_allow_dynamic_sysbus_dev(mc, TYPE_TPM_TIS_SYSBUS);
> >  #endif
> >
> > +    machine_class_allow_dynamic_sysbus_dev(mc, TYPE_RISCV_IOMMU_SYS);
> > +    object_class_property_add(oc, "iommu", "OnOffAuto",
> > +                              virt_get_iommu, virt_set_iommu,
> > +                              NULL, NULL);
> > +    object_class_property_set_description(oc, "iommu",
> > +        "Set on/off to enable/disable emulating RISC-V IOMMU platform device");
> > +
> >      if (tcg_enabled()) {
> >          object_class_property_add_bool(oc, "aclint", virt_get_aclint,
> >                                         virt_set_aclint);
> > diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h
> > index e5c474b26e..47b9a4f103 100644
> > --- a/include/hw/riscv/virt.h
> > +++ b/include/hw/riscv/virt.h
> > @@ -59,6 +59,7 @@ struct RISCVVirtState {
> >      char *oem_id;
> >      char *oem_table_id;
> >      OnOffAuto acpi;
> > +    OnOffAuto iommu;
> >      const MemMapEntry *memmap;
> >  };
> >
> > @@ -73,6 +74,7 @@ enum {
> >      VIRT_APLIC_M,
> >      VIRT_APLIC_S,
> >      VIRT_UART0,
> > +    VIRT_IOMMU,
> >      VIRT_VIRTIO,
> >      VIRT_FW_CFG,
> >      VIRT_IMSIC_M,
> > @@ -91,6 +93,7 @@ enum {
> >      VIRTIO_IRQ = 1, /* 1 to 8 */
> >      VIRTIO_COUNT = 8,
> >      PCIE_IRQ = 0x20, /* 32 to 35 */
> > +    IOMMU_IRQ = 60, /* 60 to 63 */
> >      VIRT_PLATFORM_BUS_IRQ = 64, /* 64 to 95 */
> >  };
> >
> > --
> > 2.34.1
> >
> >


best
- Tomasz