Re: [Qemu-devel] [Qemu-ppc] [PATCH v3 1/6] libqos: add PPC64 PCI support

[David Gibson <david@gibson.dropbear.id.au>]

[-- Attachment #1: Type: text/plain, Size: 17020 bytes --]

On Wed, Sep 28, 2016 at 08:51:28PM +0200, Laurent Vivier wrote:
> Signed-off-by: Laurent Vivier <lvivier@redhat.com>
> ---
>  tests/Makefile.include   |   1 +
>  tests/libqos/pci-pc.c    |  22 ----
>  tests/libqos/pci-spapr.c | 280 +++++++++++++++++++++++++++++++++++++++++++++++
>  tests/libqos/pci-spapr.h |  17 +++
>  tests/libqos/pci.c       |  22 +++-
>  tests/libqos/rtas.c      |  45 ++++++++
>  tests/libqos/rtas.h      |   4 +
>  7 files changed, 368 insertions(+), 23 deletions(-)
>  create mode 100644 tests/libqos/pci-spapr.c
>  create mode 100644 tests/libqos/pci-spapr.h
> 
> diff --git a/tests/Makefile.include b/tests/Makefile.include
> index 8162f6f..92c82d8 100644
> --- a/tests/Makefile.include
> +++ b/tests/Makefile.include
> @@ -590,6 +590,7 @@ libqos-obj-y += tests/libqos/i2c.o tests/libqos/libqos.o
>  libqos-spapr-obj-y = $(libqos-obj-y) tests/libqos/malloc-spapr.o
>  libqos-spapr-obj-y += tests/libqos/libqos-spapr.o
>  libqos-spapr-obj-y += tests/libqos/rtas.o
> +libqos-spapr-obj-y += tests/libqos/pci-spapr.o
>  libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
>  libqos-pc-obj-y += tests/libqos/malloc-pc.o tests/libqos/libqos-pc.o
>  libqos-pc-obj-y += tests/libqos/ahci.o
> diff --git a/tests/libqos/pci-pc.c b/tests/libqos/pci-pc.c
> index 1ae2d37..82066b8 100644
> --- a/tests/libqos/pci-pc.c
> +++ b/tests/libqos/pci-pc.c
> @@ -255,28 +255,6 @@ void qpci_free_pc(QPCIBus *bus)
>      g_free(s);
>  }
>  
> -void qpci_plug_device_test(const char *driver, const char *id,
> -                           uint8_t slot, const char *opts)
> -{
> -    QDict *response;
> -    char *cmd;
> -
> -    cmd = g_strdup_printf("{'execute': 'device_add',"
> -                          " 'arguments': {"
> -                          "   'driver': '%s',"
> -                          "   'addr': '%d',"
> -                          "   %s%s"
> -                          "   'id': '%s'"
> -                          "}}", driver, slot,
> -                          opts ? opts : "", opts ? "," : "",
> -                          id);
> -    response = qmp(cmd);
> -    g_free(cmd);
> -    g_assert(response);
> -    g_assert(!qdict_haskey(response, "error"));
> -    QDECREF(response);
> -}
> -
>  void qpci_unplug_acpi_device_test(const char *id, uint8_t slot)
>  {
>      QDict *response;
> diff --git a/tests/libqos/pci-spapr.c b/tests/libqos/pci-spapr.c
> new file mode 100644
> index 0000000..78df823
> --- /dev/null
> +++ b/tests/libqos/pci-spapr.c
> @@ -0,0 +1,280 @@
> +/*
> + * libqos PCI bindings for SPAPR
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + */
> +
> +#include "qemu/osdep.h"
> +#include "libqtest.h"
> +#include "libqos/pci-spapr.h"
> +#include "libqos/rtas.h"
> +
> +#include "hw/pci/pci_regs.h"
> +
> +#include "qemu-common.h"
> +#include "qemu/host-utils.h"
> +
> +
> +/* From include/hw/pci-host/spapr.h */
> +
> +#define SPAPR_PCI_BASE_BUID          0x800000020000000ULL
> +
> +#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
> +
> +#define SPAPR_PCI_WINDOW_BASE        0x10000000000ULL
> +#define SPAPR_PCI_WINDOW_SPACING     0x1000000000ULL
> +#define SPAPR_PCI_MMIO_WIN_OFF       0xA0000000
> +#define SPAPR_PCI_MMIO_WIN_SIZE      (SPAPR_PCI_WINDOW_SPACING - \
> +                                     SPAPR_PCI_MEM_WIN_BUS_OFFSET)
> +#define SPAPR_PCI_IO_WIN_OFF         0x80000000
> +#define SPAPR_PCI_IO_WIN_SIZE        0x10000
> +
> +/* index is the phb index */
> +
> +#define BUIDBASE(index)              (SPAPR_PCI_BASE_BUID + (index))
> +#define PCIBASE(index)               (SPAPR_PCI_WINDOW_BASE + \
> +                                      (index) * SPAPR_PCI_WINDOW_SPACING)
> +#define IOBASE(index)                (PCIBASE(index) + SPAPR_PCI_IO_WIN_OFF)
> +#define MMIOBASE(index)              (PCIBASE(index) + SPAPR_PCI_MMIO_WIN_OFF)
> +
> +typedef struct QPCIBusSPAPR {
> +    QPCIBus bus;
> +    QGuestAllocator *alloc;
> +
> +    uint64_t pci_hole_start;
> +    uint64_t pci_hole_size;
> +    uint64_t pci_hole_alloc;
> +
> +    uint32_t pci_iohole_start;
> +    uint32_t pci_iohole_size;
> +    uint32_t pci_iohole_alloc;
> +} QPCIBusSPAPR;
> +
> +static uint8_t qpci_spapr_io_readb(QPCIBus *bus, void *addr)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    uint8_t v;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        v = readb(IOBASE(0) + port);
> +    } else {
> +        v = readb(MMIOBASE(0) + port);
> +    }
> +    return v;
> +}
> +
> +static uint16_t qpci_spapr_io_readw(QPCIBus *bus, void *addr)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    uint16_t v;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        v = readw(IOBASE(0) + port);
> +    } else {
> +        v = readw(MMIOBASE(0) + port);
> +    }

Ok, so, I've looked through the qtest stuff in more detail now, and
I've got a better idea of how the endianness works.  Some of my
earlier comments were confused about which pieces were in the test
case side and which were on the qtest accel stub side.

So, what I think you need is an unconditional byteswap in each of the
spapr pci IO functions.  Why?

   * The plain readw() etc. functions return values read from memory
     assuming guest endianness.  For guest native values in memory, so
     far so good.
   * Generic users of the pci read/write functions in qtest expect to
     get native values.
   * But PCI devices are always LE, not guest endian

The guest endianness of spapr (as far as tswap() etc. are concerned)
is BE, even though that's not really true in practice these days, so
to correct for the PCI registers being read as BE when they should be
LE we always need a swap.

That should remove the need for swaps further up the test stack.

Of course, "guest endianness" is a poorly defined concept, so longer
term it might be better to completely replace the "readw" etc. qtest
operations with explicit "readw_le" and "readw_be" ops.

> +    return v;
> +}
> +
> +static uint32_t qpci_spapr_io_readl(QPCIBus *bus, void *addr)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    uint32_t v;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        v = readl(IOBASE(0) + port);
> +    } else {
> +        v = readl(MMIOBASE(0) + port);
> +    }
> +    return v;
> +}
> +
> +static void qpci_spapr_io_writeb(QPCIBus *bus, void *addr, uint8_t value)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        writeb(IOBASE(0) + port, value);
> +    } else {
> +        writeb(MMIOBASE(0) + port, value);
> +    }
> +}
> +
> +static void qpci_spapr_io_writew(QPCIBus *bus, void *addr, uint16_t value)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        writew(IOBASE(0) + port, value);
> +    } else {
> +        writew(MMIOBASE(0) + port, value);
> +    }
> +}
> +
> +static void qpci_spapr_io_writel(QPCIBus *bus, void *addr, uint32_t value)
> +{
> +    uint64_t port = (uint64_t)addr;
> +    if (port < SPAPR_PCI_IO_WIN_SIZE) {
> +        writel(IOBASE(0) + port, value);
> +    } else {
> +        writel(MMIOBASE(0) + port, value);
> +    }
> +}
> +
> +static uint8_t qpci_spapr_config_readb(QPCIBus *bus, int devfn, uint8_t offset)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    return qrtas_ibm_read_pci_config(s->alloc, BUIDBASE(0),
> +                                     config_addr, 1);
> +}
> +
> +static uint16_t qpci_spapr_config_readw(QPCIBus *bus, int devfn, uint8_t offset)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    return qrtas_ibm_read_pci_config(s->alloc, BUIDBASE(0),
> +                                     config_addr, 2);
> +}
> +
> +static uint32_t qpci_spapr_config_readl(QPCIBus *bus, int devfn, uint8_t offset)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    return qrtas_ibm_read_pci_config(s->alloc, BUIDBASE(0),
> +                                     config_addr, 4);
> +}
> +
> +static void qpci_spapr_config_writeb(QPCIBus *bus, int devfn, uint8_t offset,
> +                                     uint8_t value)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    qrtas_ibm_write_pci_config(s->alloc, BUIDBASE(0),
> +                               config_addr, 1, value);
> +}
> +
> +static void qpci_spapr_config_writew(QPCIBus *bus, int devfn, uint8_t offset,
> +                                     uint16_t value)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    qrtas_ibm_write_pci_config(s->alloc, BUIDBASE(0),
> +                               config_addr, 2, value);
> +}
> +
> +static void qpci_spapr_config_writel(QPCIBus *bus, int devfn, uint8_t offset,
> +                                     uint32_t value)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    uint32_t config_addr = (devfn << 8) | offset;
> +    qrtas_ibm_write_pci_config(s->alloc, BUIDBASE(0),
> +                               config_addr, 4, value);
> +}
> +
> +static void *qpci_spapr_iomap(QPCIBus *bus, QPCIDevice *dev, int barno,
> +                              uint64_t *sizeptr)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +    static const int bar_reg_map[] = {
> +        PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2,
> +        PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_5,
> +    };
> +    int bar_reg;
> +    uint32_t addr;
> +    uint64_t size;
> +    uint32_t io_type;
> +
> +    g_assert(barno >= 0 && barno <= 5);
> +    bar_reg = bar_reg_map[barno];
> +
> +    qpci_config_writel(dev, bar_reg, 0xFFFFFFFF);
> +    addr = qpci_config_readl(dev, bar_reg);
> +
> +    io_type = addr & PCI_BASE_ADDRESS_SPACE;
> +    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> +        addr &= PCI_BASE_ADDRESS_IO_MASK;
> +    } else {
> +        addr &= PCI_BASE_ADDRESS_MEM_MASK;
> +    }
> +
> +    size = (1ULL << ctzl(addr));
> +    if (size == 0) {
> +        return NULL;
> +    }
> +    if (sizeptr) {
> +        *sizeptr = size;
> +    }
> +
> +    if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
> +        uint16_t loc;
> +
> +        g_assert(QEMU_ALIGN_UP(s->pci_iohole_alloc, size) + size
> +                 <= s->pci_iohole_size);
> +        s->pci_iohole_alloc = QEMU_ALIGN_UP(s->pci_iohole_alloc, size);
> +        loc = s->pci_iohole_start + s->pci_iohole_alloc;
> +        s->pci_iohole_alloc += size;
> +
> +        qpci_config_writel(dev, bar_reg, loc | PCI_BASE_ADDRESS_SPACE_IO);
> +
> +        return (void *)(unsigned long)loc;
> +    } else {
> +        uint64_t loc;
> +
> +        g_assert(QEMU_ALIGN_UP(s->pci_hole_alloc, size) + size
> +                 <= s->pci_hole_size);
> +        s->pci_hole_alloc = QEMU_ALIGN_UP(s->pci_hole_alloc, size);
> +        loc = s->pci_hole_start + s->pci_hole_alloc;
> +        s->pci_hole_alloc += size;
> +
> +        qpci_config_writel(dev, bar_reg, loc);
> +
> +        return (void *)(unsigned long)loc;
> +    }
> +}
> +
> +static void qpci_spapr_iounmap(QPCIBus *bus, void *data)
> +{
> +    /* FIXME */
> +}
> +
> +QPCIBus *qpci_init_spapr(QGuestAllocator *alloc)
> +{
> +    QPCIBusSPAPR *ret;
> +
> +    ret = g_malloc(sizeof(*ret));
> +
> +    ret->alloc = alloc;
> +
> +    ret->bus.io_readb = qpci_spapr_io_readb;
> +    ret->bus.io_readw = qpci_spapr_io_readw;
> +    ret->bus.io_readl = qpci_spapr_io_readl;
> +
> +    ret->bus.io_writeb = qpci_spapr_io_writeb;
> +    ret->bus.io_writew = qpci_spapr_io_writew;
> +    ret->bus.io_writel = qpci_spapr_io_writel;
> +
> +    ret->bus.config_readb = qpci_spapr_config_readb;
> +    ret->bus.config_readw = qpci_spapr_config_readw;
> +    ret->bus.config_readl = qpci_spapr_config_readl;
> +
> +    ret->bus.config_writeb = qpci_spapr_config_writeb;
> +    ret->bus.config_writew = qpci_spapr_config_writew;
> +    ret->bus.config_writel = qpci_spapr_config_writel;
> +
> +    ret->bus.iomap = qpci_spapr_iomap;
> +    ret->bus.iounmap = qpci_spapr_iounmap;
> +
> +    ret->pci_hole_start = 0xC0000000;
> +    ret->pci_hole_size = SPAPR_PCI_MMIO_WIN_SIZE;
> +    ret->pci_hole_alloc = 0;
> +
> +    ret->pci_iohole_start = 0xc000;
> +    ret->pci_iohole_size = SPAPR_PCI_IO_WIN_SIZE;
> +    ret->pci_iohole_alloc = 0;
> +
> +    return &ret->bus;
> +}
> +
> +void qpci_free_spapr(QPCIBus *bus)
> +{
> +    QPCIBusSPAPR *s = container_of(bus, QPCIBusSPAPR, bus);
> +
> +    g_free(s);
> +}
> diff --git a/tests/libqos/pci-spapr.h b/tests/libqos/pci-spapr.h
> new file mode 100644
> index 0000000..4192126
> --- /dev/null
> +++ b/tests/libqos/pci-spapr.h
> @@ -0,0 +1,17 @@
> +/*
> + * libqos PCI bindings for SPAPR
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + */
> +
> +#ifndef LIBQOS_PCI_SPAPR_H
> +#define LIBQOS_PCI_SPAPR_H
> +
> +#include "libqos/malloc.h"
> +#include "libqos/pci.h"
> +
> +QPCIBus *qpci_init_spapr(QGuestAllocator *alloc);
> +void     qpci_free_spapr(QPCIBus *bus);
> +
> +#endif
> diff --git a/tests/libqos/pci.c b/tests/libqos/pci.c
> index ed78d91..c3f3382 100644
> --- a/tests/libqos/pci.c
> +++ b/tests/libqos/pci.c
> @@ -263,4 +263,24 @@ void qpci_iounmap(QPCIDevice *dev, void *data)
>      dev->bus->iounmap(dev->bus, data);
>  }
>  
> -
> +void qpci_plug_device_test(const char *driver, const char *id,
> +                           uint8_t slot, const char *opts)
> +{
> +    QDict *response;
> +    char *cmd;
> +
> +    cmd = g_strdup_printf("{'execute': 'device_add',"
> +                          " 'arguments': {"
> +                          "   'driver': '%s',"
> +                          "   'addr': '%d',"
> +                          "   %s%s"
> +                          "   'id': '%s'"
> +                          "}}", driver, slot,
> +                          opts ? opts : "", opts ? "," : "",
> +                          id);
> +    response = qmp(cmd);
> +    g_free(cmd);
> +    g_assert(response);
> +    g_assert(!qdict_haskey(response, "error"));
> +    QDECREF(response);
> +}
> diff --git a/tests/libqos/rtas.c b/tests/libqos/rtas.c
> index 820321a..0269803 100644
> --- a/tests/libqos/rtas.c
> +++ b/tests/libqos/rtas.c
> @@ -69,3 +69,48 @@ int qrtas_get_time_of_day(QGuestAllocator *alloc, struct tm *tm, uint32_t *ns)
>  
>      return res;
>  }
> +
> +uint32_t qrtas_ibm_read_pci_config(QGuestAllocator *alloc, uint64_t buid,
> +                                   uint32_t addr, uint32_t size)
> +{
> +    int res;
> +    uint32_t args[4], ret[2];
> +
> +    args[0] = addr;
> +    args[1] = buid >> 32;
> +    args[2] = buid & 0xffffffff;
> +    args[3] = size;
> +    res = qrtas_call(alloc, "ibm,read-pci-config", 4, args, 2, ret);
> +    if (res != 0) {
> +        return -1;
> +    }
> +
> +    if (ret[0] != 0) {
> +        return -1;
> +    }
> +
> +    return ret[1];
> +}
> +
> +int qrtas_ibm_write_pci_config(QGuestAllocator *alloc, uint64_t buid,
> +                               uint32_t addr, uint32_t size, uint32_t val)
> +{
> +    int res;
> +    uint32_t args[5], ret[1];
> +
> +    args[0] = addr;
> +    args[1] = buid >> 32;
> +    args[2] = buid & 0xffffffff;
> +    args[3] = size;
> +    args[4] = val;
> +    res = qrtas_call(alloc, "ibm,write-pci-config", 5, args, 1, ret);
> +    if (res != 0) {
> +        return -1;
> +    }
> +
> +    if (ret[0] != 0) {
> +        return -1;
> +    }
> +
> +    return 0;
> +}
> diff --git a/tests/libqos/rtas.h b/tests/libqos/rtas.h
> index a1b60a8..498eb19 100644
> --- a/tests/libqos/rtas.h
> +++ b/tests/libqos/rtas.h
> @@ -8,4 +8,8 @@
>  #include "libqos/malloc.h"
>  
>  int qrtas_get_time_of_day(QGuestAllocator *alloc, struct tm *tm, uint32_t *ns);
> +uint32_t qrtas_ibm_read_pci_config(QGuestAllocator *alloc, uint64_t buid,
> +                                   uint32_t addr, uint32_t size);
> +int qrtas_ibm_write_pci_config(QGuestAllocator *alloc, uint64_t buid,
> +                               uint32_t addr, uint32_t size, uint32_t val);
>  #endif /* LIBQOS_RTAS_H */

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]