Re: [PATCH v3 03/26] ARM: GICv3 ITS: allocate device and collection table

[Stefano Stabellini <sstabellini@kernel.org>]

On Fri, 31 Mar 2017, Andre Przywara wrote:
> Each ITS maps a pair of a DeviceID (for instance derived from a PCI
> b/d/f triplet) and an EventID (the MSI payload or interrupt ID) to a
> pair of LPI number and collection ID, which points to the target CPU.
> This mapping is stored in the device and collection tables, which software
> has to provide for the ITS to use.
> Allocate the required memory and hand it to the ITS.
> The maximum number of devices is limited to a compile-time constant
> exposed in Kconfig.
> 
> Signed-off-by: Andre Przywara <andre.przywara@arm.com>

Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>


> ---
>  docs/misc/xen-command-line.markdown |   9 ++
>  xen/arch/arm/gic-v3-its.c           | 168 ++++++++++++++++++++++++++++++++++++
>  xen/arch/arm/gic-v3.c               |   3 +
>  xen/include/asm-arm/gic_v3_its.h    |  64 +++++++++++++-
>  4 files changed, 243 insertions(+), 1 deletion(-)
> 
> diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
> index 619016d..c67c925 100644
> --- a/docs/misc/xen-command-line.markdown
> +++ b/docs/misc/xen-command-line.markdown
> @@ -1158,6 +1158,15 @@ based interrupts. Any higher IRQs will be available for use via PCI MSI.
>  ### maxcpus
>  > `= <integer>`
>  
> +### max\_its\_device\_bits
> +> `= <integer>`
> +
> +Specifies the maximum number of devices using MSIs on the ARM GICv3 ITS
> +controller to allocate table entries for. Each table entry uses a hardware
> +specific size, typically 8 or 16 bytes. This value is given as the number
> +of bits required to hold one device ID.
> +Defaults to the machine provided value, which is at most 32 bits.
> +
>  ### max\_lpi\_bits
>  > `= <integer>`
>  
> diff --git a/xen/arch/arm/gic-v3-its.c b/xen/arch/arm/gic-v3-its.c
> index 4056e5b..bfdb7ac 100644
> --- a/xen/arch/arm/gic-v3-its.c
> +++ b/xen/arch/arm/gic-v3-its.c
> @@ -19,8 +19,10 @@
>   */
>  
>  #include <xen/lib.h>
> +#include <xen/mm.h>
>  #include <asm/gic_v3_defs.h>
>  #include <asm/gic_v3_its.h>
> +#include <asm/io.h>
>  
>  LIST_HEAD(host_its_list);
>  
> @@ -29,6 +31,172 @@ bool gicv3_its_host_has_its(void)
>      return !list_empty(&host_its_list);
>  }
>  
> +#define BASER_ATTR_MASK                                           \
> +        ((0x3UL << GITS_BASER_SHAREABILITY_SHIFT)               | \
> +         (0x7UL << GITS_BASER_OUTER_CACHEABILITY_SHIFT)         | \
> +         (0x7UL << GITS_BASER_INNER_CACHEABILITY_SHIFT))
> +#define BASER_RO_MASK   (GENMASK_ULL(58, 56) | GENMASK_ULL(52, 48))
> +
> +/* Check that the physical address can be encoded in the PROPBASER register. */
> +static bool check_baser_phys_addr(void *vaddr, unsigned int page_bits)
> +{
> +    paddr_t paddr = virt_to_maddr(vaddr);
> +
> +    return (!(paddr & ~GENMASK_ULL(page_bits < 16 ? 47 : 51, page_bits)));
> +}
> +
> +static uint64_t encode_propbaser_phys_addr(paddr_t addr, unsigned int page_bits)
> +{
> +    uint64_t ret = addr & GENMASK_ULL(47, page_bits);
> +
> +    if ( page_bits < 16 )
> +        return ret;
> +
> +    /* For 64K pages address bits 51-48 are encoded in bits 15-12. */
> +    return ret | ((addr & GENMASK_ULL(51, 48)) >> (48 - 12));
> +}
> +
> +/* The ITS BASE registers work with page sizes of 4K, 16K or 64K. */
> +#define BASER_PAGE_BITS(sz) ((sz) * 2 + 12)
> +
> +static int its_map_baser(void __iomem *basereg, uint64_t regc,
> +                         unsigned int nr_items)
> +{
> +    uint64_t attr, reg;
> +    unsigned int entry_size = GITS_BASER_ENTRY_SIZE(regc);
> +    unsigned int pagesz = 2;    /* try 64K pages first, then go down. */
> +    unsigned int table_size;
> +    void *buffer;
> +
> +    attr  = GIC_BASER_InnerShareable << GITS_BASER_SHAREABILITY_SHIFT;
> +    attr |= GIC_BASER_CACHE_SameAsInner << GITS_BASER_OUTER_CACHEABILITY_SHIFT;
> +    attr |= GIC_BASER_CACHE_RaWaWb << GITS_BASER_INNER_CACHEABILITY_SHIFT;
> +
> +    /*
> +     * Setup the BASE register with the attributes that we like. Then read
> +     * it back and see what sticks (page size, cacheability and shareability
> +     * attributes), retrying if necessary.
> +     */
> +retry:
> +    table_size = ROUNDUP(nr_items * entry_size, BIT(BASER_PAGE_BITS(pagesz)));
> +    /* The BASE registers support at most 256 pages. */
> +    table_size = min(table_size, 256U << BASER_PAGE_BITS(pagesz));
> +
> +    buffer = _xzalloc(table_size, BIT(BASER_PAGE_BITS(pagesz)));
> +    if ( !buffer )
> +        return -ENOMEM;
> +
> +    if ( !check_baser_phys_addr(buffer, BASER_PAGE_BITS(pagesz)) )
> +    {
> +        xfree(buffer);
> +        return -ERANGE;
> +    }
> +
> +    reg  = attr;
> +    reg |= (pagesz << GITS_BASER_PAGE_SIZE_SHIFT);
> +    reg |= (table_size >> BASER_PAGE_BITS(pagesz)) - 1;
> +    reg |= regc & BASER_RO_MASK;
> +    reg |= GITS_VALID_BIT;
> +    reg |= encode_propbaser_phys_addr(virt_to_maddr(buffer),
> +                                      BASER_PAGE_BITS(pagesz));
> +
> +    writeq_relaxed(reg, basereg);
> +    regc = readq_relaxed(basereg);
> +
> +    /* The host didn't like our attributes, just use what it returned. */
> +    if ( (regc & BASER_ATTR_MASK) != attr )
> +    {
> +        /* If we can't map it shareable, drop cacheability as well. */
> +        if ( (regc & GITS_BASER_SHAREABILITY_MASK) == GIC_BASER_NonShareable )
> +        {
> +            regc &= ~GITS_BASER_INNER_CACHEABILITY_MASK;
> +            writeq_relaxed(regc, basereg);
> +        }
> +        attr = regc & BASER_ATTR_MASK;
> +    }
> +    if ( (regc & GITS_BASER_INNER_CACHEABILITY_MASK) <= GIC_BASER_CACHE_nC )
> +        clean_and_invalidate_dcache_va_range(buffer, table_size);
> +
> +    /* If the host accepted our page size, we are done. */
> +    if ( ((regc >> GITS_BASER_PAGE_SIZE_SHIFT) & 0x3UL) == pagesz )
> +        return 0;
> +
> +    xfree(buffer);
> +
> +    if ( pagesz-- > 0 )
> +        goto retry;
> +
> +    /* None of the page sizes was accepted, give up */
> +    return -EINVAL;
> +}
> +
> +/* Allow a user to limit the number of devices. */
> +static unsigned int max_its_device_bits = 32;
> +integer_param("max_its_device_bits", max_its_device_bits);
> +
> +static int gicv3_its_init_single_its(struct host_its *hw_its)
> +{
> +    uint64_t reg;
> +    int i, ret;
> +
> +    hw_its->its_base = ioremap_nocache(hw_its->addr, hw_its->size);
> +    if ( !hw_its->its_base )
> +        return -ENOMEM;
> +
> +    reg = readq_relaxed(hw_its->its_base + GITS_TYPER);
> +    hw_its->devid_bits = GITS_TYPER_DEVICE_ID_BITS(reg);
> +    hw_its->devid_bits = min(hw_its->devid_bits, max_its_device_bits);
> +
> +    for ( i = 0; i < GITS_BASER_NR_REGS; i++ )
> +    {
> +        void __iomem *basereg = hw_its->its_base + GITS_BASER0 + i * 8;
> +        unsigned int type;
> +
> +        reg = readq_relaxed(basereg);
> +        type = (reg & GITS_BASER_TYPE_MASK) >> GITS_BASER_TYPE_SHIFT;
> +        switch ( type )
> +        {
> +        case GITS_BASER_TYPE_NONE:
> +            continue;
> +        case GITS_BASER_TYPE_DEVICE:
> +            ret = its_map_baser(basereg, reg, BIT(hw_its->devid_bits));
> +            if ( ret )
> +                return ret;
> +            break;
> +        case GITS_BASER_TYPE_COLLECTION:
> +            ret = its_map_baser(basereg, reg, num_possible_cpus());
> +            if ( ret )
> +                return ret;
> +            break;
> +        /* In case this is a GICv4, provide a (dummy) vPE table as well. */
> +        case GITS_BASER_TYPE_VCPU:
> +            ret = its_map_baser(basereg, reg, 1);
> +            if ( ret )
> +                return ret;
> +            break;
> +        default:
> +            continue;
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +int gicv3_its_init(void)
> +{
> +    struct host_its *hw_its;
> +    int ret;
> +
> +    list_for_each_entry(hw_its, &host_its_list, entry)
> +    {
> +        ret = gicv3_its_init_single_its(hw_its);
> +        if ( ret )
> +            return ret;
> +    }
> +
> +    return 0;
> +}
> +
>  /* Scan the DT for any ITS nodes and create a list of host ITSes out of it. */
>  void gicv3_its_dt_init(const struct dt_device_node *node)
>  {
> diff --git a/xen/arch/arm/gic-v3.c b/xen/arch/arm/gic-v3.c
> index 36cd269..b84bc40 100644
> --- a/xen/arch/arm/gic-v3.c
> +++ b/xen/arch/arm/gic-v3.c
> @@ -1590,6 +1590,9 @@ static int __init gicv3_init(void)
>      spin_lock(&gicv3.lock);
>  
>      gicv3_dist_init();
> +    res = gicv3_its_init();
> +    if ( res )
> +        panic("GICv3: ITS: initialization failed: %d\n", res);
>      res = gicv3_cpu_init();
>      gicv3_hyp_init();
>  
> diff --git a/xen/include/asm-arm/gic_v3_its.h b/xen/include/asm-arm/gic_v3_its.h
> index 219d109..badb644 100644
> --- a/xen/include/asm-arm/gic_v3_its.h
> +++ b/xen/include/asm-arm/gic_v3_its.h
> @@ -20,6 +20,60 @@
>  #ifndef __ASM_ARM_ITS_H__
>  #define __ASM_ARM_ITS_H__
>  
> +#define GITS_CTLR                       0x000
> +#define GITS_IIDR                       0x004
> +#define GITS_TYPER                      0x008
> +#define GITS_CBASER                     0x080
> +#define GITS_CWRITER                    0x088
> +#define GITS_CREADR                     0x090
> +#define GITS_BASER_NR_REGS              8
> +#define GITS_BASER0                     0x100
> +#define GITS_BASER1                     0x108
> +#define GITS_BASER2                     0x110
> +#define GITS_BASER3                     0x118
> +#define GITS_BASER4                     0x120
> +#define GITS_BASER5                     0x128
> +#define GITS_BASER6                     0x130
> +#define GITS_BASER7                     0x138
> +
> +/* Register bits */
> +#define GITS_VALID_BIT                  BIT_ULL(63)
> +
> +#define GITS_CTLR_QUIESCENT             BIT(31)
> +#define GITS_CTLR_ENABLE                BIT(0)
> +
> +#define GITS_TYPER_DEVIDS_SHIFT         13
> +#define GITS_TYPER_DEVIDS_MASK          (0x1fUL << GITS_TYPER_DEVIDS_SHIFT)
> +#define GITS_TYPER_DEVICE_ID_BITS(r)    (((r & GITS_TYPER_DEVIDS_MASK) >> \
> +                                               GITS_TYPER_DEVIDS_SHIFT) + 1)
> +
> +#define GITS_IIDR_VALUE                 0x34c
> +
> +#define GITS_BASER_INDIRECT             BIT_ULL(62)
> +#define GITS_BASER_INNER_CACHEABILITY_SHIFT        59
> +#define GITS_BASER_TYPE_SHIFT           56
> +#define GITS_BASER_TYPE_MASK            (7ULL << GITS_BASER_TYPE_SHIFT)
> +#define GITS_BASER_OUTER_CACHEABILITY_SHIFT        53
> +#define GITS_BASER_TYPE_NONE            0UL
> +#define GITS_BASER_TYPE_DEVICE          1UL
> +#define GITS_BASER_TYPE_VCPU            2UL
> +#define GITS_BASER_TYPE_CPU             3UL
> +#define GITS_BASER_TYPE_COLLECTION      4UL
> +#define GITS_BASER_TYPE_RESERVED5       5UL
> +#define GITS_BASER_TYPE_RESERVED6       6UL
> +#define GITS_BASER_TYPE_RESERVED7       7UL
> +#define GITS_BASER_ENTRY_SIZE_SHIFT     48
> +#define GITS_BASER_ENTRY_SIZE(reg)                                       \
> +                        (((reg >> GITS_BASER_ENTRY_SIZE_SHIFT) & 0x1f) + 1)
> +#define GITS_BASER_SHAREABILITY_SHIFT   10
> +#define GITS_BASER_PAGE_SIZE_SHIFT      8
> +#define GITS_BASER_RO_MASK              (GITS_BASER_TYPE_MASK | \
> +                                        (31UL << GITS_BASER_ENTRY_SIZE_SHIFT) |\
> +                                        GITS_BASER_INDIRECT)
> +#define GITS_BASER_SHAREABILITY_MASK   (0x3ULL << GITS_BASER_SHAREABILITY_SHIFT)
> +#define GITS_BASER_OUTER_CACHEABILITY_MASK   (0x7ULL << GITS_BASER_OUTER_CACHEABILITY_SHIFT)
> +#define GITS_BASER_INNER_CACHEABILITY_MASK   (0x7ULL << GITS_BASER_INNER_CACHEABILITY_SHIFT)
> +
>  #include <xen/device_tree.h>
>  
>  /* data structure for each hardware ITS */
> @@ -28,6 +82,8 @@ struct host_its {
>      const struct dt_device_node *dt_node;
>      paddr_t addr;
>      paddr_t size;
> +    void __iomem *its_base;
> +    unsigned int devid_bits;
>  };
>  
>  
> @@ -42,8 +98,9 @@ bool gicv3_its_host_has_its(void);
>  
>  int gicv3_lpi_init_rdist(void __iomem * rdist_base);
>  
> -/* Initialize the host structures for LPIs. */
> +/* Initialize the host structures for LPIs and the host ITSes. */
>  int gicv3_lpi_init_host_lpis(unsigned int nr_lpis);
> +int gicv3_its_init(void);
>  
>  #else
>  
> @@ -67,6 +124,11 @@ static inline int gicv3_lpi_init_host_lpis(unsigned int nr_lpis)
>  {
>      return 0;
>  }
> +
> +static inline int gicv3_its_init(void)
> +{
> +    return 0;
> +}
>  #endif /* CONFIG_HAS_ITS */
>  
>  #endif
> -- 
> 2.9.0
> 

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel