Re: [PATCH v5 10/22] xen/arm: ITS: Add GITS registers emulation

[Vijay Kilari <vijay.kilari@gmail.com>]

On Wed, Jul 29, 2015 at 12:31 AM, Julien Grall <julien.grall@citrix.com> wrote:
> Hi Vijay,
>
> On 27/07/15 12:11, vijay.kilari@gmail.com wrote:
>> From: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>>
>> Emulate GITS* registers
>>
>> Signed-off-by: Vijaya Kumar K <Vijaya.Kumar@caviumnetworks.com>
>> ---
>> v4: - Removed GICR register emulation
>> ---
>>  xen/arch/arm/irq.c            |    3 +
>>  xen/arch/arm/vgic-v3-its.c    |  365 ++++++++++++++++++++++++++++++++++++++++-
>>  xen/include/asm-arm/gic-its.h |   15 ++
>>  xen/include/asm-arm/gic.h     |    1 +
>>  4 files changed, 381 insertions(+), 3 deletions(-)
>>
>> diff --git a/xen/arch/arm/irq.c b/xen/arch/arm/irq.c
>> index 1f38605..85cacb0 100644
>> --- a/xen/arch/arm/irq.c
>> +++ b/xen/arch/arm/irq.c
>> @@ -31,6 +31,9 @@
>>  static unsigned int local_irqs_type[NR_LOCAL_IRQS];
>>  static DEFINE_SPINLOCK(local_irqs_type_lock);
>>
>> +/* Number of LPI supported in XEN */
>> +unsigned int num_of_lpis = 8192;
>> +
>
> It makes little sense to introduce the support of LPIs in Xen in a patch
> called "Add GITS registers emulation".
>
> This should go in a specific ITS (not vITS) patch.
>
> Furthermore, you need to explain where to the 8192 comes from...

Two reasons for setting to 8192

1) Being LPI starts from 8192 the 8192 is 13 and next if id_bits is 14
then it can hold
    upto 16K. So 16K-8K = 8K
2) ThunderX requires more than 4K due to large number of PCIe devices

>
> Lastly I would rename num_of_lpis into nr_lpis.
>
>>  /* Describe an IRQ assigned to a guest */
>>  struct irq_guest
>>  {
>> diff --git a/xen/arch/arm/vgic-v3-its.c b/xen/arch/arm/vgic-v3-its.c
>> index 3a003d4..1c7d9b6 100644
>> --- a/xen/arch/arm/vgic-v3-its.c
>> +++ b/xen/arch/arm/vgic-v3-its.c
>> @@ -33,8 +33,16 @@
>>  #include <asm/atomic.h>
>>  #include <xen/log2.h>
>>
>> -#define DEBUG_ITS
>> -
>> +//#define DEBUG_ITS
>> +
>
> This change should go in patch #8.
>
>> +/* GITS_PIDRn register values for ARM implementations */
>> +#define GITS_PIDR0_VAL               (0x94)
>> +#define GITS_PIDR1_VAL               (0xb4)
>> +#define GITS_PIDR2_VAL               (0x3b)
>> +#define GITS_PIDR3_VAL               (0x00)
>> +#define GITS_PIDR4_VAL               (0x04)
>> +#define GITS_BASER_INIT_VAL          ((1UL << GITS_BASER_TYPE_SHIFT) | \
>> +                                     (0x7UL << GITS_BASER_ENTRY_SIZE_SHIFT))
>>  #ifdef DEBUG_ITS
>>  # define DPRINTK(fmt, args...) dprintk(XENLOG_DEBUG, fmt, ##args)
>>  #else
>> @@ -60,6 +68,14 @@ void vits_setup_hw(struct gic_its_info *its_info)
>>      vits_hw.info = its_info;
>>  }
>>
>> +static inline uint32_t vits_get_max_collections(struct domain *d)
>> +{
>> +    /* Collection ID is only 16 bit */
>
> 16 bit = 65536 not 256.

 I will correct the comment

>
> You need to explain that the ITS is only supporting 256 collections in
> hardware and that our implementation doesn't support memory provisioning
> for collection.
>
> Furthermore if the number of collection is based on 16 bits, the
> function should return uint16_t not uint32_t.
>
>
>> +    ASSERT(d->max_vcpus < 256);
>> +
>
> Please add a comment to explain why d->max_vcpus + 1 with may a
> reference to the public spec.
>
>> +    return (d->max_vcpus + 1);
>> +}
>> +
>>  static int vits_access_guest_table(struct domain *d, paddr_t entry, void *addr,
>>                                     uint32_t size, bool_t set)
>>  {
>> @@ -502,7 +518,7 @@ static int vits_read_virt_cmd(struct vcpu *v, struct vgic_its *vits,
>>      return 0;
>>  }
>>
>> -int vits_process_cmd(struct vcpu *v, struct vgic_its *vits)
>> +static int vits_process_cmd(struct vcpu *v, struct vgic_its *vits)
>
> Please, Move the static where the function has been defined.
>
>>  {
>>      its_cmd_block virt_cmd;
>>
>> @@ -527,11 +543,338 @@ err:
>>      return 0;
>>  }
>
> [..]
>
>> +static int vgic_v3_gits_mmio_read(struct vcpu *v, mmio_info_t *info)
>> +{
>> +    struct vgic_its *vits = v->domain->arch.vgic.vits;
>> +    struct hsr_dabt dabt = info->dabt;
>> +    struct cpu_user_regs *regs = guest_cpu_user_regs();
>> +    register_t *r = select_user_reg(regs, dabt.reg);
>> +    uint64_t val = 0;
>> +    uint32_t gits_reg;
>> +
>> +    gits_reg = info->gpa - vits->gits_base;
>> +    DPRINTK("%pv: vITS: GITS_MMIO_READ offset 0x%"PRIx32"\n", v, gits_reg);
>> +
>> +    switch ( gits_reg )
>> +    {
>> +    case GITS_CTLR:
>> +        if ( dabt.size != DABT_WORD ) goto bad_width;
>> +        vits_spin_lock(vits);
>> +        *r = vits->ctrl | GITS_CTLR_QUIESCENT;
>
> Why did you put GITS_CTLR_QUIESCENT?

The ITS is quiescent, has no translations in progress and has
completed all operations.
So I have set quescent by default.

[...]

>
>> +        return 1;
>> +    case GITS_BASER1 ... GITS_BASERN:
>> +        goto read_as_zero;
>> +    case GITS_PIDR0:
>> +        if ( dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        *r = GITS_PIDR0_VAL;
>> +        return 1;
>> +    case GITS_PIDR1:
>> +        if ( dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        *r = GITS_PIDR1_VAL;
>> +        return 1;
>> +    case GITS_PIDR2:
>> +        if ( dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        *r = GITS_PIDR2_VAL;
>> +        return 1;
>> +    case GITS_PIDR3:
>> +        if ( dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        *r = GITS_PIDR3_VAL;
>> +        return 1;
>> +    case GITS_PIDR4:
>> +        if ( dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        *r = GITS_PIDR4_VAL;
>> +        return 1;
>> +    case GITS_PIDR5 ... GITS_PIDR7:
>> +        goto read_as_zero_32;
>> +   default:
>> +        dprintk(XENLOG_G_ERR,
>> +                "%pv: vITS: unhandled read r%"PRId32" offset 0x%#08"PRIx32"\n",
>
> Reg is definitely not a PRId32.
>
>> +                v, dabt.reg, gits_reg);
>> +        return 0;
>> +    }
>> +
>> +bad_width:
>> +    dprintk(XENLOG_G_ERR,
>> +            "%pv: vITS: bad read width %d r%"PRId32" offset 0x%#08"PRIx32"\n",
>
> Ditto.
>
>> +            v, dabt.size, dabt.reg, gits_reg);
>> +    domain_crash_synchronous();
>> +    return 0;
>> +
>> +read_as_zero_32:
>> +    if ( dabt.size != DABT_WORD ) goto bad_width;
>> +read_as_zero:
>> +    *r = 0;
>> +    return 1;
>> +}
>> +
>> +/*
>> + * GITS_BASER.Type[58:56], GITS_BASER.Entry_size[55:48]
>> + * and GITS_BASER.Shareability[11:10] are read-only.
>
> As said on v4, implemented Shareability as fixed (i.e read-only) is
> deprecated. I'd like to see a TODO here.
>
>> + * Mask those fields while emulating GITS_BASER reg.
>> + */
>
> As said on v4,
>
> Other fields are (or could be RO) in GITS_BASER:
>     - Indirect: we only support flat table
        By default it is 0. So support flat table. Do you want it explicitly
       specify Indirect?

>     - Page_Size: it's fine to only support 4KB granularity. It also
> means less code.
        Page_size is set by guest. this is not RO

>
> I don't mind if you don't do the latter. The former is a mandatory.
>
>> +#define GITS_BASER_MASK  (~((0x7UL << GITS_BASER_TYPE_SHIFT)     | \
>> +                         (0xffUL << GITS_BASER_ENTRY_SIZE_SHIFT) | \
>> +                         (0x3UL << GITS_BASER_SHAREABILITY_SHIFT)))
>
> [..]
>
>> +    case GITS_CWRITER:
>> +        if ( dabt.size != DABT_DOUBLE_WORD && dabt.size != DABT_WORD )
>> +            goto bad_width;
>> +        vits_spin_lock(vits);
>> +        /* Only BITS[19:0] are writable */
>> +        vits->cmd_write = *r & 0xfffe0;
>> +        ret = 1;
>> +        if ( vits->ctrl & GITS_CTLR_ENABLE )
>> +        {
>> +            /* CWRITER should be within the range */
>> +            if ( vits->cmd_write < (vits->cmd_qsize & 0xfffe0) )
>> +                ret = vits_process_cmd(v, vits);
>> +        }
>
> Please do only 1 if rather than 2 nested if for only one line.
>
>> +        vits_spin_unlock(vits);
>> +        return ret;
>
> [..]
>
>>  int vits_domain_init(struct domain *d)
>>  {
>>      struct vgic_its *vits;
>>      int i;
>>
>> +    if ( is_hardware_domain(d) )
>> +        d->arch.vgic.nr_lpis = num_of_lpis;
>> +    else
>> +        d->arch.vgic.nr_lpis = NR_LPIS;
>
> NR_LPIS is defined in patch #14. And the name seems to be wrong.
>
> Anyway, I don't understand why you are trying to initialize vITS on
> guest. We agree that it should only be used on DOM0 for now until we
> effectively need it for the guest.
>
> Furthermore, it miss at least the toolstack in order to get the part
> guest ready.
>
> So please ensure that the vITS is not initialized for the guest.

In patch#18, this function is called only for DOM0