Re: [PATCH v5 08/11] xen/arm: Enable the existing x86 virtual PCI support for ARM.

[Oleksandr Andrushchenko <Oleksandr_Andrushchenko@epam.com>]

On 13.10.21 16:00, Jan Beulich wrote:
> On 13.10.2021 10:45, Roger Pau Monné wrote:
>> On Wed, Oct 06, 2021 at 06:40:34PM +0100, Rahul Singh wrote:
>>> The existing VPCI support available for X86 is adapted for Arm.
>>> When the device is added to XEN via the hyper call
>>> “PHYSDEVOP_pci_device_add”, VPCI handler for the config space
>>> access is added to the Xen to emulate the PCI devices config space.
>>>
>>> A MMIO trap handler for the PCI ECAM space is registered in XEN
>>> so that when guest is trying to access the PCI config space,XEN
>>> will trap the access and emulate read/write using the VPCI and
>>> not the real PCI hardware.
>>>
>>> For Dom0less systems scan_pci_devices() would be used to discover the
>>> PCI device in XEN and VPCI handler will be added during XEN boots.
>>>
>>> Signed-off-by: Rahul Singh <rahul.singh@arm.com>
>>> Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
>>> ---
>>> Change in v5:
>>> - Add pci_cleanup_msi(pdev) in cleanup part.
>>> - Added Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
>>> Change in v4:
>>> - Move addition of XEN_DOMCTL_CDF_vpci flag to separate patch
>>> Change in v3:
>>> - Use is_pci_passthrough_enabled() in place of pci_passthrough_enabled variable
>>> - Reject XEN_DOMCTL_CDF_vpci for x86 in arch_sanitise_domain_config()
>>> - Remove IS_ENABLED(CONFIG_HAS_VPCI) from has_vpci()
>>> Change in v2:
>>> - Add new XEN_DOMCTL_CDF_vpci flag
>>> - modify has_vpci() to include XEN_DOMCTL_CDF_vpci
>>> - enable vpci support when pci-passthough option is enabled.
>>> ---
>>> ---
>>>   xen/arch/arm/Makefile         |   1 +
>>>   xen/arch/arm/domain.c         |   4 ++
>>>   xen/arch/arm/domain_build.c   |   3 +
>>>   xen/arch/arm/vpci.c           | 102 ++++++++++++++++++++++++++++++++++
>>>   xen/arch/arm/vpci.h           |  36 ++++++++++++
>>>   xen/drivers/passthrough/pci.c |  18 ++++++
>>>   xen/include/asm-arm/domain.h  |   7 ++-
>>>   xen/include/asm-x86/pci.h     |   2 -
>>>   xen/include/public/arch-arm.h |   7 +++
>>>   xen/include/xen/pci.h         |   2 +
>>>   10 files changed, 179 insertions(+), 3 deletions(-)
>>>   create mode 100644 xen/arch/arm/vpci.c
>>>   create mode 100644 xen/arch/arm/vpci.h
>>>
>>> diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
>>> index 44d7cc81fa..fb9c976ea2 100644
>>> --- a/xen/arch/arm/Makefile
>>> +++ b/xen/arch/arm/Makefile
>>> @@ -7,6 +7,7 @@ ifneq ($(CONFIG_NO_PLAT),y)
>>>   obj-y += platforms/
>>>   endif
>>>   obj-$(CONFIG_TEE) += tee/
>>> +obj-$(CONFIG_HAS_VPCI) += vpci.o
>>>   
>>>   obj-$(CONFIG_HAS_ALTERNATIVE) += alternative.o
>>>   obj-y += bootfdt.init.o
>>> diff --git a/xen/arch/arm/domain.c b/xen/arch/arm/domain.c
>>> index 36138c1b2e..fbb52f78f1 100644
>>> --- a/xen/arch/arm/domain.c
>>> +++ b/xen/arch/arm/domain.c
>>> @@ -39,6 +39,7 @@
>>>   #include <asm/vgic.h>
>>>   #include <asm/vtimer.h>
>>>   
>>> +#include "vpci.h"
>>>   #include "vuart.h"
>>>   
>>>   DEFINE_PER_CPU(struct vcpu *, curr_vcpu);
>>> @@ -767,6 +768,9 @@ int arch_domain_create(struct domain *d,
>>>       if ( is_hardware_domain(d) && (rc = domain_vuart_init(d)) )
>>>           goto fail;
>>>   
>>> +    if ( (rc = domain_vpci_init(d)) != 0 )
>>> +        goto fail;
>>> +
>>>       return 0;
>>>   
>>>   fail:
>>> diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
>>> index c5afbe2e05..f4c89bde8c 100644
>>> --- a/xen/arch/arm/domain_build.c
>>> +++ b/xen/arch/arm/domain_build.c
>>> @@ -3053,6 +3053,9 @@ void __init create_dom0(void)
>>>       if ( iommu_enabled )
>>>           dom0_cfg.flags |= XEN_DOMCTL_CDF_iommu;
>>>   
>>> +    if ( is_pci_passthrough_enabled() )
>>> +        dom0_cfg.flags |= XEN_DOMCTL_CDF_vpci;
>>> +
>>>       dom0 = domain_create(0, &dom0_cfg, true);
>>>       if ( IS_ERR(dom0) || (alloc_dom0_vcpu0(dom0) == NULL) )
>>>           panic("Error creating domain 0\n");
>>> diff --git a/xen/arch/arm/vpci.c b/xen/arch/arm/vpci.c
>>> new file mode 100644
>>> index 0000000000..76c12b9281
>>> --- /dev/null
>>> +++ b/xen/arch/arm/vpci.c
>>> @@ -0,0 +1,102 @@
>>> +/*
>>> + * xen/arch/arm/vpci.c
>>> + *
>>> + * This program is free software; you can redistribute it and/or modify
>>> + * it under the terms of the GNU General Public License as published by
>>> + * the Free Software Foundation; either version 2 of the License, or
>>> + * (at your option) any later version.
>>> + *
>>> + * This program is distributed in the hope that it will be useful,
>>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>>> + * GNU General Public License for more details.
>>> + */
>>> +#include <xen/sched.h>
>>> +
>>> +#include <asm/mmio.h>
>>> +
>>> +#define REGISTER_OFFSET(addr)  ( (addr) & 0x00000fff)
>>> +
>>> +/* Do some sanity checks. */
>>> +static bool vpci_mmio_access_allowed(unsigned int reg, unsigned int len)
>>> +{
>>> +    /* Check access size. */
>>> +    if ( len > 8 )
>>> +        return false;
>>> +
>>> +    /* Check that access is size aligned. */
>>> +    if ( (reg & (len - 1)) )
>>> +        return false;
>>> +
>>> +    return true;
>>> +}
>>> +
>>> +static int vpci_mmio_read(struct vcpu *v, mmio_info_t *info,
>>> +                          register_t *r, void *p)
>>> +{
>>> +    unsigned int reg;
>>> +    pci_sbdf_t sbdf;
>>> +    unsigned long data = ~0UL;
>>> +    unsigned int size = 1U << info->dabt.size;
>>> +
>>> +    sbdf.sbdf = MMCFG_BDF(info->gpa);
>>> +    reg = REGISTER_OFFSET(info->gpa);
>>> +
>>> +    if ( !vpci_mmio_access_allowed(reg, size) )
>>> +        return 0;
>>> +
>>> +    data = vpci_read(sbdf, reg, min(4u, size));
>>> +    if ( size == 8 )
>>> +        data |= (uint64_t)vpci_read(sbdf, reg + 4, 4) << 32;
>>> +
>>> +    *r = data;
>>> +
>>> +    return 1;
>>> +}
>>> +
>>> +static int vpci_mmio_write(struct vcpu *v, mmio_info_t *info,
>>> +                           register_t r, void *p)
>>> +{
>>> +    unsigned int reg;
>>> +    pci_sbdf_t sbdf;
>>> +    unsigned long data = r;
>>> +    unsigned int size = 1U << info->dabt.size;
>>> +
>>> +    sbdf.sbdf = MMCFG_BDF(info->gpa);
>>> +    reg = REGISTER_OFFSET(info->gpa);
>>> +
>>> +    if ( !vpci_mmio_access_allowed(reg, size) )
>>> +        return 0;
>>> +
>>> +    vpci_write(sbdf, reg, min(4u, size), data);
>>> +    if ( size == 8 )
>>> +        vpci_write(sbdf, reg + 4, 4, data >> 32);
>> I think those two helpers (and vpci_mmio_access_allowed) are very
>> similar to the existing x86 ones (see vpci_mmcfg_{read,write}), up to
>> the point where I would consider moving the shared code to vpci.c as
>> vpci_ecam_{read,write} and call them from the arch specific trap
>> handlers.
> Except that please can we stick to mcfg or mmcfg instead of ecam
> in names, as that's how the thing has been named in Xen from its
> introduction? I've just grep-ed the code base (case insensitively)
> and found no mention of ECAM. There are only a few "became".
I do understand that this is historically that we do not have ECAM in Xen,
but PCI is not about Xen. Thus, I think it is also acceptable to use
a commonly known ECAM for the code that works with ECAM.
At least for the new code
> Jan
>
>