Re: [PATCH v1 06/14] xen/arm: Add support for PCI ecam operations

[Julien Grall <julien@xen.org>]

Hi Rahul,

On 19/08/2021 13:02, Rahul Singh wrote:
> Add support for PCI ecam operations to access the PCI
> configuration space.
> 
> Signed-off-by: Rahul Singh <rahul.singh@arm.com>
> ---
>   xen/arch/arm/pci/Makefile           |  1 +
>   xen/arch/arm/pci/ecam.c             | 63 +++++++++++++++++++++++++++++
>   xen/arch/arm/pci/pci-access.c       | 53 ++++++++++++++++++++++++
>   xen/arch/arm/pci/pci-host-common.c  | 13 +++++-
>   xen/arch/arm/pci/pci-host-generic.c |  8 +++-
>   xen/include/asm-arm/pci.h           | 32 +++++++++++++++
>   6 files changed, 167 insertions(+), 3 deletions(-)
>   create mode 100644 xen/arch/arm/pci/ecam.c
> 
> diff --git a/xen/arch/arm/pci/Makefile b/xen/arch/arm/pci/Makefile
> index f3d97f859e..6f32fbbe67 100644
> --- a/xen/arch/arm/pci/Makefile
> +++ b/xen/arch/arm/pci/Makefile
> @@ -2,3 +2,4 @@ obj-y += pci.o
>   obj-y += pci-access.o
>   obj-y += pci-host-generic.o
>   obj-y += pci-host-common.o
> +obj-y += ecam.o
> diff --git a/xen/arch/arm/pci/ecam.c b/xen/arch/arm/pci/ecam.c
> new file mode 100644
> index 0000000000..91c691b41f
> --- /dev/null
> +++ b/xen/arch/arm/pci/ecam.c
> @@ -0,0 +1,63 @@
> +/*
> + * Copyright (C) 2021 Arm Ltd.
> + *
> + * Based on Linux drivers/pci/ecam.c
> + * Copyright 2016 Broadcom
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <xen/pci.h>
> +#include <xen/sched.h>
> +
> +/*
> + * Function to implement the pci_ops ->map_bus method.
> + */
> +void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
> +                                      uint32_t sbdf, uint32_t where)
> +{
> +    const struct pci_config_window *cfg = bridge->sysdata;
> +    unsigned int devfn_shift = cfg->ops->bus_shift - 8;
> +    void __iomem *base;
> +
> +    pci_sbdf_t sbdf_t = (pci_sbdf_t) sbdf ;

AFAICT, pci_sbdf is an union between a 32-bit and a structure. So please 
don't use the cast and use the 32-bit field to assign the value.

Also, there is an extra space before ';'.

> +    unsigned int busn = sbdf_t.bus;
> +
> +    if ( busn < cfg->busn_start || busn > cfg->busn_end )
> +        return NULL;
> +
> +    busn -= cfg->busn_start;
> +    base = cfg->win + (busn << cfg->ops->bus_shift);
> +
> +    return base + (PCI_DEVFN(sbdf_t.dev, sbdf_t.fn) << devfn_shift) + where;

How about using PCI_DEVFN2(sbdf)? This would allow you to drop the use 
of sbdf_t completely (sbdf_t.bus could be replaced with PCI_BUS(sdbf)).

> +}
> +
> +/* ECAM ops */
> +const struct pci_ecam_ops pci_generic_ecam_ops = {
> +    .bus_shift  = 20,
> +    .pci_ops    = {
> +        .map_bus                = pci_ecam_map_bus,
> +        .read                   = pci_generic_config_read,
> +        .write                  = pci_generic_config_write,
> +    }
> +};
> +
> +/*
> + * Local variables:
> + * mode: C
> + * c-file-style: "BSD"
> + * c-basic-offset: 4
> + * tab-width: 4
> + * indent-tabs-mode: nil
> + * End:
> + */
> diff --git a/xen/arch/arm/pci/pci-access.c b/xen/arch/arm/pci/pci-access.c
> index b938047c03..f39f6a3a38 100644
> --- a/xen/arch/arm/pci/pci-access.c
> +++ b/xen/arch/arm/pci/pci-access.c
> @@ -15,6 +15,59 @@
>    */
>   
>   #include <xen/pci.h>
> +#include <asm/io.h>
> +
> +int pci_generic_config_read(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                            uint32_t reg, uint32_t len, uint32_t *value)
> +{
> +    void __iomem *addr = bridge->ops->map_bus(bridge, sbdf, reg);

Please add a newline here.

> +    if (!addr) {

You seem to use a mix of Xen and Linux coding style. If the file 
contains mostly Xen code, then we should use the former.

> +        *value = ~0;
> +        return -ENODEV;
> +    }
> +
> +    switch (len)
> +    {
> +    case 1:
> +        *value = readb(addr);
> +        break;
> +    case 2:
> +        *value = readw(addr);
> +        break;
> +    case 4:
> +        *value = readl(addr);
> +        break;
> +    default:
> +        BUG();
> +    }
> +
> +    return 0;
> +}
> +
> +int pci_generic_config_write(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                            uint32_t reg, uint32_t len, uint32_t value)
> +{
> +    void __iomem *addr = bridge->ops->map_bus(bridge, sbdf, reg);
> +    if (!addr)
> +        return -ENODEV;
> +
> +    switch (len)
> +    {
> +    case 1:
> +        writeb(value, addr);
> +        break;
> +    case 2:
> +        writew(value, addr);
> +        break;
> +    case 4:
> +        writel(value, addr);
> +        break;
> +    default:
> +        BUG();
> +    }
> +
> +    return 0;
> +}
>   
>   static uint32_t pci_config_read(pci_sbdf_t sbdf, unsigned int reg,
>                                   unsigned int len)
> diff --git a/xen/arch/arm/pci/pci-host-common.c b/xen/arch/arm/pci/pci-host-common.c
> index 9dd9b02271..c582527e92 100644
> --- a/xen/arch/arm/pci/pci-host-common.c
> +++ b/xen/arch/arm/pci/pci-host-common.c
> @@ -68,6 +68,7 @@ static void pci_ecam_free(struct pci_config_window *cfg)
>   }
>   
>   static struct pci_config_window *gen_pci_init(struct dt_device_node *dev,
> +                                              const struct pci_ecam_ops *ops,
>                                                 int ecam_reg_idx)
>   {
>       int err;
> @@ -96,6 +97,7 @@ static struct pci_config_window *gen_pci_init(struct dt_device_node *dev,
>   
>       cfg->phys_addr = addr;
>       cfg->size = size;
> +    cfg->ops = ops;
>   
>       /*
>        * On 64-bit systems, we do a single ioremap for the whole config space
> @@ -111,6 +113,13 @@ static struct pci_config_window *gen_pci_init(struct dt_device_node *dev,
>       printk("ECAM at [mem %lx-%lx] for [bus %x-%x] \n",cfg->phys_addr,
>               cfg->phys_addr + cfg->size - 1, cfg->busn_start, cfg->busn_end);
>   
> +    if ( ops->init )
> +    {
> +        err = ops->init(cfg);
> +        if (err)
> +            goto err_exit;
> +    }
> +
>       return cfg;
>   
>   err_exit_remap:
> @@ -216,6 +225,7 @@ static int pci_bus_find_domain_nr(struct dt_device_node *dev)
>   }
>   
>   int pci_host_common_probe(struct dt_device_node *dev,
> +                          const struct pci_ecam_ops *ops,
>                             int ecam_reg_idx)
>   {
>       struct pci_host_bridge *bridge;
> @@ -227,7 +237,7 @@ int pci_host_common_probe(struct dt_device_node *dev,
>           return -ENOMEM;
>   
>       /* Parse and map our Configuration Space windows */
> -    cfg = gen_pci_init(dev, ecam_reg_idx);
> +    cfg = gen_pci_init(dev, ops, ecam_reg_idx);
>       if ( !cfg )
>       {
>           err = -ENOMEM;
> @@ -236,6 +246,7 @@ int pci_host_common_probe(struct dt_device_node *dev,
>   
>       bridge->dt_node = dev;
>       bridge->sysdata = cfg;
> +    bridge->ops = &ops->pci_ops;
>       bridge->bus_start = cfg->busn_start;
>       bridge->bus_end = cfg->busn_end;
>   
> diff --git a/xen/arch/arm/pci/pci-host-generic.c b/xen/arch/arm/pci/pci-host-generic.c
> index 13d0f7f999..2d652e8910 100644
> --- a/xen/arch/arm/pci/pci-host-generic.c
> +++ b/xen/arch/arm/pci/pci-host-generic.c
> @@ -23,20 +23,24 @@
>   #include <asm/pci.h>
>   
>   static const struct dt_device_match gen_pci_dt_match[] = {
> -    { .compatible = "pci-host-ecam-generic" },
> +    { .compatible = "pci-host-ecam-generic",
> +      .data =       &pci_generic_ecam_ops },
> +
>       { },
>   };
>   
>   static int gen_pci_dt_init(struct dt_device_node *dev, const void *data)
>   {
>       const struct dt_device_match *of_id;
> +    const struct pci_ecam_ops *ops;
>   
>       of_id = dt_match_node(gen_pci_dt_match, dev->dev.of_node);
> +    ops = (struct pci_ecam_ops *) of_id->data;
>   
>       printk(XENLOG_INFO "Found PCI host bridge %s compatible:%s \n",
>              dt_node_full_name(dev), of_id->compatible);
>   
> -    return pci_host_common_probe(dev, 0);
> +    return pci_host_common_probe(dev, ops, 0);
>   }
>   
>   DT_DEVICE_START(pci_gen, "PCI HOST GENERIC", DEVICE_PCI)
> diff --git a/xen/include/asm-arm/pci.h b/xen/include/asm-arm/pci.h
> index 58a51e724e..22866244d2 100644
> --- a/xen/include/asm-arm/pci.h
> +++ b/xen/include/asm-arm/pci.h
> @@ -37,6 +37,7 @@ struct pci_config_window {
>       uint8_t         busn_start;
>       uint8_t         busn_end;
>       void __iomem    *win;
> +    const struct    pci_ecam_ops *ops;
>   };
>   
>   /*
> @@ -50,10 +51,41 @@ struct pci_host_bridge {
>       u8 bus_start;                    /* Bus start of this bridge. */
>       u8 bus_end;                      /* Bus end of this bridge. */
>       void *sysdata;                   /* Pointer to the config space window*/
> +    const struct pci_ops *ops;
>   };
>   
> +struct pci_ops {
> +    void __iomem *(*map_bus)(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                             uint32_t offset);
> +    int (*read)(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                uint32_t reg, uint32_t len, uint32_t *value);
> +    int (*write)(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                 uint32_t reg, uint32_t len, uint32_t value);
> +};
> +
> +/*
> + * struct to hold pci ops and bus shift of the config window
> + * for a PCI controller.
> + */
> +struct pci_ecam_ops {
> +    unsigned int            bus_shift;
> +    struct pci_ops          pci_ops;
> +    int (*init)(struct pci_config_window *);
> +};
> +
> +/* Default ECAM ops */
> +extern const struct pci_ecam_ops pci_generic_ecam_ops;
> +
>   int pci_host_common_probe(struct dt_device_node *dev,
> +                          const struct pci_ecam_ops *ops,
>                             int ecam_reg_idx);
> +int pci_generic_config_read(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                            uint32_t reg, uint32_t len, uint32_t *value);
> +int pci_generic_config_write(struct pci_host_bridge *bridge, uint32_t sbdf,
> +                            uint32_t reg, uint32_t len, uint32_t value);
> +void __iomem *pci_ecam_map_bus(struct pci_host_bridge *bridge,
> +                               uint32_t sbdf, uint32_t where);
> +
>   #else   /*!CONFIG_HAS_PCI*/
>   
>   struct arch_pci_dev { };
> 

Cheers,

-- 
Julien Grall