[Qemu-devel] [PATCH 0/3] pci/pcie: implement ARI enable bit correctly

[Qemu-devel] [PATCH 0/3] pci/pcie: implement ARI enable bit correctly

[Isaku Yamahata]

This patch series implements ARI enable bit in root/downstream port
correctly.

Isaku Yamahata (3):
  pci: replace the magic, 256, for the maximum of devfn
  pci: use devfn for pci_find_device() instead of (slot, fn) pair
  pci/pcie: make pci_find_device() ARI aware.

 hw/pci-hotplug.c   |    5 +++--
 hw/pci.c           |   11 +++++++++--
 hw/pci.h           |    4 +++-
 hw/pci_host.c      |    2 +-
 hw/pci_internals.h |    2 +-
 hw/pcie.c          |   33 +++++++++++++++++++++++++++++----
 hw/pcie.h          |    2 +-
 hw/pcie_host.c     |    3 +--
 8 files changed, 48 insertions(+), 14 deletions(-)

[Qemu-devel] [PATCH 1/3] pci: replace the magic, 256, for the maximum of devfn

[Isaku Yamahata]

Introduce symbol PCI_DEVFN_MAX for the maximum of devfn
and replace the magic, 256.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
---
 hw/pci.h           |    2 ++
 hw/pci_internals.h |    2 +-
 2 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/hw/pci.h b/hw/pci.h
index 0d2753f..ffb04e8 100644
--- a/hw/pci.h
+++ b/hw/pci.h
@@ -16,7 +16,9 @@
 #define PCI_DEVFN(slot, func)   ((((slot) & 0x1f) << 3) | ((func) & 0x07))
 #define PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
 #define PCI_FUNC(devfn)         ((devfn) & 0x07)
+#define PCI_SLOT_MAX            32
 #define PCI_FUNC_MAX            8
+#define PCI_DEVFN_MAX           (PCI_SLOT_MAX * PCI_FUNC_MAX)
 
 /* Class, Vendor and Device IDs from Linux's pci_ids.h */
 #include "pci_ids.h"
diff --git a/hw/pci_internals.h b/hw/pci_internals.h
index e3c93a3..f17e681 100644
--- a/hw/pci_internals.h
+++ b/hw/pci_internals.h
@@ -22,7 +22,7 @@ struct PCIBus {
     pci_hotplug_fn hotplug;
     DeviceState *hotplug_qdev;
     void *irq_opaque;
-    PCIDevice *devices[256];
+    PCIDevice *devices[PCI_DEVFN_MAX];
     PCIDevice *parent_dev;
     target_phys_addr_t mem_base;
 
-- 
1.7.1.1

[Qemu-devel] [PATCH 2/3] pci: use devfn for pci_find_device() instead of (slot, fn) pair

[Isaku Yamahata]

(slot, fn) pair is somewhat confusing because of ARI.
So use devfn for pci_find_device() instead of (slot, fn).

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
---
 hw/pci-hotplug.c |    5 +++--
 hw/pci.c         |    5 +++--
 hw/pci.h         |    2 +-
 hw/pci_host.c    |    2 +-
 hw/pcie_host.c   |    3 +--
 5 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/hw/pci-hotplug.c b/hw/pci-hotplug.c
index 270a982..9715235 100644
--- a/hw/pci-hotplug.c
+++ b/hw/pci-hotplug.c
@@ -126,7 +126,8 @@ void drive_hot_add(Monitor *mon, const QDict *qdict)
         if (pci_read_devaddr(mon, pci_addr, &dom, &pci_bus, &slot)) {
             goto err;
         }
-        dev = pci_find_device(pci_find_root_bus(dom), pci_bus, slot, 0);
+        dev = pci_find_device(pci_find_root_bus(dom), pci_bus,
+                              PCI_DEVFN(slot, 0));
         if (!dev) {
             monitor_printf(mon, "no pci device with address %s\n", pci_addr);
             goto err;
@@ -276,7 +277,7 @@ static int pci_device_hot_remove(Monitor *mon, const char *pci_addr)
         return -1;
     }
 
-    d = pci_find_device(pci_find_root_bus(dom), bus, slot, 0);
+    d = pci_find_device(pci_find_root_bus(dom), bus, PCI_DEVFN(slot, 0));
     if (!d) {
         monitor_printf(mon, "slot %d empty\n", slot);
         return -1;
diff --git a/hw/pci.c b/hw/pci.c
index 79a46e7..ac16029 100644
--- a/hw/pci.c
+++ b/hw/pci.c
@@ -1599,14 +1599,15 @@ PCIBus *pci_find_bus(PCIBus *bus, int bus_num)
     return NULL;
 }
 
-PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int slot, int function)
+PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn)
 {
+    assert(devfn < PCI_DEVFN_MAX);
     bus = pci_find_bus(bus, bus_num);
 
     if (!bus)
         return NULL;
 
-    return bus->devices[PCI_DEVFN(slot, function)];
+    return bus->devices[devfn];
 }
 
 static int pci_qdev_init(DeviceState *qdev, DeviceInfo *base)
diff --git a/hw/pci.h b/hw/pci.h
index ffb04e8..f6d20f6 100644
--- a/hw/pci.h
+++ b/hw/pci.h
@@ -253,7 +253,7 @@ void pci_for_each_device(PCIBus *bus, int bus_num, void (*fn)(PCIBus *bus, PCIDe
 PCIBus *pci_find_root_bus(int domain);
 int pci_find_domain(const PCIBus *bus);
 PCIBus *pci_find_bus(PCIBus *bus, int bus_num);
-PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int slot, int function);
+PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn);
 int pci_qdev_find_device(const char *id, PCIDevice **pdev);
 PCIBus *pci_get_bus_devfn(int *devfnp, const char *devaddr);
 
diff --git a/hw/pci_host.c b/hw/pci_host.c
index 7c40155..728e2d4 100644
--- a/hw/pci_host.c
+++ b/hw/pci_host.c
@@ -44,7 +44,7 @@ static inline PCIDevice *pci_dev_find_by_addr(PCIBus *bus, uint32_t addr)
     uint8_t bus_num = addr >> 16;
     uint8_t devfn = addr >> 8;
 
-    return pci_find_device(bus, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn));
+    return pci_find_device(bus, bus_num, devfn);
 }
 
 void pci_data_write(PCIBus *s, uint32_t addr, uint32_t val, int len)
diff --git a/hw/pcie_host.c b/hw/pcie_host.c
index 21069ee..b749865 100644
--- a/hw/pcie_host.c
+++ b/hw/pcie_host.c
@@ -49,8 +49,7 @@ static inline PCIDevice *pcie_dev_find_by_mmcfg_addr(PCIBus *s,
                                                      uint32_t mmcfg_addr)
 {
     return pci_find_device(s, PCIE_MMCFG_BUS(mmcfg_addr),
-                           PCI_SLOT(PCIE_MMCFG_DEVFN(mmcfg_addr)),
-                           PCI_FUNC(PCIE_MMCFG_DEVFN(mmcfg_addr)));
+                           PCIE_MMCFG_DEVFN(mmcfg_addr));
 }
 
 static void pcie_mmcfg_data_write(PCIBus *s,
-- 
1.7.1.1

[Qemu-devel] [PATCH 3/3] pci/pcie: make pci_find_device() ARI aware.

[Isaku Yamahata]

make pci_find_device() ARI aware.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
---
 hw/pci.c  |    6 ++++++
 hw/pcie.c |   33 +++++++++++++++++++++++++++++----
 hw/pcie.h |    2 +-
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/hw/pci.c b/hw/pci.c
index ac16029..daba310 100644
--- a/hw/pci.c
+++ b/hw/pci.c
@@ -1601,12 +1601,18 @@ PCIBus *pci_find_bus(PCIBus *bus, int bus_num)
 
 PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn)
 {
+    PCIDevice *d;
     assert(devfn < PCI_DEVFN_MAX);
     bus = pci_find_bus(bus, bus_num);
 
     if (!bus)
         return NULL;
 
+    d = bus->parent_dev;
+    /* ARI: See the comment above the pcie_check_slot() for details */
+    if (d && !pcie_check_slot(d, devfn)) {
+        return NULL;
+    }
     return bus->devices[devfn];
 }
 
diff --git a/hw/pcie.c b/hw/pcie.c
index 6a113a9..ae4b531 100644
--- a/hw/pcie.c
+++ b/hw/pcie.c
@@ -424,13 +424,38 @@ void pcie_cap_ari_reset(PCIDevice *dev)
     pci_long_test_and_clear_mask(devctl2, PCI_EXP_DEVCTL2_ARI);
 }
 
-bool pcie_cap_is_ari_enabled(const PCIDevice *dev)
+/*
+ * 6.13 Alternative routing-ID Interpretation(ARI)
+ * 7.8.16 Device control 2 register
+ *   ARI forwarding Enable
+ *
+ * With PCI Express Endpoints, there's a single device behind
+ * each downstream port bus, and bits 3:7 of the function number get
+ * encoded in the slot number (the Express spec calls it the Device
+ * Number). This allows > 8 functions, but
+ * these extended functions are only accessible when the
+ * Alternative routing-ID Interpretation (ARI)
+ * capability is enabled in the root/downstream port. With that capability
+ * disabled the port enforces the Device Number field being 0.
+ */
+bool pcie_check_slot(const PCIDevice *dev, int devfn)
 {
+    uint8_t type;
+
     if (!pci_is_express(dev)) {
-        return false;
+        return true;
     }
-    if (!dev->exp.exp_cap) {
-        return false;
+
+    type = pcie_cap_get_type(dev);
+    if (!(type == PCI_EXP_TYPE_ROOT_PORT ||
+          type == PCI_EXP_TYPE_DOWNSTREAM)) {
+        return true;
+    }
+
+    if (!PCI_SLOT(devfn)) {
+        /* With ARI, this means function < 8.
+           functions < 8 are always accesible. */
+        return true;
     }
 
     return pci_get_long(dev->config + dev->exp.exp_cap + PCI_EXP_DEVCTL2) &
diff --git a/hw/pcie.h b/hw/pcie.h
index bc909e2..0c84662 100644
--- a/hw/pcie.h
+++ b/hw/pcie.h
@@ -119,7 +119,7 @@ void pcie_cap_flr_write_config(PCIDevice *dev,
 
 void pcie_cap_ari_init(PCIDevice *dev);
 void pcie_cap_ari_reset(PCIDevice *dev);
-bool pcie_cap_is_ari_enabled(const PCIDevice *dev);
+bool pcie_check_slot(const PCIDevice *dev, int devfn);
 
 /* PCI express extended capability helper functions */
 uint16_t pcie_find_capability(PCIDevice *dev, uint16_t cap_id);
-- 
1.7.1.1

[Qemu-devel] Re: [PATCH 1/3] pci: replace the magic, 256, for the maximum of devfn

[Michael S. Tsirkin]

On Wed, Jan 26, 2011 at 06:45:18PM +0900, Isaku Yamahata wrote:
> Introduce symbol PCI_DEVFN_MAX for the maximum of devfn
> and replace the magic, 256.
> 
> Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
> ---
>  hw/pci.h           |    2 ++
>  hw/pci_internals.h |    2 +-
>  2 files changed, 3 insertions(+), 1 deletions(-)
> 
> diff --git a/hw/pci.h b/hw/pci.h
> index 0d2753f..ffb04e8 100644
> --- a/hw/pci.h
> +++ b/hw/pci.h
> @@ -16,7 +16,9 @@
>  #define PCI_DEVFN(slot, func)   ((((slot) & 0x1f) << 3) | ((func) & 0x07))
>  #define PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
>  #define PCI_FUNC(devfn)         ((devfn) & 0x07)
> +#define PCI_SLOT_MAX            32
>  #define PCI_FUNC_MAX            8
> +#define PCI_DEVFN_MAX           (PCI_SLOT_MAX * PCI_FUNC_MAX)

SLOT_MAX makes sense. I'd prefer open-coding PCI_DEVFN_MAX
if there's only one user.

>  
>  /* Class, Vendor and Device IDs from Linux's pci_ids.h */
>  #include "pci_ids.h"
> diff --git a/hw/pci_internals.h b/hw/pci_internals.h
> index e3c93a3..f17e681 100644
> --- a/hw/pci_internals.h
> +++ b/hw/pci_internals.h
> @@ -22,7 +22,7 @@ struct PCIBus {
>      pci_hotplug_fn hotplug;
>      DeviceState *hotplug_qdev;
>      void *irq_opaque;
> -    PCIDevice *devices[256];
> +    PCIDevice *devices[PCI_DEVFN_MAX];
>      PCIDevice *parent_dev;
>      target_phys_addr_t mem_base;
>  
> -- 
> 1.7.1.1

[Qemu-devel] Re: [PATCH 2/3] pci: use devfn for pci_find_device() instead of (slot, fn) pair

[Michael S. Tsirkin]

On Wed, Jan 26, 2011 at 06:45:19PM +0900, Isaku Yamahata wrote:
> (slot, fn) pair is somewhat confusing because of ARI.
> So use devfn for pci_find_device() instead of (slot, fn).
> 
> Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>

I'd prefer using uint8_t for devfn. This way we don't need
to bother to range-check it either.

> ---
>  hw/pci-hotplug.c |    5 +++--
>  hw/pci.c         |    5 +++--
>  hw/pci.h         |    2 +-
>  hw/pci_host.c    |    2 +-
>  hw/pcie_host.c   |    3 +--
>  5 files changed, 9 insertions(+), 8 deletions(-)
> 
> diff --git a/hw/pci-hotplug.c b/hw/pci-hotplug.c
> index 270a982..9715235 100644
> --- a/hw/pci-hotplug.c
> +++ b/hw/pci-hotplug.c
> @@ -126,7 +126,8 @@ void drive_hot_add(Monitor *mon, const QDict *qdict)
>          if (pci_read_devaddr(mon, pci_addr, &dom, &pci_bus, &slot)) {
>              goto err;
>          }
> -        dev = pci_find_device(pci_find_root_bus(dom), pci_bus, slot, 0);
> +        dev = pci_find_device(pci_find_root_bus(dom), pci_bus,
> +                              PCI_DEVFN(slot, 0));
>          if (!dev) {
>              monitor_printf(mon, "no pci device with address %s\n", pci_addr);
>              goto err;
> @@ -276,7 +277,7 @@ static int pci_device_hot_remove(Monitor *mon, const char *pci_addr)
>          return -1;
>      }
>  
> -    d = pci_find_device(pci_find_root_bus(dom), bus, slot, 0);
> +    d = pci_find_device(pci_find_root_bus(dom), bus, PCI_DEVFN(slot, 0));
>      if (!d) {
>          monitor_printf(mon, "slot %d empty\n", slot);
>          return -1;
> diff --git a/hw/pci.c b/hw/pci.c
> index 79a46e7..ac16029 100644
> --- a/hw/pci.c
> +++ b/hw/pci.c
> @@ -1599,14 +1599,15 @@ PCIBus *pci_find_bus(PCIBus *bus, int bus_num)
>      return NULL;
>  }
>  
> -PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int slot, int function)
> +PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn)
>  {
> +    assert(devfn < PCI_DEVFN_MAX);
>      bus = pci_find_bus(bus, bus_num);
>  
>      if (!bus)
>          return NULL;
>  
> -    return bus->devices[PCI_DEVFN(slot, function)];
> +    return bus->devices[devfn];
>  }
>  
>  static int pci_qdev_init(DeviceState *qdev, DeviceInfo *base)
> diff --git a/hw/pci.h b/hw/pci.h
> index ffb04e8..f6d20f6 100644
> --- a/hw/pci.h
> +++ b/hw/pci.h
> @@ -253,7 +253,7 @@ void pci_for_each_device(PCIBus *bus, int bus_num, void (*fn)(PCIBus *bus, PCIDe
>  PCIBus *pci_find_root_bus(int domain);
>  int pci_find_domain(const PCIBus *bus);
>  PCIBus *pci_find_bus(PCIBus *bus, int bus_num);
> -PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int slot, int function);
> +PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn);
>  int pci_qdev_find_device(const char *id, PCIDevice **pdev);
>  PCIBus *pci_get_bus_devfn(int *devfnp, const char *devaddr);
>  
> diff --git a/hw/pci_host.c b/hw/pci_host.c
> index 7c40155..728e2d4 100644
> --- a/hw/pci_host.c
> +++ b/hw/pci_host.c
> @@ -44,7 +44,7 @@ static inline PCIDevice *pci_dev_find_by_addr(PCIBus *bus, uint32_t addr)
>      uint8_t bus_num = addr >> 16;
>      uint8_t devfn = addr >> 8;
>  
> -    return pci_find_device(bus, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn));
> +    return pci_find_device(bus, bus_num, devfn);
>  }
>  
>  void pci_data_write(PCIBus *s, uint32_t addr, uint32_t val, int len)
> diff --git a/hw/pcie_host.c b/hw/pcie_host.c
> index 21069ee..b749865 100644
> --- a/hw/pcie_host.c
> +++ b/hw/pcie_host.c
> @@ -49,8 +49,7 @@ static inline PCIDevice *pcie_dev_find_by_mmcfg_addr(PCIBus *s,
>                                                       uint32_t mmcfg_addr)
>  {
>      return pci_find_device(s, PCIE_MMCFG_BUS(mmcfg_addr),
> -                           PCI_SLOT(PCIE_MMCFG_DEVFN(mmcfg_addr)),
> -                           PCI_FUNC(PCIE_MMCFG_DEVFN(mmcfg_addr)));
> +                           PCIE_MMCFG_DEVFN(mmcfg_addr));
>  }
>  
>  static void pcie_mmcfg_data_write(PCIBus *s,
> -- 
> 1.7.1.1

[Qemu-devel] Re: [PATCH 3/3] pci/pcie: make pci_find_device() ARI aware.

[Michael S. Tsirkin]

On Wed, Jan 26, 2011 at 06:45:20PM +0900, Isaku Yamahata wrote:
> make pci_find_device() ARI aware.
> 
> Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
> ---
>  hw/pci.c  |    6 ++++++
>  hw/pcie.c |   33 +++++++++++++++++++++++++++++----
>  hw/pcie.h |    2 +-
>  3 files changed, 36 insertions(+), 5 deletions(-)
> 
> diff --git a/hw/pci.c b/hw/pci.c
> index ac16029..daba310 100644
> --- a/hw/pci.c
> +++ b/hw/pci.c
> @@ -1601,12 +1601,18 @@ PCIBus *pci_find_bus(PCIBus *bus, int bus_num)
>  
>  PCIDevice *pci_find_device(PCIBus *bus, int bus_num, int devfn)
>  {
> +    PCIDevice *d;
>      assert(devfn < PCI_DEVFN_MAX);
>      bus = pci_find_bus(bus, bus_num);
>  
>      if (!bus)
>          return NULL;
>  
> +    d = bus->parent_dev;
> +    /* ARI: See the comment above the pcie_check_slot() for details */
> +    if (d && !pcie_check_slot(d, devfn)) {
> +        return NULL;
> +    }
>      return bus->devices[devfn];
>  }
>  
> diff --git a/hw/pcie.c b/hw/pcie.c
> index 6a113a9..ae4b531 100644
> --- a/hw/pcie.c
> +++ b/hw/pcie.c
> @@ -424,13 +424,38 @@ void pcie_cap_ari_reset(PCIDevice *dev)
>      pci_long_test_and_clear_mask(devctl2, PCI_EXP_DEVCTL2_ARI);
>  }
>  
> -bool pcie_cap_is_ari_enabled(const PCIDevice *dev)
> +/*
> + * 6.13 Alternative routing-ID Interpretation(ARI)
> + * 7.8.16 Device control 2 register
> + *   ARI forwarding Enable
> + *
> + * With PCI Express Endpoints, there's a single device behind
> + * each downstream port bus, and bits 3:7 of the function number get
> + * encoded in the slot number (the Express spec calls it the Device
> + * Number). This allows > 8 functions, but
> + * these extended functions are only accessible when the
> + * Alternative routing-ID Interpretation (ARI)
> + * capability is enabled in the root/downstream port. With that capability
> + * disabled the port enforces the Device Number field being 0.
> + */
> +bool pcie_check_slot(const PCIDevice *dev, int devfn)
>  {
> +    uint8_t type;
> +
>      if (!pci_is_express(dev)) {
> -        return false;
> +        return true;
>      }
> -    if (!dev->exp.exp_cap) {
> -        return false;
> +
> +    type = pcie_cap_get_type(dev);
> +    if (!(type == PCI_EXP_TYPE_ROOT_PORT ||
> +          type == PCI_EXP_TYPE_DOWNSTREAM)) {
> +        return true;
> +    }
> +
> +    if (!PCI_SLOT(devfn)) {
> +        /* With ARI, this means function < 8.
> +           functions < 8 are always accesible. */
> +        return true;
>      }
>  
>      return pci_get_long(dev->config + dev->exp.exp_cap + PCI_EXP_DEVCTL2) &
> diff --git a/hw/pcie.h b/hw/pcie.h
> index bc909e2..0c84662 100644
> --- a/hw/pcie.h
> +++ b/hw/pcie.h
> @@ -119,7 +119,7 @@ void pcie_cap_flr_write_config(PCIDevice *dev,
>  
>  void pcie_cap_ari_init(PCIDevice *dev);
>  void pcie_cap_ari_reset(PCIDevice *dev);
> -bool pcie_cap_is_ari_enabled(const PCIDevice *dev);
> +bool pcie_check_slot(const PCIDevice *dev, int devfn);

I think it's better to make it static inline.
This way pci.c depends on pcie.h but not pcie.c

>  
>  /* PCI express extended capability helper functions */
>  uint16_t pcie_find_capability(PCIDevice *dev, uint16_t cap_id);
> -- 
> 1.7.1.1