[PATCH 0/2] PCI: SUpport to workaround bus level HW issues

[PATCH 0/2] PCI: SUpport to workaround bus level HW issues

[James Puthukattukaran]

There are bugs in certain PCIe switches that cause access violations when an
endpoint device is hotplugged. In particular, there's an issue with
certain IDT switches that trigger a ACS violation when bringing up a newly
plugged PCIe endpoint device. This is a major issue for platforms
designed to issue a fatal reset in the case of this event.

The first patch provides a framework for intercepting and working around
issues with parent devices to the endpoint being brought up.

The second patch provides the actual patch for the IDT switch issue using
that framework. The ACS feature is disabled in the IDT switch prior to endpoint
device detection and then re-enabled subsequent to that.

James

-v2: move workaround to pci_bus_read_dev_vendor_id() from pci_bus_check_dev()
     and move enable_acs_sv to drivers/pci/pci.c -- by Yinghai
-v3: add bus->self check for root bus and virtual bus for sriov vfs.
-v4: only do workaround for IDT switches
-v5: tweak pci_std_enable_acs_sv to deal with unimplemented SV
-v6: Added errata verbiage verbatim and resolved patch format issues
-v7: changed int to bool for found and idt_workaround declarations. Also
     added bugzilla https://bugzilla.kernel.org/show_bug.cgi?id=196979
-v8: Rewrote the patch by adding a new acs quirk to keep the workaround
     out of the main code path
-v9: changed function name from pci_dev_specific_fixup_acs_quirk to
     pci_bus_acs_quirk. Also, tested for FLR and hot reset scenarios and did
     not see issues where workaround was required. The issue seems to be
     related only to cold reset/power on situation.
-v10: Moved the contents of pci_bus_read_vendor_id into an internal function
      __pci_bus_read_vendor_id
-v11: Split the patch into two patches. The first a general quirk framework.
-v12: Add pci_bus_generic_read_dev_vendor_id() to carry out default behavior
      when detecting endpoint and pci_bus_specific_read_dev_vendor_id for 
      bus quirk behavior
---

James Puthukattukaran (2):
  PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI
    switch     specific issues prior to accessing newly added endpoint
  PCI: Implement workaround for the ACS bug in the IDT switch

 drivers/pci/pci.h    |   4 ++
 drivers/pci/probe.c  |  19 ++++++-
 drivers/pci/quirks.c | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 160 insertions(+), 1 deletion(-)

-- 
1.8.3.1

[PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[James Puthukattukaran]

This patch provides a framework in which it would be possible to implement
bus specific quirks prior to accessing an endpoint device beneath that bus.
The routine, pci_bus_specific_read_dev_vendor_id, can be called prior to
accessing the end point device itself in order to workaround potential issues
with the parent device (switch). If there is nothing specific to be done for
a particular switch device, it falls through to check for the endpoint device
i.e pci_bus_generic_read_dev_vendor_id().

Signed-off: James Puthukattukaran <james.puthukattukaran@oracle.com>
---
 drivers/pci/pci.h    |  2 ++
 drivers/pci/probe.c  | 19 ++++++++++++++++++-
 drivers/pci/quirks.c | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 023f7cf..39ea6ee 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -225,6 +225,8 @@ enum pci_bar_type {
 int pci_configure_extended_tags(struct pci_dev *dev, void *ign);
 bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
 				int crs_timeout);
+int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
+				u32 *pl, int crs_timeout);
 int pci_setup_device(struct pci_dev *dev);
 int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 		    struct resource *res, unsigned int reg);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index ac91b6f..d8bf71b 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2097,7 +2097,7 @@ static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
 	return true;
 }
 
-bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
 				int timeout)
 {
 	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
@@ -2113,6 +2113,23 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
 
 	return true;
 }
+
+
+bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+					int timeout)
+{
+	int ret;
+
+	/* An opportunity to implement something specific for this device.
+	 * For ex, impelement a quirk prior to even accessing the device
+	 */
+	ret = pci_bus_specific_read_dev_vendor_id(bus, devfn, l, timeout);
+	if (ret >= 0)
+		return (ret >= 0);
+
+	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
+}
+
 EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
 
 /*
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 2990ad1..c32c5ec 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -4741,3 +4741,44 @@ static void quirk_gpu_hda(struct pci_dev *hda)
 			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
 			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
+
+
+static const struct pci_bus_specific_quirk{
+	u16 vendor;
+	u16 device;
+	int (*bus_quirk)(struct pci_bus *bus, int devfn, u32 *l, int timeout);
+} pci_bus_specific_quirks[] = {
+	{0}
+};
+
+/*
+ * This routine provides the ability to implement a bus specific quirk
+ * prior to doing config accesses to the endpoint device itself. For ex, there
+ * could be HW problems with the switch above the endpoint that causes issues
+ * when accessing the endpoint device. Such workarounds "specific" to the
+ * parent could be implemented prior or subsequent to accesses to the
+ * endpoint itself
+ *
+ */
+int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+					int timeout)
+{
+	const struct pci_bus_specific_quirk *i;
+	struct pci_dev *dev;
+
+	if (!bus || !bus->self)
+		return -ENOTTY;
+
+	dev = bus->self;
+
+	/* Implement any quirks in the "bus" (switch, for ex) that causes
+	 * issues in accessing the endpoint
+	 */
+	for (i = pci_bus_specific_quirks; i->bus_quirk; i++) {
+		if ((i->vendor == dev->vendor ||
+		     i->vendor == (u16)PCI_ANY_ID) &&
+		    (i->device == dev->device || i->device == (u16)PCI_ANY_ID))
+			return(i->bus_quirk(bus, devfn, l, timeout));
+	}
+	return -ENOTTY;
+}
-- 
1.8.3.1

[PATCH 2/2] PCI: Implement workaround for the ACS bug in the IDT switch

[James Puthukattukaran]

The IDT switch incorrectly flags an ACS source violation on a read config
request to an end point device on the completion (IDT 89H32H8G3-YC,
errata #36) even though the PCI Express spec states that completions are
never affected by ACS source violation (PCI Spec 3.1, Section 6.12.1.1). Here's
the specific copy of the errata text

"Item #36 - Downstream port applies ACS Source Validation to Completions
Section 6.12.1.1 of the PCI Express Base Specification 3.1 states
that completions are never affected
by ACS Source Validation. However, completions received by a
downstream port of the PCIe switch from a device that has not yet
captured a PCIe bus number are incorrectly dropped by ACS source
validation by the switch downstream port.

Workaround: Issue a CfgWr1 to the downstream device before issuing
the first CfgRd1 to the device.
This allows the downstream device to capture its bus number; ACS
source validation no longer stops
completions from being forwarded by the downstream port. It has been
observed that Microsoft Windows implements this workaround already;
however, some versions of Linux and other operating systems may not. "

The suggested workaround by IDT is to issue a configuration write to the
downstream device before issuing the first config read. This allows the
downstream device to capture its bus number, thus avoiding the ACS
violation on the completion. In order to make sure that the device is ready
for config accesses, we do what is currently done in making config reads
till it succeeds and then do the config write as specified by the errata.
However, to avoid hitting the errata issue when doing config reads, we
disable ACS SV around this process.

The patch does the following -

1. Disable ACS source violation if enabled.
2. Wait for config space access to become available by reading vendor id
3. Do a config write to the end point (errata workaround)
4. Enable ACS source validation (if it was enabled to begin with)

Signed-off-by: James Puthukattukaran <james.puthukattukaran@oracle.com>
---
 drivers/pci/pci.h    |  2 ++
 drivers/pci/quirks.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 39ea6ee..d0d588d 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -227,6 +227,8 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
 				int crs_timeout);
 int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
 				u32 *pl, int crs_timeout);
+bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn,
+				u32 *pl, int crs_timeout);
 int pci_setup_device(struct pci_dev *dev);
 int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 		    struct resource *res, unsigned int reg);
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index c32c5ec..89cd47d 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -4742,12 +4742,109 @@ static void quirk_gpu_hda(struct pci_dev *hda)
 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
 			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
 
+/*
+ * The IDT switch incorrectly flags an ACS source violation on a read config
+ * request to an end point device on the completion (IDT 89H32H8G3-YC,
+ * errata #36) even though the PCI Express spec states that completions are
+ * never affected by ACS source violation (PCI Spec 3.1, Section 6.12.1.1).
+ * Here's * the specific copy of the errata text --
+ *
+ * "Item #36 - Downstream port applies ACS Source Validation to Completions
+ * Section 6.12.1.1 of the PCI Express Base Specification 3.1 states
+ * that completions are never affected
+ * by ACS Source Validation. However, completions received by a
+ * downstream port of the PCIe switch from a device that has not yet
+ * captured a PCIe bus number are incorrectly dropped by ACS source
+ * validation by the switch downstream port."
+ *
+ * The suggested workaround by IDT is to issue a configuration write to the
+ * downstream device before issuing the first config read. This allows the
+ * downstream device to capture its bus number, thus avoiding the ACS
+ * violation on the completion. In order to make sure that the device is ready
+ * for config accesses, we do what is currently done in making config reads
+ * till it succeeds and then do the config write as specified by the errata.
+ * However, to avoid hitting the errata issue when doing config reads, we
+ * disable ACS SV around this process.
+ */
+static int pci_idt_acs_quirk(struct pci_bus *bus, int devfn, int enable,
+				bool found)
+{
+	int pos;
+	u16 cap;
+	u16 ctrl;
+	int retval;
+	struct pci_dev *dev = bus->self;
+
+
+	/* Write 0 to the devfn device under the PCIE switch (bus->self)
+	 * as part of forcing the devfn number to latch with the device
+	 * below
+	 */
+	if (found)
+		pci_bus_write_config_word(bus, devfn, PCI_VENDOR_ID, 0);
+
+
+	/* Enable/disable ACS SV feature (based on enable flag) */
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
+	if (!pos)
+		return -ENODEV;
+
+	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
+
+	if (!(cap & PCI_ACS_SV))
+		return -ENODEV;
+
+	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
+
+	retval = !!(ctrl & cap & PCI_ACS_SV);
+	if (enable)
+		ctrl |= (cap & PCI_ACS_SV);
+	else
+		ctrl &= ~(cap & PCI_ACS_SV);
+
+	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
+
+	/* return the previous state of the ACS SV state i.e was SV enabled
+	 * or disabled?
+	 */
+	return retval;
+}
+
+static int pci_idt_bus_quirk(struct pci_bus *bus, int devfn, u32 *l,
+				int timeout)
+{
+	int enable;
+	bool found;
+
+	/*
+	 * Disable acs for the IDT switch before attempting the initial
+	 * config accesses to the endpoint device.
+	 */
+	enable = pci_idt_acs_quirk(bus, devfn, 0, false);
+
+	/*
+	 * found indicates whether the endpoint device was identified
+	 * as present or not
+	 */
+
+	found = pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
+
+	/*
+	 * re-enable acs feature for the switch again if it was enabled to
+	 * start with
+	 */
+	if (enable > 0)
+		pci_idt_acs_quirk(bus, devfn, enable, found);
+
+	return found ? 1 : 0;
+}
 
 static const struct pci_bus_specific_quirk{
 	u16 vendor;
 	u16 device;
 	int (*bus_quirk)(struct pci_bus *bus, int devfn, u32 *l, int timeout);
 } pci_bus_specific_quirks[] = {
+	{ PCI_VENDOR_ID_IDT, 0x80b5, pci_idt_bus_quirk},
 	{0}
 };
 
-- 
1.8.3.1

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[Alex Williamson]

On Tue, 24 Apr 2018 10:53:08 -0400
James Puthukattukaran <james.puthukattukaran@oracle.com> wrote:

> This patch provides a framework in which it would be possible to implement
> bus specific quirks prior to accessing an endpoint device beneath that bus.
> The routine, pci_bus_specific_read_dev_vendor_id, can be called prior to
> accessing the end point device itself in order to workaround potential issues
> with the parent device (switch). If there is nothing specific to be done for
> a particular switch device, it falls through to check for the endpoint device
> i.e pci_bus_generic_read_dev_vendor_id().
> 
> Signed-off: James Puthukattukaran <james.puthukattukaran@oracle.com>
            ^ -by

> ---
>  drivers/pci/pci.h    |  2 ++
>  drivers/pci/probe.c  | 19 ++++++++++++++++++-
>  drivers/pci/quirks.c | 41 +++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 61 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index 023f7cf..39ea6ee 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -225,6 +225,8 @@ enum pci_bar_type {
>  int pci_configure_extended_tags(struct pci_dev *dev, void *ign);
>  bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
>  				int crs_timeout);
> +int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
> +				u32 *pl, int crs_timeout);
>  int pci_setup_device(struct pci_dev *dev);
>  int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
>  		    struct resource *res, unsigned int reg);
> diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
> index ac91b6f..d8bf71b 100644
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -2097,7 +2097,7 @@ static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
>  	return true;
>  }
>  
> -bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
> +bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
>  				int timeout)
>  {
>  	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
> @@ -2113,6 +2113,23 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
>  
>  	return true;
>  }
> +
> +
> +bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
> +					int timeout)
> +{
> +	int ret;
> +
> +	/* An opportunity to implement something specific for this device.
> +	 * For ex, impelement a quirk prior to even accessing the device

s/impelement/implement/

Otherwise,

Reviewed-by: Alex Williamson <alex.williamson@redhat.com>

> +	 */
> +	ret = pci_bus_specific_read_dev_vendor_id(bus, devfn, l, timeout);
> +	if (ret >= 0)
> +		return (ret >= 0);
> +
> +	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
> +}
> +
>  EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
>  
>  /*
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index 2990ad1..c32c5ec 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -4741,3 +4741,44 @@ static void quirk_gpu_hda(struct pci_dev *hda)
>  			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
>  DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
>  			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
> +
> +
> +static const struct pci_bus_specific_quirk{
> +	u16 vendor;
> +	u16 device;
> +	int (*bus_quirk)(struct pci_bus *bus, int devfn, u32 *l, int timeout);
> +} pci_bus_specific_quirks[] = {
> +	{0}
> +};
> +
> +/*
> + * This routine provides the ability to implement a bus specific quirk
> + * prior to doing config accesses to the endpoint device itself. For ex, there
> + * could be HW problems with the switch above the endpoint that causes issues
> + * when accessing the endpoint device. Such workarounds "specific" to the
> + * parent could be implemented prior or subsequent to accesses to the
> + * endpoint itself
> + *
> + */
> +int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
> +					int timeout)
> +{
> +	const struct pci_bus_specific_quirk *i;
> +	struct pci_dev *dev;
> +
> +	if (!bus || !bus->self)
> +		return -ENOTTY;
> +
> +	dev = bus->self;
> +
> +	/* Implement any quirks in the "bus" (switch, for ex) that causes
> +	 * issues in accessing the endpoint
> +	 */
> +	for (i = pci_bus_specific_quirks; i->bus_quirk; i++) {
> +		if ((i->vendor == dev->vendor ||
> +		     i->vendor == (u16)PCI_ANY_ID) &&
> +		    (i->device == dev->device || i->device == (u16)PCI_ANY_ID))
> +			return(i->bus_quirk(bus, devfn, l, timeout));
> +	}
> +	return -ENOTTY;
> +}

Re: [PATCH 2/2] PCI: Implement workaround for the ACS bug in the IDT switch

[Alex Williamson]

On Tue, 24 Apr 2018 10:54:26 -0400
James Puthukattukaran <james.puthukattukaran@oracle.com> wrote:

> The IDT switch incorrectly flags an ACS source violation on a read config
> request to an end point device on the completion (IDT 89H32H8G3-YC,
> errata #36) even though the PCI Express spec states that completions are
> never affected by ACS source violation (PCI Spec 3.1, Section 6.12.1.1). Here's
> the specific copy of the errata text
> 
> "Item #36 - Downstream port applies ACS Source Validation to Completions
> Section 6.12.1.1 of the PCI Express Base Specification 3.1 states
> that completions are never affected
> by ACS Source Validation. However, completions received by a
> downstream port of the PCIe switch from a device that has not yet
> captured a PCIe bus number are incorrectly dropped by ACS source
> validation by the switch downstream port.
> 
> Workaround: Issue a CfgWr1 to the downstream device before issuing
> the first CfgRd1 to the device.
> This allows the downstream device to capture its bus number; ACS
> source validation no longer stops
> completions from being forwarded by the downstream port. It has been
> observed that Microsoft Windows implements this workaround already;
> however, some versions of Linux and other operating systems may not. "
> 
> The suggested workaround by IDT is to issue a configuration write to the
> downstream device before issuing the first config read. This allows the
> downstream device to capture its bus number, thus avoiding the ACS
> violation on the completion. In order to make sure that the device is ready
> for config accesses, we do what is currently done in making config reads
> till it succeeds and then do the config write as specified by the errata.
> However, to avoid hitting the errata issue when doing config reads, we
> disable ACS SV around this process.
> 
> The patch does the following -
> 
> 1. Disable ACS source violation if enabled.
> 2. Wait for config space access to become available by reading vendor id
> 3. Do a config write to the end point (errata workaround)
> 4. Enable ACS source validation (if it was enabled to begin with)
> 
> Signed-off-by: James Puthukattukaran <james.puthukattukaran@oracle.com>
> ---
>  drivers/pci/pci.h    |  2 ++
>  drivers/pci/quirks.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 99 insertions(+)
> 
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index 39ea6ee..d0d588d 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -227,6 +227,8 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
>  				int crs_timeout);
>  int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
>  				u32 *pl, int crs_timeout);
> +bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn,
> +				u32 *pl, int crs_timeout);
>  int pci_setup_device(struct pci_dev *dev);
>  int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
>  		    struct resource *res, unsigned int reg);
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index c32c5ec..89cd47d 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -4742,12 +4742,109 @@ static void quirk_gpu_hda(struct pci_dev *hda)
>  DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
>  			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
>  
> +/*
> + * The IDT switch incorrectly flags an ACS source violation on a read config
> + * request to an end point device on the completion (IDT 89H32H8G3-YC,
> + * errata #36) even though the PCI Express spec states that completions are
> + * never affected by ACS source violation (PCI Spec 3.1, Section 6.12.1.1).
> + * Here's * the specific copy of the errata text --
> + *
> + * "Item #36 - Downstream port applies ACS Source Validation to Completions
> + * Section 6.12.1.1 of the PCI Express Base Specification 3.1 states
> + * that completions are never affected
> + * by ACS Source Validation. However, completions received by a
> + * downstream port of the PCIe switch from a device that has not yet
> + * captured a PCIe bus number are incorrectly dropped by ACS source
> + * validation by the switch downstream port."
> + *
> + * The suggested workaround by IDT is to issue a configuration write to the
> + * downstream device before issuing the first config read. This allows the
> + * downstream device to capture its bus number, thus avoiding the ACS
> + * violation on the completion. In order to make sure that the device is ready
> + * for config accesses, we do what is currently done in making config reads
> + * till it succeeds and then do the config write as specified by the errata.
> + * However, to avoid hitting the errata issue when doing config reads, we
> + * disable ACS SV around this process.
> + */
> +static int pci_idt_acs_quirk(struct pci_bus *bus, int devfn, int enable,
> +				bool found)
> +{
> +	int pos;
> +	u16 cap;
> +	u16 ctrl;
> +	int retval;
> +	struct pci_dev *dev = bus->self;
> +
> +
> +	/* Write 0 to the devfn device under the PCIE switch (bus->self)
> +	 * as part of forcing the devfn number to latch with the device
> +	 * below
> +	 */
> +	if (found)
> +		pci_bus_write_config_word(bus, devfn, PCI_VENDOR_ID, 0);
> +
> +
> +	/* Enable/disable ACS SV feature (based on enable flag) */
> +	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
> +	if (!pos)
> +		return -ENODEV;
> +
> +	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
> +
> +	if (!(cap & PCI_ACS_SV))
> +		return -ENODEV;
> +
> +	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
> +
> +	retval = !!(ctrl & cap & PCI_ACS_SV);
> +	if (enable)
> +		ctrl |= (cap & PCI_ACS_SV);
> +	else
> +		ctrl &= ~(cap & PCI_ACS_SV);
> +
> +	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
> +
> +	/* return the previous state of the ACS SV state i.e was SV enabled
> +	 * or disabled?
> +	 */
> +	return retval;
> +}
> +
> +static int pci_idt_bus_quirk(struct pci_bus *bus, int devfn, u32 *l,
> +				int timeout)
> +{
> +	int enable;
> +	bool found;
> +
> +	/*
> +	 * Disable acs for the IDT switch before attempting the initial
> +	 * config accesses to the endpoint device.
> +	 */
> +	enable = pci_idt_acs_quirk(bus, devfn, 0, false);
> +
> +	/*
> +	 * found indicates whether the endpoint device was identified
> +	 * as present or not
> +	 */
> +
> +	found = pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
> +
> +	/*
> +	 * re-enable acs feature for the switch again if it was enabled to
> +	 * start with
> +	 */


Inconsistent comment style even within the same patch.  Otherwise,

Reviewed-by: Alex Williamson <alex.williamson@redhat.com>


> +	if (enable > 0)
> +		pci_idt_acs_quirk(bus, devfn, enable, found);
> +
> +	return found ? 1 : 0;
> +}
>  
>  static const struct pci_bus_specific_quirk{
>  	u16 vendor;
>  	u16 device;
>  	int (*bus_quirk)(struct pci_bus *bus, int devfn, u32 *l, int timeout);
>  } pci_bus_specific_quirks[] = {
> +	{ PCI_VENDOR_ID_IDT, 0x80b5, pci_idt_bus_quirk},
>  	{0}
>  };
>  

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 1136 bytes --]

Hi James,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on pci/next]
[also build test ERROR on v4.17-rc2 next-20180424]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/James-Puthukattukaran/PCI-SUpport-to-workaround-bus-level-HW-issues/20180426-092305
base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
config: ia64-allnoconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 7.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=ia64 

All errors (new ones prefixed by >>):

   drivers/pci/probe.o: In function `pci_bus_read_dev_vendor_id':
>> probe.c:(.text+0x3e02): undefined reference to `pci_bus_specific_read_dev_vendor_id'

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 5446 bytes --]

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 1227 bytes --]

Hi James,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on pci/next]
[also build test ERROR on v4.17-rc2 next-20180426]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/James-Puthukattukaran/PCI-SUpport-to-workaround-bus-level-HW-issues/20180426-092305
base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
config: alpha-allnoconfig (attached as .config)
compiler: alpha-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=alpha 

All errors (new ones prefixed by >>):

   drivers/pci/probe.o: In function `pci_bus_read_dev_vendor_id':
>> (.text+0x201c): undefined reference to `pci_bus_specific_read_dev_vendor_id'
   (.text+0x2044): undefined reference to `pci_bus_specific_read_dev_vendor_id'

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 4391 bytes --]

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[James Puthukattukaran]

On 04/26/2018 02:26 AM, kbuild test robot wrote:
> Hi James,
> 
> Thank you for the patch! Yet something to improve:
> 
> [auto build test ERROR on pci/next]
> [also build test ERROR on v4.17-rc2 next-20180424]
> [if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
> 
> url:    https://github.com/0day-ci/linux/commits/James-Puthukattukaran/PCI-SUpport-to-workaround-bus-level-HW-issues/20180426-092305
> base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
> config: ia64-allnoconfig (attached as .config)
> compiler: ia64-linux-gcc (GCC) 7.2.0
> reproduce:
>         wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>         chmod +x ~/bin/make.cross
>         # save the attached .config to linux build tree
>         make.cross ARCH=ia64 
> 
> All errors (new ones prefixed by >>):
> 
>    drivers/pci/probe.o: In function `pci_bus_read_dev_vendor_id':
>>> probe.c:(.text+0x3e02): undefined reference to `pci_bus_specific_read_dev_vendor_id'

Alex/Sinan -

Would it be an acceptable fix for this problem?


--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2097,6 +2097,13 @@ static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
 	return true;
 }
 
+int __weak pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn, 
+					u32 *l, int timeout)
+{
+	return -ENOTTY;
+}
+
+
 bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
 				int timeout)
 {


I ran the repro steps and it completes without issues. In addition, I will fix the typos Alex pointed out in the last submission and resubmit.
thanks
James

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[Alex Williamson]

On Thu, 26 Apr 2018 15:32:00 -0400
James Puthukattukaran <james.puthukattukaran@oracle.com> wrote:

> On 04/26/2018 02:26 AM, kbuild test robot wrote:
> > Hi James,
> > 
> > Thank you for the patch! Yet something to improve:
> > 
> > [auto build test ERROR on pci/next]
> > [also build test ERROR on v4.17-rc2 next-20180424]
> > [if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
> > 
> > url:    https://github.com/0day-ci/linux/commits/James-Puthukattukaran/PCI-SUpport-to-workaround-bus-level-HW-issues/20180426-092305
> > base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
> > config: ia64-allnoconfig (attached as .config)
> > compiler: ia64-linux-gcc (GCC) 7.2.0
> > reproduce:
> >         wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
> >         chmod +x ~/bin/make.cross
> >         # save the attached .config to linux build tree
> >         make.cross ARCH=ia64 
> > 
> > All errors (new ones prefixed by >>):
> > 
> >    drivers/pci/probe.o: In function `pci_bus_read_dev_vendor_id':  
> >>> probe.c:(.text+0x3e02): undefined reference to `pci_bus_specific_read_dev_vendor_id'  
> 
> Alex/Sinan -
> 
> Would it be an acceptable fix for this problem?
> 
> 
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -2097,6 +2097,13 @@ static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
>  	return true;
>  }
>  
> +int __weak pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn, 
> +					u32 *l, int timeout)
> +{
> +	return -ENOTTY;
> +}
> +
> +

Close, but I'd suggest following the example set by other quirks,
generally there's a stub definition of it in pci.h when !#defined
CONFIG_PCI_QUIRKS.  See for example pci_dev_specific_acs_enabled().
Thanks,

Alex

>  bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
>  				int timeout)
>  {
> 
> 
> I ran the repro steps and it completes without issues. In addition, I will fix the typos Alex pointed out in the last submission and resubmit.
> thanks
> James

Re: [PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[Sinan Kaya]

On 4/26/2018 3:32 PM, James Puthukattukaran wrote:
>> All errors (new ones prefixed by >>):
>>
>>    drivers/pci/probe.o: In function `pci_bus_read_dev_vendor_id':
>>>> probe.c:(.text+0x3e02): undefined reference to `pci_bus_specific_read_dev_vendor_id'
> Alex/Sinan -
> 
> Would it be an acceptable fix for this problem?
> 
> 
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -2097,6 +2097,13 @@ static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
>  	return true;
>  }
>  
> +int __weak pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn, 
> +					u32 *l, int timeout)
> +{
> +	return -ENOTTY;
> +}
> +
> +

I don't think so. pci_bus_specific_read_dev_vendor_id() is defined in quirks.c if
CONFIG_PCI_QUIRKS is defined only. 

You need to add something similar to drivers/pci.h file

#ifdef CONFIG_PCI_QUIRKS
int pci_dev_specific_reset(struct pci_dev *dev, int probe);
#else
static inline int pci_dev_specific_reset(struct pci_dev *dev, int probe)
{
	return -ENOTTY;
}
#endif 


while you are there please convert

> +
> +

to 

> +

in your patch and bump the version number on your next post using 

--subject-prefix="PATCH V2"

to your git send-email command.

-- 
Sinan Kaya
Qualcomm Datacenter Technologies, Inc. as an affiliate of Qualcomm Technologies, Inc.
Qualcomm Technologies, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project.

[PATCH 1/2] PCI: Add pci_bus_specific_read_dev_vendor_id() to workaround PCI switch specific issues prior to accessing newly added endpoint

[James Puthukattukaran]

This patch provides a framework in which it would be possible to implement
bus specific quirks prior to accessing an endpoint device beneath that bus.
The routine, pci_bus_specific_read_dev_vendor_id, can be called prior to
accessing the end point device itself in order to workaround potential
issues
with the parent device (switch). If there is nothing specific to be done for
a particular switch device, it falls through to check for the endpoint
device
i.e pci_bus_generic_read_dev_vendor_id().

Signed-off: James Puthukattukaran <james.puthukattukaran@oracle.com>
---
 drivers/pci/pci.h    |  2 ++
 drivers/pci/probe.c  | 19 ++++++++++++++++++-
 drivers/pci/quirks.c | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 023f7cf..2d06689 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -225,6 +225,8 @@ enum pci_bar_type {
 int pci_configure_extended_tags(struct pci_dev *dev, void *ign);
 bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
 				int crs_timeout);
+int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
+				u32 *pl, int crs_timeout);
 int pci_setup_device(struct pci_dev *dev);
 int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 		    struct resource *res, unsigned int reg);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index ac91b6f..b4d8cbd 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2097,7 +2097,7 @@ static bool pci_bus_wait_crs(struct pci_bus *bus,
int devfn, u32 *l,
 	return true;
 }

-bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn,
u32 *l,
 				int timeout)
 {
 	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
@@ -2113,6 +2113,23 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus
*bus, int devfn, u32 *l,

 	return true;
 }
+
+
+bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+					int timeout)
+{
+	int ret;
+
+	/* An opportunity to implement something specific for this device.
+         * For ex, impelement a quirk prior to even accessing the device
+         */
+	ret = pci_bus_specific_read_dev_vendor_id(bus, devfn, l, timeout);
+	if (ret >= 0)
+		return (ret >= 0);
+
+	return(pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout));
+}
+
 EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);

 /*
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 2990ad1..c637162 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -4741,3 +4741,44 @@ static void quirk_gpu_hda(struct pci_dev *hda)
 			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
 			      PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, quirk_gpu_hda);
+
+
+static const struct pci_bus_specific_quirk{
+	u16 vendor;
+	u16 device;
+	int (*bus_quirk)(struct pci_bus *bus, int devfn, u32 *l, int timeout);
+} pci_bus_specific_quirks[] = {
+	{0}
+};
+
+/*
+ * This routine provides the ability to implement a bus specific quirk
+ * prior to doing config accesses to the endpoint device itself. For
ex, there
+ * could be HW problems with the switch above the endpoint that causes
issues
+ * when accessing the endpoint device. Such workarounds "specific" to the
+ * parent could be implemented prior or subsequent to accesses to the
+ * endpoint itself
+ *
+ */
+int pci_bus_specific_read_dev_vendor_id(struct pci_bus *bus, int devfn,
u32 *l,
+					int timeout)
+{
+	const struct pci_bus_specific_quirk *i;
+	struct pci_dev *dev;
+
+	if (!bus || !bus->self)
+	        return -ENOTTY;
+
+	dev = bus->self;
+
+	/* Implement any quirks in the "bus" (switch, for ex) that causes
+	 * issues in accessing the endpoint */
+	for (i = pci_bus_specific_quirks; i->bus_quirk; i++) {
+		if ((i->vendor == dev->vendor ||
+			i->vendor == (u16)PCI_ANY_ID) &&
+			(i->device == dev->device ||
+			i->device == (u16)PCI_ANY_ID))
+				return(i->bus_quirk(bus, devfn, l, timeout));
+	}
+	return -ENOTTY;
+}
-- 
1.8.3.1