[PATCH 0/2] VMD subdevice secondary bus resets

[PATCH 0/2] VMD subdevice secondary bus resets

[Jon Derrick]

This set adds some resets for VMD. It's very common code but doesn't
seem to fit well anywhere that can also be exported if VMD is built as a
module.

Jon Derrick (2):
  PCI: vmd: Reset the VMD subdevice domain on probe
  PCI: Add a reset quirk for VMD

 drivers/pci/controller/vmd.c | 32 ++++++++++++++++++++++++
 drivers/pci/quirks.c         | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)

-- 
2.18.1

[PATCH 1/2] PCI: vmd: Reset the VMD subdevice domain on probe

[Jon Derrick]

The VMD subdevice domain resource requirements may have changed
in-between module loads. Generic PCI resource assignment code may rely
on existing resource configuration rather than the VMD preference of
re-examining the domain. Add a Secondary Bus Reset to the VMD subdevice
domain during driver attachment to clear the PCI config space of the
subdevices.

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/controller/vmd.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 50b3520d261d..676acff3622f 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -445,6 +445,36 @@ static struct pci_ops vmd_ops = {
 	.write		= vmd_pci_write,
 };
 
+static void vmd_domain_reset_sbr(struct vmd_dev *vmd)
+{
+	char __iomem *base;
+	int rp;
+	u16 ctl;
+
+	/*
+	 * Subdevice config space is mapped linearly using 4k config space
+	 * increments. Use increments of 0x8000 to locate root ports devices.
+	 */
+	for (rp = 0; rp < 4; rp++) {
+		base = vmd->cfgbar + rp * 0x8000;
+		if (readl(base + PCI_COMMAND) == 0xFFFFFFFF)
+			continue;
+
+		/* pci_reset_secondary_bus() */
+		ctl = readw(base + PCI_BRIDGE_CONTROL);
+		ctl |= PCI_BRIDGE_CTL_BUS_RESET;
+		writew(ctl, base + PCI_BRIDGE_CONTROL);
+		readw(base + PCI_BRIDGE_CONTROL);
+		msleep(2);
+
+		ctl &= ~PCI_BRIDGE_CTL_BUS_RESET;
+		writew(ctl, base + PCI_BRIDGE_CONTROL);
+		readw(base + PCI_BRIDGE_CONTROL);
+	}
+
+	ssleep(1);
+}
+
 static void vmd_attach_resources(struct vmd_dev *vmd)
 {
 	vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];
@@ -784,6 +814,7 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	if (!vmd->cfgbar)
 		return -ENOMEM;
 
+	vmd_domain_reset_sbr(dev);
 	pci_set_master(dev);
 	if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
 	    dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
-- 
2.18.1

[PATCH 2/2] PCI: Add a reset quirk for VMD

[Jon Derrick]

VMD domains should be reset in-between special attachment such as VFIO
users. VMD does not offer a reset however the subdevice domain itself
can be reset starting at the Root Bus. Add a Secondary Bus Reset on each
of the individual root port devices immediately downstream of the VMD
root bus.

Signed-off-by: Jon Derrick <jonathan.derrick@intel.com>
---
 drivers/pci/controller/vmd.c |  3 ++-
 drivers/pci/quirks.c         | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 676acff3622f..fdc1a206f73e 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -14,6 +14,7 @@
 #include <linux/srcu.h>
 #include <linux/rculist.h>
 #include <linux/rcupdate.h>
+#include <linux/delay.h>
 
 #include <asm/irqdomain.h>
 #include <asm/device.h>
@@ -814,7 +815,7 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	if (!vmd->cfgbar)
 		return -ENOMEM;
 
-	vmd_domain_reset_sbr(dev);
+	vmd_domain_reset_sbr(vmd);
 	pci_set_master(dev);
 	if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&
 	    dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32)))
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 8f409d442bc0..f6a9c2b2625a 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3801,6 +3801,49 @@ static int reset_ivb_igd(struct pci_dev *dev, int probe)
 	return 0;
 }
 
+/* Issues SBR to VMD domain to clear PCI configuration */
+static int reset_vmd_sbr(struct pci_dev *dev, int probe)
+{
+	char __iomem *cfgbar, *base;
+	int rp;
+	u16 ctl;
+
+	if (probe)
+		return 0;
+
+	if (dev->dev.driver)
+		return 0;
+
+	cfgbar = pci_iomap(dev, 0, 0);
+	if (!cfgbar)
+		return -ENOMEM;
+
+	/*
+	 * Subdevice config space is mapped linearly using 4k config space
+	 * increments. Use increments of 0x8000 to locate root ports devices.
+	 */
+	for (rp = 0; rp < 4; rp++) {
+		base = cfgbar + rp * 0x8000;
+		if (readl(base + PCI_COMMAND) == 0xFFFFFFFF)
+			continue;
+
+		/* pci_reset_secondary_bus() */
+		ctl = readw(base + PCI_BRIDGE_CONTROL);
+		ctl |= PCI_BRIDGE_CTL_BUS_RESET;
+		writew(ctl, base + PCI_BRIDGE_CONTROL);
+		readw(base + PCI_BRIDGE_CONTROL);
+		msleep(2);
+
+		ctl &= ~PCI_BRIDGE_CTL_BUS_RESET;
+		writew(ctl, base + PCI_BRIDGE_CONTROL);
+		readw(base + PCI_BRIDGE_CONTROL);
+	}
+
+	ssleep(1);
+	pci_iounmap(dev, cfgbar);
+	return 0;
+}
+
 /* Device-specific reset method for Chelsio T4-based adapters */
 static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
 {
@@ -3976,6 +4019,11 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 		reset_ivb_igd },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_M2_VGA,
 		reset_ivb_igd },
+	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_201D, reset_vmd_sbr },
+	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0, reset_vmd_sbr },
+	{ PCI_VENDOR_ID_INTEL, 0x467f, reset_vmd_sbr },
+	{ PCI_VENDOR_ID_INTEL, 0x4c3d, reset_vmd_sbr },
+	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B, reset_vmd_sbr },
 	{ PCI_VENDOR_ID_SAMSUNG, 0xa804, nvme_disable_and_flr },
 	{ PCI_VENDOR_ID_INTEL, 0x0953, delay_250ms_after_flr },
 	{ PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
-- 
2.18.1

Re: [PATCH 0/2] VMD subdevice secondary bus resets

[Lorenzo Pieralisi]

On Sun, Sep 27, 2020 at 09:05:55PM -0400, Jon Derrick wrote:
> This set adds some resets for VMD. It's very common code but doesn't
> seem to fit well anywhere that can also be exported if VMD is built as a
> module.
> 
> Jon Derrick (2):
>   PCI: vmd: Reset the VMD subdevice domain on probe
>   PCI: Add a reset quirk for VMD
> 
>  drivers/pci/controller/vmd.c | 32 ++++++++++++++++++++++++
>  drivers/pci/quirks.c         | 48 ++++++++++++++++++++++++++++++++++++
>  2 files changed, 80 insertions(+)

I can queue it up but I need Bjorn's ACK on patch (2).

Lorenzo

Re: [PATCH 0/2] VMD subdevice secondary bus resets

[Derrick, Jonathan]

On Thu, 2020-11-19 at 11:37 +0000, Lorenzo Pieralisi wrote:
> On Sun, Sep 27, 2020 at 09:05:55PM -0400, Jon Derrick wrote:
> > This set adds some resets for VMD. It's very common code but doesn't
> > seem to fit well anywhere that can also be exported if VMD is built as a
> > module.
> > 
> > Jon Derrick (2):
> >   PCI: vmd: Reset the VMD subdevice domain on probe
> >   PCI: Add a reset quirk for VMD
> > 
> >  drivers/pci/controller/vmd.c | 32 ++++++++++++++++++++++++
> >  drivers/pci/quirks.c         | 48 ++++++++++++++++++++++++++++++++++++
> >  2 files changed, 80 insertions(+)
> 
> I can queue it up but I need Bjorn's ACK on patch (2).
> 
> Lorenzo

I just noticed 2/2 fixes something in 1/2, so I will send a v2 for this
set.