[PATCH V5 0/2] PCI/DPC: Add eDPC support

[PATCH V5 0/2] PCI/DPC: Add eDPC support

[Dongdong Liu]

This patchset is to add eDPC support and use a local
"struct device *dev" for brevity and consistency in DPC driver.

v4->v5:
- Fix some compile warnings.

v3->v4:
- Use pcie_device's device to keep consistency.
- Adjust patch sequence.
 
v2->v3: 
- Add a separate patch to use a local "struct device *dev" 
  for brevity and consistency in  DPC driver.
- Fix comments on PATCH V2.
- Rebase on v4.13-rc5.

v1->v2: 
- Use a stack local variable instead of the allocated memory for
  collecting RP PIO information.
- Fix the condition of RP PIO error.
- Rebase on v4.13-rc1.

Dongdong Liu (2):
  PCI/DPC: Add eDPC support
  PCI/DPC: Add local struct device

 drivers/pci/pcie/pcie-dpc.c   | 187 +++++++++++++++++++++++++++++++++++++++---
 include/uapi/linux/pci_regs.h |  10 +++
 2 files changed, 187 insertions(+), 10 deletions(-)

-- 
1.9.1

[PATCH V5 1/2] PCI/DPC: Add eDPC support

[Dongdong Liu]

This code is to add eDPC support. Get and print the RP PIO error
information when the trigger condition is RP PIO error.

For more information on eDPC, please see PCI Express Base Specification
Revision 3.1, section 6.2.10.3, or view the PCI-SIG eDPC ECN here:
https://pcisig.com/sites/default/files/specification_documents/ECN_Enhanced_DPC_2012-11-19_final.pdf

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/pci/pcie/pcie-dpc.c   | 162 ++++++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/pci_regs.h |  10 +++
 2 files changed, 172 insertions(+)

diff --git a/drivers/pci/pcie/pcie-dpc.c b/drivers/pci/pcie/pcie-dpc.c
index c39f32e..8428d74 100644
--- a/drivers/pci/pcie/pcie-dpc.c
+++ b/drivers/pci/pcie/pcie-dpc.c
@@ -16,11 +16,55 @@
 #include <linux/pcieport_if.h>
 #include "../pci.h"
 
+struct rp_pio_header_log_regs {
+	u32 dw0;
+	u32 dw1;
+	u32 dw2;
+	u32 dw3;
+};
+
+struct dpc_rp_pio_regs {
+	u32 status;
+	u32 mask;
+	u32 severity;
+	u32 syserror;
+	u32 exception;
+
+	struct rp_pio_header_log_regs header_log;
+	u32 impspec_log;
+	u32 tlp_prefix_log[4];
+	u32 log_size;
+	u16 first_error;
+};
+
 struct dpc_dev {
 	struct pcie_device	*dev;
 	struct work_struct	work;
 	int			cap_pos;
 	bool			rp;
+	u32			rp_pio_status;
+};
+
+static const char * const rp_pio_error_string[] = {
+	"Configuration Request received UR Completion",	 /* Bit Position 0  */
+	"Configuration Request received CA Completion",	 /* Bit Position 1  */
+	"Configuration Request Completion Timeout",	 /* Bit Position 2  */
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	"I/O Request received UR Completion",		 /* Bit Position 8  */
+	"I/O Request received CA Completion",		 /* Bit Position 9  */
+	"I/O Request Completion Timeout",		 /* Bit Position 10 */
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	"Memory Request received UR Completion",	 /* Bit Position 16 */
+	"Memory Request received CA Completion",	 /* Bit Position 17 */
+	"Memory Request Completion Timeout",		 /* Bit Position 18 */
 };
 
 static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
@@ -79,10 +123,124 @@ static void interrupt_event_handler(struct work_struct *work)
 	dpc_wait_link_inactive(pdev);
 	if (dpc->rp && dpc_wait_rp_inactive(dpc))
 		return;
+	if (dpc->rp && dpc->rp_pio_status) {
+		pci_write_config_dword(pdev,
+				      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_STATUS,
+				      dpc->rp_pio_status);
+		dpc->rp_pio_status = 0;
+	}
+
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS,
 		PCI_EXP_DPC_STATUS_TRIGGER | PCI_EXP_DPC_STATUS_INTERRUPT);
 }
 
+static void dpc_rp_pio_print_tlp_header(struct device *dev,
+					struct rp_pio_header_log_regs *t)
+{
+	dev_err(dev, "TLP Header: %#010x %#010x %#010x %#010x\n",
+		t->dw0, t->dw1, t->dw2, t->dw3);
+}
+
+static void dpc_rp_pio_print_error(struct dpc_dev *dpc,
+				   struct dpc_rp_pio_regs *rp_pio)
+{
+	struct device *dev = &dpc->dev->device;
+	int i;
+	u32 status;
+
+	dev_err(dev, "rp_pio_status: %#010x, rp_pio_mask: %#010x\n",
+		rp_pio->status, rp_pio->mask);
+
+	dev_err(dev, "RP PIO severity=%#010x, syserror=%#010x, exception=%#010x\n",
+		rp_pio->severity, rp_pio->syserror, rp_pio->exception);
+
+	status = (rp_pio->status & ~rp_pio->mask);
+
+	for (i = 0; i < ARRAY_SIZE(rp_pio_error_string); i++) {
+		if (!(status & (1 << i)))
+			continue;
+
+		dev_err(dev, "[%2d] %s%s\n", i, rp_pio_error_string[i],
+			rp_pio->first_error == i ? " (First)" : "");
+	}
+
+	dpc_rp_pio_print_tlp_header(dev, &rp_pio->header_log);
+	if (rp_pio->log_size == 4)
+		return;
+	dev_err(dev, "RP PIO ImpSpec Log %#010x\n", rp_pio->impspec_log);
+
+	for (i = 0; i < rp_pio->log_size - 5; i++)
+		dev_err(dev, "TLP Prefix Header: dw%d, %#010x\n", i,
+			rp_pio->tlp_prefix_log[i]);
+}
+
+static void dpc_rp_pio_get_info(struct dpc_dev *dpc,
+				struct dpc_rp_pio_regs *rp_pio)
+{
+	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
+	int i;
+	u16 cap;
+	u16 status;
+
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_STATUS,
+			      &rp_pio->status);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_MASK,
+			      &rp_pio->mask);
+
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_SEVERITY,
+			      &rp_pio->severity);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_SYSERROR,
+			      &rp_pio->syserror);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_EXCEPTION,
+			      &rp_pio->exception);
+
+	/* Get First Error Pointer */
+	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
+	rp_pio->first_error = (status & 0x1f00) >> 8;
+
+	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CAP, &cap);
+	rp_pio->log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
+	if (rp_pio->log_size < 4 || rp_pio->log_size > 9) {
+		dev_err(dev, "RP PIO log size %u is invalid\n",
+			rp_pio->log_size);
+		return;
+	}
+
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG,
+			      &rp_pio->header_log.dw0);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 4,
+			      &rp_pio->header_log.dw1);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 8,
+			      &rp_pio->header_log.dw2);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 12,
+			      &rp_pio->header_log.dw3);
+	if (rp_pio->log_size == 4)
+		return;
+
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG,
+			      &rp_pio->impspec_log);
+	for (i = 0; i < rp_pio->log_size - 5; i++)
+		pci_read_config_dword(pdev,
+			dpc->cap_pos + PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG,
+			&rp_pio->tlp_prefix_log[i]);
+}
+
+static void dpc_process_rp_pio_error(struct dpc_dev *dpc)
+{
+	struct dpc_rp_pio_regs rp_pio_regs;
+
+	dpc_rp_pio_get_info(dpc, &rp_pio_regs);
+	dpc_rp_pio_print_error(dpc, &rp_pio_regs);
+
+	dpc->rp_pio_status = rp_pio_regs.status;
+}
+
 static irqreturn_t dpc_irq(int irq, void *context)
 {
 	struct dpc_dev *dpc = (struct dpc_dev *)context;
@@ -109,6 +267,10 @@ static irqreturn_t dpc_irq(int irq, void *context)
 			 (ext_reason == 0) ? "RP PIO error" :
 			 (ext_reason == 1) ? "software trigger" :
 					     "reserved error");
+		/* show RP PIO error detail information */
+		if (reason == 3 && ext_reason == 0)
+			dpc_process_rp_pio_error(dpc);
+
 		schedule_work(&dpc->work);
 	}
 	return IRQ_HANDLED;
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index c22d3eb..1ce9627 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -967,6 +967,7 @@
 #define  PCI_EXP_DPC_CAP_RP_EXT		0x20	/* Root Port Extensions for DPC */
 #define  PCI_EXP_DPC_CAP_POISONED_TLP	0x40	/* Poisoned TLP Egress Blocking Supported */
 #define  PCI_EXP_DPC_CAP_SW_TRIGGER	0x80	/* Software Triggering Supported */
+#define  PCI_EXP_DPC_RP_PIO_LOG_SIZE	0xF00	/* RP PIO log size */
 #define  PCI_EXP_DPC_CAP_DL_ACTIVE	0x1000	/* ERR_COR signal on DL_Active supported */
 
 #define PCI_EXP_DPC_CTL			6	/* DPC control */
@@ -980,6 +981,15 @@
 
 #define PCI_EXP_DPC_SOURCE_ID		10	/* DPC Source Identifier */
 
+#define PCI_EXP_DPC_RP_PIO_STATUS	 0x0C	/* RP PIO Status */
+#define PCI_EXP_DPC_RP_PIO_MASK		 0x10	/* RP PIO MASK */
+#define PCI_EXP_DPC_RP_PIO_SEVERITY	 0x14	/* RP PIO Severity */
+#define PCI_EXP_DPC_RP_PIO_SYSERROR	 0x18	/* RP PIO SysError */
+#define PCI_EXP_DPC_RP_PIO_EXCEPTION	 0x1C	/* RP PIO Exception */
+#define PCI_EXP_DPC_RP_PIO_HEADER_LOG	 0x20	/* RP PIO Header Log */
+#define PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG	 0x30	/* RP PIO ImpSpec Log */
+#define PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG 0x34	/* RP PIO TLP Prefix Log */
+
 /* Precision Time Measurement */
 #define PCI_PTM_CAP			0x04	    /* PTM Capability */
 #define  PCI_PTM_CAP_REQ		0x00000001  /* Requester capable */
-- 
1.9.1

[PATCH V5 2/2] PCI/DPC: Add local struct device

[Dongdong Liu]

Use a local "struct device *dev" for brevity and consistency in DPC driver.
No functional change intended.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/pci/pcie/pcie-dpc.c | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/drivers/pci/pcie/pcie-dpc.c b/drivers/pci/pcie/pcie-dpc.c
index 8428d74..2d976a6 100644
--- a/drivers/pci/pcie/pcie-dpc.c
+++ b/drivers/pci/pcie/pcie-dpc.c
@@ -71,6 +71,7 @@ static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
 {
 	unsigned long timeout = jiffies + HZ;
 	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 status;
 
 	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
@@ -80,15 +81,17 @@ static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
 		pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
 	}
 	if (status & PCI_EXP_DPC_RP_BUSY) {
-		dev_warn(&pdev->dev, "DPC root port still busy\n");
+		dev_warn(dev, "DPC root port still busy\n");
 		return -EBUSY;
 	}
 	return 0;
 }
 
-static void dpc_wait_link_inactive(struct pci_dev *pdev)
+static void dpc_wait_link_inactive(struct dpc_dev *dpc)
 {
 	unsigned long timeout = jiffies + HZ;
+	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 lnk_status;
 
 	pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
@@ -98,7 +101,7 @@ static void dpc_wait_link_inactive(struct pci_dev *pdev)
 		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
 	}
 	if (lnk_status & PCI_EXP_LNKSTA_DLLLA)
-		dev_warn(&pdev->dev, "Link state not disabled for DPC event\n");
+		dev_warn(dev, "Link state not disabled for DPC event\n");
 }
 
 static void interrupt_event_handler(struct work_struct *work)
@@ -120,7 +123,7 @@ static void interrupt_event_handler(struct work_struct *work)
 	}
 	pci_unlock_rescan_remove();
 
-	dpc_wait_link_inactive(pdev);
+	dpc_wait_link_inactive(dpc);
 	if (dpc->rp && dpc_wait_rp_inactive(dpc))
 		return;
 	if (dpc->rp && dpc->rp_pio_status) {
@@ -245,6 +248,7 @@ static irqreturn_t dpc_irq(int irq, void *context)
 {
 	struct dpc_dev *dpc = (struct dpc_dev *)context;
 	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 status, source;
 
 	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
@@ -253,14 +257,14 @@ static irqreturn_t dpc_irq(int irq, void *context)
 	if (!status || status == (u16)(~0))
 		return IRQ_NONE;
 
-	dev_info(&dpc->dev->device, "DPC containment event, status:%#06x source:%#06x\n",
+	dev_info(dev, "DPC containment event, status:%#06x source:%#06x\n",
 		status, source);
 
 	if (status & PCI_EXP_DPC_STATUS_TRIGGER) {
 		u16 reason = (status >> 1) & 0x3;
 		u16 ext_reason = (status >> 5) & 0x3;
 
-		dev_warn(&dpc->dev->device, "DPC %s detected, remove downstream devices\n",
+		dev_warn(dev, "DPC %s detected, remove downstream devices\n",
 			 (reason == 0) ? "unmasked uncorrectable error" :
 			 (reason == 1) ? "ERR_NONFATAL" :
 			 (reason == 2) ? "ERR_FATAL" :
@@ -281,10 +285,11 @@ static int dpc_probe(struct pcie_device *dev)
 {
 	struct dpc_dev *dpc;
 	struct pci_dev *pdev = dev->port;
+	struct device *device = &dev->device;
 	int status;
 	u16 ctl, cap;
 
-	dpc = devm_kzalloc(&dev->device, sizeof(*dpc), GFP_KERNEL);
+	dpc = devm_kzalloc(device, sizeof(*dpc), GFP_KERNEL);
 	if (!dpc)
 		return -ENOMEM;
 
@@ -293,10 +298,10 @@ static int dpc_probe(struct pcie_device *dev)
 	INIT_WORK(&dpc->work, interrupt_event_handler);
 	set_service_data(dev, dpc);
 
-	status = devm_request_irq(&dev->device, dev->irq, dpc_irq, IRQF_SHARED,
+	status = devm_request_irq(device, dev->irq, dpc_irq, IRQF_SHARED,
 				  "pcie-dpc", dpc);
 	if (status) {
-		dev_warn(&dev->device, "request IRQ%d failed: %d\n", dev->irq,
+		dev_warn(device, "request IRQ%d failed: %d\n", dev->irq,
 			 status);
 		return status;
 	}
@@ -309,7 +314,7 @@ static int dpc_probe(struct pcie_device *dev)
 	ctl = (ctl & 0xfff4) | PCI_EXP_DPC_CTL_EN_NONFATAL | PCI_EXP_DPC_CTL_INT_EN;
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
 
-	dev_info(&dev->device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
+	dev_info(device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
 		cap & 0xf, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
 		FLAG(cap, PCI_EXP_DPC_CAP_POISONED_TLP),
 		FLAG(cap, PCI_EXP_DPC_CAP_SW_TRIGGER), (cap >> 8) & 0xf,
-- 
1.9.1

Re: [PATCH V5 1/2] PCI/DPC: Add eDPC support

[Keith Busch]

On Sat, Aug 19, 2017 at 02:07:20AM -0700, Dongdong Liu wrote:
> This code is to add eDPC support. Get and print the RP PIO error
> information when the trigger condition is RP PIO error.
> 
> For more information on eDPC, please see PCI Express Base Specification
> Revision 3.1, section 6.2.10.3, or view the PCI-SIG eDPC ECN here:
> https://pcisig.com/sites/default/files/specification_documents/ECN_Enhanced_DPC_2012-11-19_final.pdf
> 
> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>

This looks good to me.

Reviewed-by: Keith Busch <keith.busch@intel.com>

Re: [PATCH V5 2/2] PCI/DPC: Add local struct device

[Keith Busch]

On Sat, Aug 19, 2017 at 05:07:21PM +0800, Dongdong Liu wrote:
> Use a local "struct device *dev" for brevity and consistency in DPC driver.
> No functional change intended.
> 
> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>

Looks good to me.

Reviewed-by: Keith Busch <keith.busch@intel.com>

Re: [PATCH V5 0/2] PCI/DPC: Add eDPC support

[Bjorn Helgaas]

On Sat, Aug 19, 2017 at 05:07:19PM +0800, Dongdong Liu wrote:
> This patchset is to add eDPC support and use a local
> "struct device *dev" for brevity and consistency in DPC driver.
> 
> v4->v5:
> - Fix some compile warnings.
> 
> v3->v4:
> - Use pcie_device's device to keep consistency.
> - Adjust patch sequence.
>  
> v2->v3: 
> - Add a separate patch to use a local "struct device *dev" 
>   for brevity and consistency in  DPC driver.
> - Fix comments on PATCH V2.
> - Rebase on v4.13-rc5.
> 
> v1->v2: 
> - Use a stack local variable instead of the allocated memory for
>   collecting RP PIO information.
> - Fix the condition of RP PIO error.
> - Rebase on v4.13-rc1.
> 
> Dongdong Liu (2):
>   PCI/DPC: Add eDPC support
>   PCI/DPC: Add local struct device
> 
>  drivers/pci/pcie/pcie-dpc.c   | 187 +++++++++++++++++++++++++++++++++++++++---
>  include/uapi/linux/pci_regs.h |  10 +++
>  2 files changed, 187 insertions(+), 10 deletions(-)

Applied with Keith's reviewed-by to pci/dpc for v4.14, thanks!

Re: [PATCH V5 0/2] PCI/DPC: Add eDPC support

[Dongdong Liu]

e\x1c( 2017/8/25 0:32, Bjorn Helgaas e\x06\x19i\x01\x13:
> On Sat, Aug 19, 2017 at 05:07:19PM +0800, Dongdong Liu wrote:
>> This patchset is to add eDPC support and use a local
>> "struct device *dev" for brevity and consistency in DPC driver.
>>
>> v4->v5:
>> - Fix some compile warnings.
>>
>> v3->v4:
>> - Use pcie_device's device to keep consistency.
>> - Adjust patch sequence.
>>
>> v2->v3:
>> - Add a separate patch to use a local "struct device *dev"
>>   for brevity and consistency in  DPC driver.
>> - Fix comments on PATCH V2.
>> - Rebase on v4.13-rc5.
>>
>> v1->v2:
>> - Use a stack local variable instead of the allocated memory for
>>   collecting RP PIO information.
>> - Fix the condition of RP PIO error.
>> - Rebase on v4.13-rc1.
>>
>> Dongdong Liu (2):
>>   PCI/DPC: Add eDPC support
>>   PCI/DPC: Add local struct device
>>
>>  drivers/pci/pcie/pcie-dpc.c   | 187 +++++++++++++++++++++++++++++++++++++++---
>>  include/uapi/linux/pci_regs.h |  10 +++
>>  2 files changed, 187 insertions(+), 10 deletions(-)
>
> Applied with Keith's reviewed-by to pci/dpc for v4.14, thanks!

Thanks Bjorn and Keith.

>
> .
>