[PATCH v1 0/5] powerpc/pci: PCI slot unified reset

[PATCH v1 0/5] powerpc/pci: PCI slot unified reset

[Gavin Shan]

The patchset corresponds to skiboot changes, which manages PCI slots
in a unified way: OPAL APIs used to do slot reset, power management,
presence status retrival. The patchset shouldn't be merged before
the OPAL firmware counterpart is merged.

The kernel changes have been split into 2 parts: (A) Use the unified
PCI slot reset OPAL API - this patchset; (B) Changes to rpaphp.ko to
support PCI slot hotplug via sysfs entries for PowerPC PowerNV platform.

This patchset affects EEH, PCI reset logics, which is used mainly by
VFIO. A couple of cases have been tested out on P7/P8 boxes and looks
good: EEH on PowerNV, EEH support for guest, PCI passthrou.

Gavin Shan (5):
  powerpc/powernv: Use PCI slot reset infrastructure
  powerpc/powernv: Refactor ioda_eeh_reset()
  powerpc/powernv: Avoid do complete reset for twice
  powerpc/powernv: Issue fundamental reset if required
  powerpc/powernv: Drop opal_pci_reinit()

 arch/powerpc/include/asm/eeh.h                 |   2 +-
 arch/powerpc/include/asm/opal.h                |  31 +---
 arch/powerpc/kernel/eeh_pe.c                   |   3 -
 arch/powerpc/platforms/powernv/eeh-ioda.c      | 230 ++++++++++++++-----------
 arch/powerpc/platforms/powernv/eeh-powernv.c   |  24 +--
 arch/powerpc/platforms/powernv/opal-wrappers.S |   1 -
 arch/powerpc/platforms/powernv/pci-ioda.c      |   5 +-
 arch/powerpc/platforms/pseries/eeh_pseries.c   |   3 +-
 8 files changed, 139 insertions(+), 160 deletions(-)

Thanks,
Gavin

[PATCH 1/5] powerpc/powernv: Use PCI slot reset infrastructure

[Gavin Shan]

The patch reworks the reset logic to reuse PCI slot reset
infrastructure implemented in OPAL firmware. For one particular
PCI slot, we have switch to hot reset in case the firmware can't
support reset for it.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/include/asm/eeh.h            |   1 +
 arch/powerpc/include/asm/opal.h           |  25 ++---
 arch/powerpc/platforms/powernv/eeh-ioda.c | 149 ++++++++++++++++--------------
 arch/powerpc/platforms/powernv/pci-ioda.c |   5 +-
 4 files changed, 89 insertions(+), 91 deletions(-)

diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 9c11d1e..5847721 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -192,6 +192,7 @@ enum {
 #define EEH_RESET_DEACTIVATE	0	/* Deactivate the PE reset	*/
 #define EEH_RESET_HOT		1	/* Hot reset			*/
 #define EEH_RESET_FUNDAMENTAL	3	/* Fundamental reset		*/
+#define EEH_RESET_COMPLETE	4	/* PHB complete reset		*/
 #define EEH_LOG_TEMP		1	/* EEH temporary error log	*/
 #define EEH_LOG_PERM		2	/* EEH permanent error log	*/
 
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index 9124b0e..0b8b4b1 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -388,28 +388,17 @@ enum OpalM64EnableAction {
 };
 
 enum OpalPciResetScope {
-	OPAL_RESET_PHB_COMPLETE		= 1,
-	OPAL_RESET_PCI_LINK		= 2,
-	OPAL_RESET_PHB_ERROR		= 3,
-	OPAL_RESET_PCI_HOT		= 4,
-	OPAL_RESET_PCI_FUNDAMENTAL	= 5,
-	OPAL_RESET_PCI_IODA_TABLE	= 6
+	OPAL_RESET_PCI_HOT		= 0,
+	OPAL_RESET_PCI_FUNDAMENTAL	= 1,
+	OPAL_RESET_PHB_COMPLETE		= 2,
+	OPAL_RESET_PCI_IODA_TABLE	= 3,
+	OPAL_RESET_PHB_ERROR		= 4
 };
 
 enum OpalPciReinitScope {
 	OPAL_REINIT_PCI_DEV = 1000
 };
 
-enum OpalPciResetState {
-	OPAL_DEASSERT_RESET = 0,
-	OPAL_ASSERT_RESET = 1
-};
-
-enum OpalPciMaskAction {
-	OPAL_UNMASK_ERROR_TYPE = 0,
-	OPAL_MASK_ERROR_TYPE = 1
-};
-
 enum OpalSlotLedType {
 	OPAL_SLOT_LED_ID_TYPE = 0,
 	OPAL_SLOT_LED_FAULT_TYPE = 1
@@ -906,7 +895,7 @@ int64_t opal_pci_map_pe_dma_window(uint64_t phb_id, uint16_t pe_number, uint16_t
 int64_t opal_pci_map_pe_dma_window_real(uint64_t phb_id, uint16_t pe_number,
 					uint16_t dma_window_number, uint64_t pci_start_addr,
 					uint64_t pci_mem_size);
-int64_t opal_pci_reset(uint64_t phb_id, uint8_t reset_scope, uint8_t assert_state);
+int64_t opal_pci_reset(uint64_t id, uint8_t reset_scope);
 
 int64_t opal_pci_get_hub_diag_data(uint64_t hub_id, void *diag_buffer,
 				   uint64_t diag_buffer_len);
@@ -922,7 +911,7 @@ int64_t opal_get_epow_status(__be64 *status);
 int64_t opal_set_system_attention_led(uint8_t led_action);
 int64_t opal_pci_next_error(uint64_t phb_id, __be64 *first_frozen_pe,
 			    __be16 *pci_error_type, __be16 *severity);
-int64_t opal_pci_poll(uint64_t phb_id);
+int64_t opal_pci_poll(uint64_t id);
 int64_t opal_return_cpu(void);
 int64_t opal_check_token(uint64_t token);
 int64_t opal_reinit_cpus(uint64_t flags);
diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
index 43aba2d..e5fa4ff 100644
--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
@@ -490,12 +490,12 @@ static int ioda_eeh_get_state(struct eeh_pe *pe)
 	return ioda_eeh_get_pe_state(pe);
 }
 
-static s64 ioda_eeh_phb_poll(struct pnv_phb *phb)
+static s64 ioda_eeh_phb_poll(uint64_t id)
 {
 	s64 rc = OPAL_HARDWARE;
 
 	while (1) {
-		rc = opal_pci_poll(phb->opal_id);
+		rc = opal_pci_poll(id);
 		if (rc <= 0)
 			break;
 
@@ -511,23 +511,29 @@ static s64 ioda_eeh_phb_poll(struct pnv_phb *phb)
 int ioda_eeh_phb_reset(struct pci_controller *hose, int option)
 {
 	struct pnv_phb *phb = hose->private_data;
+	uint8_t reset_scope;
 	s64 rc = OPAL_HARDWARE;
 
 	pr_debug("%s: Reset PHB#%x, option=%d\n",
 		 __func__, hose->global_number, option);
 
-	/* Issue PHB complete reset request */
-	if (option == EEH_RESET_FUNDAMENTAL ||
-	    option == EEH_RESET_HOT)
-		rc = opal_pci_reset(phb->opal_id,
-				OPAL_RESET_PHB_COMPLETE,
-				OPAL_ASSERT_RESET);
-	else if (option == EEH_RESET_DEACTIVATE)
-		rc = opal_pci_reset(phb->opal_id,
-				OPAL_RESET_PHB_COMPLETE,
-				OPAL_DEASSERT_RESET);
-	if (rc < 0)
-		goto out;
+	switch (option) {
+	case EEH_RESET_HOT:
+		reset_scope = OPAL_RESET_PCI_HOT;
+		break;
+	case EEH_RESET_FUNDAMENTAL:
+		reset_scope = OPAL_RESET_PCI_FUNDAMENTAL;
+		break;
+	case EEH_RESET_COMPLETE:
+		reset_scope = OPAL_RESET_PHB_COMPLETE;
+		break;
+	case EEH_RESET_DEACTIVATE:
+		return 0;
+	default:
+		pr_warn("%s: Unsupported option %d\n",
+			__func__, option);
+		return -EINVAL;
+	}
 
 	/*
 	 * Poll state of the PHB until the request is done
@@ -535,61 +541,17 @@ int ioda_eeh_phb_reset(struct pci_controller *hose, int option)
 	 * reset followed by hot reset on root bus. So we also
 	 * need the PCI bus settlement delay.
 	 */
-	rc = ioda_eeh_phb_poll(phb);
-	if (option == EEH_RESET_DEACTIVATE) {
-		if (system_state < SYSTEM_RUNNING)
-			udelay(1000 * EEH_PE_RST_SETTLE_TIME);
-		else
-			msleep(EEH_PE_RST_SETTLE_TIME);
-	}
-out:
-	if (rc != OPAL_SUCCESS)
-		return -EIO;
-
-	return 0;
-}
-
-static int ioda_eeh_root_reset(struct pci_controller *hose, int option)
-{
-	struct pnv_phb *phb = hose->private_data;
-	s64 rc = OPAL_SUCCESS;
+	rc = opal_pci_reset(phb->opal_id, reset_scope);
+	if (rc > 0)
+		rc = ioda_eeh_phb_poll(phb->opal_id);
 
-	pr_debug("%s: Reset PHB#%x, option=%d\n",
-		 __func__, hose->global_number, option);
-
-	/*
-	 * During the reset deassert time, we needn't care
-	 * the reset scope because the firmware does nothing
-	 * for fundamental or hot reset during deassert phase.
-	 */
-	if (option == EEH_RESET_FUNDAMENTAL)
-		rc = opal_pci_reset(phb->opal_id,
-				OPAL_RESET_PCI_FUNDAMENTAL,
-				OPAL_ASSERT_RESET);
-	else if (option == EEH_RESET_HOT)
-		rc = opal_pci_reset(phb->opal_id,
-				OPAL_RESET_PCI_HOT,
-				OPAL_ASSERT_RESET);
-	else if (option == EEH_RESET_DEACTIVATE)
-		rc = opal_pci_reset(phb->opal_id,
-				OPAL_RESET_PCI_HOT,
-				OPAL_DEASSERT_RESET);
-	if (rc < 0)
-		goto out;
-
-	/* Poll state of the PHB until the request is done */
-	rc = ioda_eeh_phb_poll(phb);
-	if (option == EEH_RESET_DEACTIVATE)
-		msleep(EEH_PE_RST_SETTLE_TIME);
-out:
 	if (rc != OPAL_SUCCESS)
 		return -EIO;
 
 	return 0;
 }
 
-static int ioda_eeh_bridge_reset(struct pci_dev *dev, int option)
-
+static int __ioda_eeh_bridge_reset(struct pci_dev *dev, int option)
 {
 	struct device_node *dn = pci_device_to_OF_node(dev);
 	struct eeh_dev *edev = of_node_to_eeh_dev(dn);
@@ -639,14 +601,63 @@ static int ioda_eeh_bridge_reset(struct pci_dev *dev, int option)
 	return 0;
 }
 
+static int ioda_eeh_bridge_reset(struct pci_dev *dev, int option)
+{
+	struct pci_controller *hose;
+	struct pnv_phb *phb;
+	struct device_node *dn = dev ? pci_device_to_OF_node(dev) : NULL;
+	const char *label = NULL;
+	uint64_t id = 0x1000000000000000;
+	uint8_t reset_scope;
+	s64 rc;
+
+	/*
+	 * If the firmware can't handle it, we still need simulate a hot
+	 * reset on the secondary bus. It should be the rare case.
+	 */
+	if (dn)
+		label = of_get_property(dn, "ibm,slot-label", NULL);
+	if (!label)
+		return __ioda_eeh_bridge_reset(dev, option);
+
+	/* The firmware can handle the request */
+	switch (option) {
+	case EEH_RESET_HOT:
+		reset_scope = OPAL_RESET_PCI_HOT;
+		break;
+	case EEH_RESET_FUNDAMENTAL:
+		reset_scope = OPAL_RESET_PCI_FUNDAMENTAL;
+		break;
+	case EEH_RESET_DEACTIVATE:
+		return 0;
+	case EEH_RESET_COMPLETE:
+	default:
+		pr_warn("%s: Unsupported option %d\n",
+			__func__, option);
+		return -EINVAL;
+	}
+
+	hose = pci_bus_to_host(dev->bus);
+	phb = hose->private_data;
+	id |= ((phb->opal_id << 16) | (dev->bus->number << 8) | (dev->devfn));
+	rc = opal_pci_reset(id, reset_scope);
+	if (rc > 0)
+		rc = ioda_eeh_phb_poll(id);
+
+	if (rc != OPAL_SUCCESS)
+		return -EIO;
+
+	return 0;
+}
+
 void pnv_pci_reset_secondary_bus(struct pci_dev *dev)
 {
 	struct pci_controller *hose;
 
 	if (pci_is_root_bus(dev->bus)) {
 		hose = pci_bus_to_host(dev->bus);
-		ioda_eeh_root_reset(hose, EEH_RESET_HOT);
-		ioda_eeh_root_reset(hose, EEH_RESET_DEACTIVATE);
+		ioda_eeh_phb_reset(hose, EEH_RESET_HOT);
+		ioda_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
 	} else {
 		ioda_eeh_bridge_reset(dev, EEH_RESET_HOT);
 		ioda_eeh_bridge_reset(dev, EEH_RESET_DEACTIVATE);
@@ -686,7 +697,7 @@ static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 	 * state explicitly after BAR restore.
 	 */
 	if (pe->type & EEH_PE_PHB) {
-		ret = ioda_eeh_phb_reset(hose, option);
+		ret = ioda_eeh_phb_reset(hose, EEH_RESET_COMPLETE);
 	} else {
 		struct pnv_phb *phb;
 		s64 rc;
@@ -703,8 +714,7 @@ static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 		    (option == EEH_RESET_HOT ||
 		    option == EEH_RESET_FUNDAMENTAL)) {
 			rc = opal_pci_reset(phb->opal_id,
-					    OPAL_RESET_PHB_ERROR,
-					    OPAL_ASSERT_RESET);
+					    OPAL_RESET_PHB_ERROR);
 			if (rc != OPAL_SUCCESS) {
 				pr_warn("%s: Failure %lld clearing "
 					"error injection registers\n",
@@ -714,9 +724,8 @@ static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 		}
 
 		bus = eeh_pe_bus_get(pe);
-		if (pci_is_root_bus(bus) ||
-		    pci_is_root_bus(bus->parent))
-			ret = ioda_eeh_root_reset(hose, option);
+		if (pci_is_root_bus(bus))
+			ret = ioda_eeh_phb_reset(hose, option);
 		else
 			ret = ioda_eeh_bridge_reset(bus->self, option);
 	}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index ba1b4cb..5b0fcea 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1913,8 +1913,7 @@ static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus,
 
 static void pnv_pci_ioda_shutdown(struct pnv_phb *phb)
 {
-	opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
-		       OPAL_ASSERT_RESET);
+	opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE);
 }
 
 static void __init pnv_pci_init_ioda_phb(struct device_node *np,
@@ -2090,7 +2089,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np,
 	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
 
 	/* Reset IODA tables to a clean state */
-	rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
+	rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE);
 	if (rc)
 		pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
 
-- 
1.8.3.2

[PATCH 2/5] powerpc/powernv: Refactor ioda_eeh_reset()

[Gavin Shan]

The patch refactors ioda_eeh_reset() to avoid unnecessary nested
if statements, so that the code looks simplified to earn a bit
more readibility, no logic changed.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/eeh-ioda.c | 58 ++++++++++++++-----------------
 1 file changed, 26 insertions(+), 32 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
index e5fa4ff..3b82384 100644
--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
@@ -679,8 +679,9 @@ void pnv_pci_reset_secondary_bus(struct pci_dev *dev)
 static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 {
 	struct pci_controller *hose = pe->phb;
+	struct pnv_phb *phb;
 	struct pci_bus *bus;
-	int ret;
+	s64 rc;
 
 	/*
 	 * For PHB reset, we always have complete reset. For those PEs whose
@@ -696,41 +697,34 @@ static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 	 * reset. The side effect is that EEH core has to clear the frozen
 	 * state explicitly after BAR restore.
 	 */
-	if (pe->type & EEH_PE_PHB) {
-		ret = ioda_eeh_phb_reset(hose, EEH_RESET_COMPLETE);
-	} else {
-		struct pnv_phb *phb;
-		s64 rc;
+	if (pe->type & EEH_PE_PHB)
+		return ioda_eeh_phb_reset(hose, EEH_RESET_COMPLETE);
 
-		/*
-		 * The frozen PE might be caused by PAPR error injection
-		 * registers, which are expected to be cleared after hitting
-		 * frozen PE as stated in the hardware spec. Unfortunately,
-		 * that's not true on P7IOC. So we have to clear it manually
-		 * to avoid recursive EEH errors during recovery.
-		 */
-		phb = hose->private_data;
-		if (phb->model == PNV_PHB_MODEL_P7IOC &&
-		    (option == EEH_RESET_HOT ||
-		    option == EEH_RESET_FUNDAMENTAL)) {
-			rc = opal_pci_reset(phb->opal_id,
-					    OPAL_RESET_PHB_ERROR);
-			if (rc != OPAL_SUCCESS) {
-				pr_warn("%s: Failure %lld clearing "
-					"error injection registers\n",
-					__func__, rc);
-				return -EIO;
-			}
+	/*
+	 * The frozen PE might be caused by PAPR error injection
+	 * registers, which are expected to be cleared after hitting
+	 * frozen PE as stated in the hardware spec. Unfortunately,
+	 * that's not true on P7IOC. So we have to clear it manually
+	 * to avoid recursive EEH errors during recovery.
+	 */
+	phb = hose->private_data;
+	if (phb->model == PNV_PHB_MODEL_P7IOC &&
+	    (option == EEH_RESET_HOT ||
+	     option == EEH_RESET_FUNDAMENTAL)) {
+		rc = opal_pci_reset(phb->opal_id,
+				    OPAL_RESET_PHB_ERROR);
+		if (rc != OPAL_SUCCESS) {
+			pr_warn("%s: Error %lld clearing error injection\n",
+				__func__, rc);
+			return -EIO;
 		}
-
-		bus = eeh_pe_bus_get(pe);
-		if (pci_is_root_bus(bus))
-			ret = ioda_eeh_phb_reset(hose, option);
-		else
-			ret = ioda_eeh_bridge_reset(bus->self, option);
 	}
 
-	return ret;
+	bus = eeh_pe_bus_get(pe);
+	if (pci_is_root_bus(bus))
+		return ioda_eeh_phb_reset(hose, option);
+
+	return ioda_eeh_bridge_reset(bus->self, option);
 }
 
 /**
-- 
1.8.3.2

[PATCH 3/5] powerpc/powernv: Avoid do complete reset for twice

[Gavin Shan]

For PHB complete reset case, EEH_RESET_DEACTIVATE followed by
EEH_RESET_{HOT,FUNDAMENTAL} requests are issued by EEH core.
All of them are translated to raising complete reset wrongly
and we did complete reset for twice unnecessarily for each
transaction.

The patch fixes above issue by translating the request properly.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/eeh-ioda.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
index 3b82384..d02a696 100644
--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
@@ -697,8 +697,13 @@ static int ioda_eeh_reset(struct eeh_pe *pe, int option)
 	 * reset. The side effect is that EEH core has to clear the frozen
 	 * state explicitly after BAR restore.
 	 */
-	if (pe->type & EEH_PE_PHB)
-		return ioda_eeh_phb_reset(hose, EEH_RESET_COMPLETE);
+	if (pe->type & EEH_PE_PHB) {
+		if (option == EEH_RESET_HOT ||
+		    option == EEH_RESET_FUNDAMENTAL)
+			return ioda_eeh_phb_reset(hose, EEH_RESET_COMPLETE);
+
+		return ioda_eeh_phb_reset(hose, option);
+	}
 
 	/*
 	 * The frozen PE might be caused by PAPR error injection
-- 
1.8.3.2

[PATCH 4/5] powerpc/powernv: Issue fundamental reset if required

[Gavin Shan]

pnv_pci_reset_secondary_bus() is the backend for resetting the
secondary bus for the specified PCI bridge. We always issue hot
reset, which isn't enough for some devices who requires fundamental
reset explicitly. The patch switches to fundamental reset if the
devices ask for that.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/platforms/powernv/eeh-ioda.c | 34 +++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
index d02a696..4bf2fb5 100644
--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
@@ -650,18 +650,34 @@ static int ioda_eeh_bridge_reset(struct pci_dev *dev, int option)
 	return 0;
 }
 
+static int pnv_pci_dev_reset_type(struct pci_dev *pdev, void *data)
+{
+	int *freset = data;
+
+	/*
+	 * Stop the iteration immediately if any one
+	 * PCI device requires fundamental reset
+	 */
+	*freset |= pdev->needs_freset;
+	return *freset;
+}
+
+/*
+ * When the function is called, there should have
+ * valid bridge. We don't need cover root bus here
+ */
 void pnv_pci_reset_secondary_bus(struct pci_dev *dev)
 {
-	struct pci_controller *hose;
+	int option = EEH_RESET_HOT;
+	int freset = 0;
 
-	if (pci_is_root_bus(dev->bus)) {
-		hose = pci_bus_to_host(dev->bus);
-		ioda_eeh_phb_reset(hose, EEH_RESET_HOT);
-		ioda_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
-	} else {
-		ioda_eeh_bridge_reset(dev, EEH_RESET_HOT);
-		ioda_eeh_bridge_reset(dev, EEH_RESET_DEACTIVATE);
-	}
+	/* Need fundamental reset ? */
+	pci_walk_bus(dev->subordinate, pnv_pci_dev_reset_type, &freset);
+	if (freset)
+		option = EEH_RESET_FUNDAMENTAL;
+
+	ioda_eeh_bridge_reset(dev, option);
+	ioda_eeh_bridge_reset(dev, EEH_RESET_DEACTIVATE);
 }
 
 /**
-- 
1.8.3.2

[PATCH 5/5] powerpc/powernv: Drop opal_pci_reinit()

[Gavin Shan]

With unified PCI slot reset infrastructure supported by the firmware,
we needn't reinitialize the affected devices after PE reset. OPAL API
opal_pci_reinit() and related logic, including EEH restore_config()
callback can be killed safely. All the work covered by opal_pci_reinit()
should be done in firmware insteadly.

Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
---
 arch/powerpc/include/asm/eeh.h                 |  1 -
 arch/powerpc/include/asm/opal.h                |  6 ------
 arch/powerpc/kernel/eeh_pe.c                   |  3 ---
 arch/powerpc/platforms/powernv/eeh-powernv.c   | 24 +-----------------------
 arch/powerpc/platforms/powernv/opal-wrappers.S |  1 -
 arch/powerpc/platforms/pseries/eeh_pseries.c   |  3 +--
 6 files changed, 2 insertions(+), 36 deletions(-)

diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 5847721..687fc5c 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -214,7 +214,6 @@ struct eeh_ops {
 	int (*read_config)(struct device_node *dn, int where, int size, u32 *val);
 	int (*write_config)(struct device_node *dn, int where, int size, u32 val);
 	int (*next_error)(struct eeh_pe **pe);
-	int (*restore_config)(struct device_node *dn);
 };
 
 extern int eeh_subsystem_flags;
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index 0b8b4b1..b59811f 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -108,7 +108,6 @@ struct opal_sg_list {
 #define OPAL_PCI_GET_HUB_DIAG_DATA		50
 #define OPAL_PCI_GET_PHB_DIAG_DATA		51
 #define OPAL_PCI_FENCE_PHB			52
-#define OPAL_PCI_REINIT				53
 #define OPAL_PCI_MASK_PE_ERROR			54
 #define OPAL_SET_SLOT_LED_STATUS		55
 #define OPAL_GET_EPOW_STATUS			56
@@ -395,10 +394,6 @@ enum OpalPciResetScope {
 	OPAL_RESET_PHB_ERROR		= 4
 };
 
-enum OpalPciReinitScope {
-	OPAL_REINIT_PCI_DEV = 1000
-};
-
 enum OpalSlotLedType {
 	OPAL_SLOT_LED_ID_TYPE = 0,
 	OPAL_SLOT_LED_FAULT_TYPE = 1
@@ -904,7 +899,6 @@ int64_t opal_pci_get_phb_diag_data(uint64_t phb_id, void *diag_buffer,
 int64_t opal_pci_get_phb_diag_data2(uint64_t phb_id, void *diag_buffer,
 				    uint64_t diag_buffer_len);
 int64_t opal_pci_fence_phb(uint64_t phb_id);
-int64_t opal_pci_reinit(uint64_t phb_id, uint64_t reinit_scope, uint64_t data);
 int64_t opal_pci_mask_pe_error(uint64_t phb_id, uint16_t pe_number, uint8_t error_type, uint8_t mask_action);
 int64_t opal_set_slot_led_status(uint64_t phb_id, uint64_t slot_id, uint8_t led_type, uint8_t led_action);
 int64_t opal_get_epow_status(__be64 *status);
diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c
index fa950fb..98a1f12 100644
--- a/arch/powerpc/kernel/eeh_pe.c
+++ b/arch/powerpc/kernel/eeh_pe.c
@@ -814,9 +814,6 @@ static void *eeh_restore_one_device_bars(void *data, void *flag)
 	else
 		eeh_restore_device_bars(edev, dn);
 
-	if (eeh_ops->restore_config)
-		eeh_ops->restore_config(dn);
-
 	return NULL;
 }
 
diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c
index e261869..d57ff59c 100644
--- a/arch/powerpc/platforms/powernv/eeh-powernv.c
+++ b/arch/powerpc/platforms/powernv/eeh-powernv.c
@@ -469,27 +469,6 @@ static int powernv_eeh_next_error(struct eeh_pe **pe)
 	return -EEXIST;
 }
 
-static int powernv_eeh_restore_config(struct device_node *dn)
-{
-	struct eeh_dev *edev = of_node_to_eeh_dev(dn);
-	struct pnv_phb *phb;
-	s64 ret;
-
-	if (!edev)
-		return -EEXIST;
-
-	phb = edev->phb->private_data;
-	ret = opal_pci_reinit(phb->opal_id,
-			      OPAL_REINIT_PCI_DEV, edev->config_addr);
-	if (ret) {
-		pr_warn("%s: Can't reinit PCI dev 0x%x (%lld)\n",
-			__func__, edev->config_addr, ret);
-		return -EIO;
-	}
-
-	return 0;
-}
-
 static struct eeh_ops powernv_eeh_ops = {
 	.name                   = "powernv",
 	.init                   = powernv_eeh_init,
@@ -506,8 +485,7 @@ static struct eeh_ops powernv_eeh_ops = {
 	.err_inject		= powernv_eeh_err_inject,
 	.read_config            = powernv_eeh_read_config,
 	.write_config           = powernv_eeh_write_config,
-	.next_error		= powernv_eeh_next_error,
-	.restore_config		= powernv_eeh_restore_config
+	.next_error		= powernv_eeh_next_error
 };
 
 /**
diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S
index feb549a..b6474a1 100644
--- a/arch/powerpc/platforms/powernv/opal-wrappers.S
+++ b/arch/powerpc/platforms/powernv/opal-wrappers.S
@@ -209,7 +209,6 @@ OPAL_CALL(opal_pci_reset,			OPAL_PCI_RESET);
 OPAL_CALL(opal_pci_get_hub_diag_data,		OPAL_PCI_GET_HUB_DIAG_DATA);
 OPAL_CALL(opal_pci_get_phb_diag_data,		OPAL_PCI_GET_PHB_DIAG_DATA);
 OPAL_CALL(opal_pci_fence_phb,			OPAL_PCI_FENCE_PHB);
-OPAL_CALL(opal_pci_reinit,			OPAL_PCI_REINIT);
 OPAL_CALL(opal_pci_mask_pe_error,		OPAL_PCI_MASK_PE_ERROR);
 OPAL_CALL(opal_set_slot_led_status,		OPAL_SET_SLOT_LED_STATUS);
 OPAL_CALL(opal_get_epow_status,			OPAL_GET_EPOW_STATUS);
diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c
index a6c7e19..e69aff6 100644
--- a/arch/powerpc/platforms/pseries/eeh_pseries.c
+++ b/arch/powerpc/platforms/pseries/eeh_pseries.c
@@ -719,8 +719,7 @@ static struct eeh_ops pseries_eeh_ops = {
 	.err_inject		= NULL,
 	.read_config		= pseries_eeh_read_config,
 	.write_config		= pseries_eeh_write_config,
-	.next_error		= NULL,
-	.restore_config		= NULL
+	.next_error		= NULL
 };
 
 /**
-- 
1.8.3.2

Re: [PATCH v1 0/5] powerpc/pci: PCI slot unified reset

[Benjamin Herrenschmidt]

On Tue, 2014-11-25 at 09:38 +1100, Gavin Shan wrote:
> The patchset corresponds to skiboot changes, which manages PCI slots
> in a unified way: OPAL APIs used to do slot reset, power management,
> presence status retrival. The patchset shouldn't be merged before
> the OPAL firmware counterpart is merged.

This doesn't work. We need to have both backward and forward
compatibility.

IE. New kernels must work with old OPAL and vice-versa.

> The kernel changes have been split into 2 parts: (A) Use the unified
> PCI slot reset OPAL API - this patchset; (B) Changes to rpaphp.ko to
> support PCI slot hotplug via sysfs entries for PowerPC PowerNV platform.

I don't think we should be using rpaphp... we should be creating our new
backend, there is almost no commonalities between RPA/PAPR and OPAL.

> This patchset affects EEH, PCI reset logics, which is used mainly by
> VFIO. A couple of cases have been tested out on P7/P8 boxes and looks
> good: EEH on PowerNV, EEH support for guest, PCI passthrou.
> 
> Gavin Shan (5):
>   powerpc/powernv: Use PCI slot reset infrastructure
>   powerpc/powernv: Refactor ioda_eeh_reset()
>   powerpc/powernv: Avoid do complete reset for twice
>   powerpc/powernv: Issue fundamental reset if required
>   powerpc/powernv: Drop opal_pci_reinit()
> 
>  arch/powerpc/include/asm/eeh.h                 |   2 +-
>  arch/powerpc/include/asm/opal.h                |  31 +---
>  arch/powerpc/kernel/eeh_pe.c                   |   3 -
>  arch/powerpc/platforms/powernv/eeh-ioda.c      | 230 ++++++++++++++-----------
>  arch/powerpc/platforms/powernv/eeh-powernv.c   |  24 +--
>  arch/powerpc/platforms/powernv/opal-wrappers.S |   1 -
>  arch/powerpc/platforms/powernv/pci-ioda.c      |   5 +-
>  arch/powerpc/platforms/pseries/eeh_pseries.c   |   3 +-
>  8 files changed, 139 insertions(+), 160 deletions(-)
> 
> Thanks,
> Gavin

Re: [PATCH v1 0/5] powerpc/pci: PCI slot unified reset

[Gavin Shan]

On Wed, Nov 26, 2014 at 09:54:53AM +1100, Benjamin Herrenschmidt wrote:
>On Tue, 2014-11-25 at 09:38 +1100, Gavin Shan wrote:
>> The patchset corresponds to skiboot changes, which manages PCI slots
>> in a unified way: OPAL APIs used to do slot reset, power management,
>> presence status retrival. The patchset shouldn't be merged before
>> the OPAL firmware counterpart is merged.
>
>This doesn't work. We need to have both backward and forward
>compatibility.
>
>IE. New kernels must work with old OPAL and vice-versa.
>

Yes, it's known issue as you remind me yesterday. I'll fix it in next
revision, from skiboot (I guess it would be official name to be used
now) side.

>> The kernel changes have been split into 2 parts: (A) Use the unified
>> PCI slot reset OPAL API - this patchset; (B) Changes to rpaphp.ko to
>> support PCI slot hotplug via sysfs entries for PowerPC PowerNV platform.
>
>I don't think we should be using rpaphp... we should be creating our new
>backend, there is almost no commonalities between RPA/PAPR and OPAL.
>

Ok. I'll create powernv-php as you suggested in another thread. Thank you
for your time on this.

Thanks,
Gavin

>> This patchset affects EEH, PCI reset logics, which is used mainly by
>> VFIO. A couple of cases have been tested out on P7/P8 boxes and looks
>> good: EEH on PowerNV, EEH support for guest, PCI passthrou.
>> 
>> Gavin Shan (5):
>>   powerpc/powernv: Use PCI slot reset infrastructure
>>   powerpc/powernv: Refactor ioda_eeh_reset()
>>   powerpc/powernv: Avoid do complete reset for twice
>>   powerpc/powernv: Issue fundamental reset if required
>>   powerpc/powernv: Drop opal_pci_reinit()
>> 
>>  arch/powerpc/include/asm/eeh.h                 |   2 +-
>>  arch/powerpc/include/asm/opal.h                |  31 +---
>>  arch/powerpc/kernel/eeh_pe.c                   |   3 -
>>  arch/powerpc/platforms/powernv/eeh-ioda.c      | 230 ++++++++++++++-----------
>>  arch/powerpc/platforms/powernv/eeh-powernv.c   |  24 +--
>>  arch/powerpc/platforms/powernv/opal-wrappers.S |   1 -
>>  arch/powerpc/platforms/powernv/pci-ioda.c      |   5 +-
>>  arch/powerpc/platforms/pseries/eeh_pseries.c   |   3 +-
>>  8 files changed, 139 insertions(+), 160 deletions(-)
>> 
>> Thanks,
>> Gavin
>
>

Re: [PATCH v1 0/5] powerpc/pci: PCI slot unified reset

[Gavin Shan]

On Tue, Nov 25, 2014 at 09:38:42AM +1100, Gavin Shan wrote:

Please ignore this revision. v2 is on the way...

Thanks,
Gavin

>The patchset corresponds to skiboot changes, which manages PCI slots
>in a unified way: OPAL APIs used to do slot reset, power management,
>presence status retrival. The patchset shouldn't be merged before
>the OPAL firmware counterpart is merged.
>
>The kernel changes have been split into 2 parts: (A) Use the unified
>PCI slot reset OPAL API - this patchset; (B) Changes to rpaphp.ko to
>support PCI slot hotplug via sysfs entries for PowerPC PowerNV platform.
>
>This patchset affects EEH, PCI reset logics, which is used mainly by
>VFIO. A couple of cases have been tested out on P7/P8 boxes and looks
>good: EEH on PowerNV, EEH support for guest, PCI passthrou.
>
>Gavin Shan (5):
>  powerpc/powernv: Use PCI slot reset infrastructure
>  powerpc/powernv: Refactor ioda_eeh_reset()
>  powerpc/powernv: Avoid do complete reset for twice
>  powerpc/powernv: Issue fundamental reset if required
>  powerpc/powernv: Drop opal_pci_reinit()
>
> arch/powerpc/include/asm/eeh.h                 |   2 +-
> arch/powerpc/include/asm/opal.h                |  31 +---
> arch/powerpc/kernel/eeh_pe.c                   |   3 -
> arch/powerpc/platforms/powernv/eeh-ioda.c      | 230 ++++++++++++++-----------
> arch/powerpc/platforms/powernv/eeh-powernv.c   |  24 +--
> arch/powerpc/platforms/powernv/opal-wrappers.S |   1 -
> arch/powerpc/platforms/powernv/pci-ioda.c      |   5 +-
> arch/powerpc/platforms/pseries/eeh_pseries.c   |   3 +-
> 8 files changed, 139 insertions(+), 160 deletions(-)
>
>Thanks,
>Gavin