[PATCH v3 0/4] PCI: layerscape: Add suspend/resume support for ls1043 and ls1021

[PATCH v3 0/4] PCI: layerscape: Add suspend/resume support for ls1043 and ls1021

[Frank Li]

Add suspend/resume support for ls1043 and ls1021.
Change log see each patch

Frank Li (4):
  PCI: layerscape: Add function pointer for exit_from_l2()
  PCI: layerscape: Add suspend/resume for ls1021a
  PCI: layerscape: Rename pf_* as pf_lut_*
  PCI: layerscape: Add suspend/resume for ls1043a

 drivers/pci/controller/dwc/pci-layerscape.c | 217 ++++++++++++++++++--
 1 file changed, 196 insertions(+), 21 deletions(-)

-- 
2.34.1

[PATCH v3 1/4] PCI: layerscape: Add function pointer for exit_from_l2()

[Frank Li]

Since difference SoCs require different sequence for exiting L2, let's add
a separate "exit_from_l2()" callback. This callback can be used to execute
SoC specific sequence.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---

Notes:
    Change from v2 to v3
    - fixed according to mani's feedback
      1. update commit message
      2. move dw_pcie_host_ops to next patch
      3. check return value from exit_from_l2()
    Change from v1 to v2
    - change subject 'a' to 'A'
    
    Change from v1 to v2
    - change subject 'a' to 'A'

 drivers/pci/controller/dwc/pci-layerscape.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
index 37956e09c65bd..aea89926bcc4f 100644
--- a/drivers/pci/controller/dwc/pci-layerscape.c
+++ b/drivers/pci/controller/dwc/pci-layerscape.c
@@ -39,6 +39,7 @@
 
 struct ls_pcie_drvdata {
 	const u32 pf_off;
+	int (*exit_from_l2)(struct dw_pcie_rp *pp);
 	bool pm_support;
 };
 
@@ -125,7 +126,7 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
 		dev_err(pcie->pci->dev, "PME_Turn_off timeout\n");
 }
 
-static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
+static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct ls_pcie *pcie = to_ls_pcie(pci);
@@ -150,6 +151,8 @@ static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
 				 10000);
 	if (ret)
 		dev_err(pcie->pci->dev, "L2 exit timeout\n");
+
+	return ret;
 }
 
 static int ls_pcie_host_init(struct dw_pcie_rp *pp)
@@ -180,6 +183,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
 static const struct ls_pcie_drvdata layerscape_drvdata = {
 	.pf_off = 0xc0000,
 	.pm_support = true,
+	.exit_from_l2 = ls_pcie_exit_from_l2,
 };
 
 static const struct of_device_id ls_pcie_of_match[] = {
@@ -247,11 +251,14 @@ static int ls_pcie_suspend_noirq(struct device *dev)
 static int ls_pcie_resume_noirq(struct device *dev)
 {
 	struct ls_pcie *pcie = dev_get_drvdata(dev);
+	int ret;
 
 	if (!pcie->drvdata->pm_support)
 		return 0;
 
-	ls_pcie_exit_from_l2(&pcie->pci->pp);
+	ret = pcie->drvdata->exit_from_l2(&pcie->pci->pp);
+	if (ret)
+		return ret;
 
 	return dw_pcie_resume_noirq(pcie->pci);
 }
-- 
2.34.1

[PATCH v3 2/4] PCI: layerscape: Add suspend/resume for ls1021a

[Frank Li]

ls1021a add suspend/resume support.

Implement callback ls1021a_pcie_send_turnoff_msg(), which write scfg's
SCFG_PEXPMWRCR to issue PME_Turn_off message.

Implement ls1021a_pcie_exit_from_l2() to let controller exit L2 state.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---

Notes:
    Change from v2 to v3
    - update according to mani's feedback
    change from v1 to v2
    - change subject 'a' to 'A'

 drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
index aea89926bcc4f..6f47cfe146c44 100644
--- a/drivers/pci/controller/dwc/pci-layerscape.c
+++ b/drivers/pci/controller/dwc/pci-layerscape.c
@@ -35,11 +35,21 @@
 #define PF_MCR_PTOMR		BIT(0)
 #define PF_MCR_EXL2S		BIT(1)
 
+/* LS1021A PEXn PM Write Control Register */
+#define SCFG_PEXPMWRCR(idx)	(0x5c + (idx) * 0x64)
+#define PMXMTTURNOFF		BIT(31)
+#define SCFG_PEXSFTRSTCR	0x190
+#define PEXSR(idx)		BIT(idx)
+
 #define PCIE_IATU_NUM		6
 
+#define LS_PCIE_DRV_SCFG	BIT(0)
+
 struct ls_pcie_drvdata {
 	const u32 pf_off;
+	const struct dw_pcie_host_ops *ops;
 	int (*exit_from_l2)(struct dw_pcie_rp *pp);
+	int flags;
 	bool pm_support;
 };
 
@@ -47,6 +57,8 @@ struct ls_pcie {
 	struct dw_pcie *pci;
 	const struct ls_pcie_drvdata *drvdata;
 	void __iomem *pf_base;
+	struct regmap *scfg;
+	int index;
 	bool big_endian;
 };
 
@@ -171,13 +183,65 @@ static int ls_pcie_host_init(struct dw_pcie_rp *pp)
 	return 0;
 }
 
+static void ls1021a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+
+	/* Send PME_Turn_Off message */
+	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
+	val |= PMXMTTURNOFF;
+	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
+
+	/*
+	 * There is no specific register to check for PME_To_Ack from endpoint.
+	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
+	 */
+	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
+
+	/*
+	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
+	 * to complete the PME_Turn_Off handshake.
+	 */
+	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
+	val &= ~PMXMTTURNOFF;
+	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
+}
+
+static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+
+	/* Only way exit from l2 is that do software reset */
+	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
+	val |= PEXSR(pcie->index);
+	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
+
+	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
+	val &= ~PEXSR(pcie->index);
+	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
+
+	return 0;
+}
+
 static const struct dw_pcie_host_ops ls_pcie_host_ops = {
 	.host_init = ls_pcie_host_init,
 	.pme_turn_off = ls_pcie_send_turnoff_msg,
 };
 
+static const struct dw_pcie_host_ops ls1021a_pcie_host_ops = {
+	.host_init = ls_pcie_host_init,
+	.pme_turn_off = ls1021a_pcie_send_turnoff_msg,
+};
+
 static const struct ls_pcie_drvdata ls1021a_drvdata = {
-	.pm_support = false,
+	.pm_support = true,
+	.ops = &ls1021a_pcie_host_ops,
+	.exit_from_l2 = ls1021a_pcie_exit_from_l2,
+	.flags = LS_PCIE_DRV_SCFG,
 };
 
 static const struct ls_pcie_drvdata layerscape_drvdata = {
@@ -205,6 +269,8 @@ static int ls_pcie_probe(struct platform_device *pdev)
 	struct dw_pcie *pci;
 	struct ls_pcie *pcie;
 	struct resource *dbi_base;
+	u32 index[2];
+	int ret;
 
 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
 	if (!pcie)
@@ -220,6 +286,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
 	pci->pp.ops = &ls_pcie_host_ops;
 
 	pcie->pci = pci;
+	pci->pp.ops = pcie->drvdata->ops ? pcie->drvdata->ops : &ls_pcie_host_ops;
 
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
@@ -230,6 +297,23 @@ static int ls_pcie_probe(struct platform_device *pdev)
 
 	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
 
+	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
+
+		pcie->scfg = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,pcie-scfg");
+		if (IS_ERR(pcie->scfg)) {
+			dev_err(dev, "No syscfg phandle specified\n");
+			return PTR_ERR(pcie->scfg);
+		}
+
+		ret = of_property_read_u32_array(dev->of_node, "fsl,pcie-scfg", index, 2);
+		if (ret) {
+			pcie->scfg = NULL;
+			return ret;
+		}
+
+		pcie->index = index[1];
+	}
+
 	if (!ls_pcie_is_bridge(pcie))
 		return -ENODEV;
 
-- 
2.34.1

[PATCH v3 3/4] PCI: layerscape: Rename pf_* as pf_lut_*

[Frank Li]

'pf' and 'lut' is just difference name in difference chips, but basic it is
a MMIO base address plus an offset.

Rename it to avoid duplicate pf_* and lut_* in driver.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---

Notes:
    change from v1 to v3
    - new patch at v3

 drivers/pci/controller/dwc/pci-layerscape.c | 34 ++++++++++-----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
index 6f47cfe146c44..4b663b20d8612 100644
--- a/drivers/pci/controller/dwc/pci-layerscape.c
+++ b/drivers/pci/controller/dwc/pci-layerscape.c
@@ -46,7 +46,7 @@
 #define LS_PCIE_DRV_SCFG	BIT(0)
 
 struct ls_pcie_drvdata {
-	const u32 pf_off;
+	const u32 pf_lut_off;
 	const struct dw_pcie_host_ops *ops;
 	int (*exit_from_l2)(struct dw_pcie_rp *pp);
 	int flags;
@@ -56,13 +56,13 @@ struct ls_pcie_drvdata {
 struct ls_pcie {
 	struct dw_pcie *pci;
 	const struct ls_pcie_drvdata *drvdata;
-	void __iomem *pf_base;
+	void __iomem *pf_lut_base;
 	struct regmap *scfg;
 	int index;
 	bool big_endian;
 };
 
-#define ls_pcie_pf_readl_addr(addr)	ls_pcie_pf_readl(pcie, addr)
+#define ls_pcie_pf_lut_readl_addr(addr)	ls_pcie_pf_lut_readl(pcie, addr)
 #define to_ls_pcie(x)	dev_get_drvdata((x)->dev)
 
 static bool ls_pcie_is_bridge(struct ls_pcie *pcie)
@@ -103,20 +103,20 @@ static void ls_pcie_fix_error_response(struct ls_pcie *pcie)
 	iowrite32(PCIE_ABSERR_SETTING, pci->dbi_base + PCIE_ABSERR);
 }
 
-static u32 ls_pcie_pf_readl(struct ls_pcie *pcie, u32 off)
+static u32 ls_pcie_pf_lut_readl(struct ls_pcie *pcie, u32 off)
 {
 	if (pcie->big_endian)
-		return ioread32be(pcie->pf_base + off);
+		return ioread32be(pcie->pf_lut_base + off);
 
-	return ioread32(pcie->pf_base + off);
+	return ioread32(pcie->pf_lut_base + off);
 }
 
-static void ls_pcie_pf_writel(struct ls_pcie *pcie, u32 off, u32 val)
+static void ls_pcie_pf_lut_writel(struct ls_pcie *pcie, u32 off, u32 val)
 {
 	if (pcie->big_endian)
-		iowrite32be(val, pcie->pf_base + off);
+		iowrite32be(val, pcie->pf_lut_base + off);
 	else
-		iowrite32(val, pcie->pf_base + off);
+		iowrite32(val, pcie->pf_lut_base + off);
 }
 
 static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
@@ -126,11 +126,11 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
 	u32 val;
 	int ret;
 
-	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
 	val |= PF_MCR_PTOMR;
-	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
 
-	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
+	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
 				 val, !(val & PF_MCR_PTOMR),
 				 PCIE_PME_TO_L2_TIMEOUT_US/10,
 				 PCIE_PME_TO_L2_TIMEOUT_US);
@@ -149,15 +149,15 @@ static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
 	 * Set PF_MCR_EXL2S bit in LS_PCIE_PF_MCR register for the link
 	 * to exit L2 state.
 	 */
-	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
 	val |= PF_MCR_EXL2S;
-	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
 
 	/*
 	 * L2 exit timeout of 10ms is not defined in the specifications,
 	 * it was chosen based on empirical observations.
 	 */
-	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
+	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
 				 val, !(val & PF_MCR_EXL2S),
 				 1000,
 				 10000);
@@ -245,7 +245,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
 };
 
 static const struct ls_pcie_drvdata layerscape_drvdata = {
-	.pf_off = 0xc0000,
+	.pf_lut_off = 0xc0000,
 	.pm_support = true,
 	.exit_from_l2 = ls_pcie_exit_from_l2,
 };
@@ -295,7 +295,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
 
 	pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian");
 
-	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
+	pcie->pf_lut_base = pci->dbi_base + pcie->drvdata->pf_lut_off;
 
 	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
 
-- 
2.34.1

[PATCH v3 4/4] PCI: layerscape: Add suspend/resume for ls1043a

[Frank Li]

ls1043a add suspend/resume support.
Implement ls1043a_pcie_send_turnoff_msg() to send PME_Turn_Off message.
Implement ls1043a_pcie_exit_from_l2() to exit from L2 state.

Signed-off-by: Frank Li <Frank.Li@nxp.com>
---

Notes:
    Change from v2 to v3
    - Remove ls_pcie_lut_readl(writel) function
    
    Change from v1 to v2
    - Update subject 'a' to 'A'

 drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
index 4b663b20d8612..9656224960b0c 100644
--- a/drivers/pci/controller/dwc/pci-layerscape.c
+++ b/drivers/pci/controller/dwc/pci-layerscape.c
@@ -41,6 +41,15 @@
 #define SCFG_PEXSFTRSTCR	0x190
 #define PEXSR(idx)		BIT(idx)
 
+/* LS1043A PEX PME control register */
+#define SCFG_PEXPMECR		0x144
+#define PEXPME(idx)		BIT(31 - (idx) * 4)
+
+/* LS1043A PEX LUT debug register */
+#define LS_PCIE_LDBG	0x7fc
+#define LDBG_SR		BIT(30)
+#define LDBG_WE		BIT(31)
+
 #define PCIE_IATU_NUM		6
 
 #define LS_PCIE_DRV_SCFG	BIT(0)
@@ -227,6 +236,68 @@ static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
 	return 0;
 }
 
+static void ls1043a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+
+	if (!pcie->scfg) {
+		dev_dbg(pcie->pci->dev, "SYSCFG is NULL\n");
+		return;
+	}
+
+	/* Send Turn_off message */
+	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
+	val |= PEXPME(pcie->index);
+	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
+
+	/*
+	 * There is no specific register to check for PME_To_Ack from endpoint.
+	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
+	 */
+	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
+
+	/*
+	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
+	 * to complete the PME_Turn_Off handshake.
+	 */
+	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
+	val &= ~PEXPME(pcie->index);
+	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
+}
+
+static int ls1043a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	u32 val;
+
+	/*
+	 * Only way let PEX module exit L2 is do a software reset.
+	 * LDBG_WE: allows the user to have write access to the PEXDBG[SR] for both setting and
+	 *	    clearing the soft reset on the PEX module.
+	 * LDBG_SR: When SR is set to 1, the PEX module enters soft reset.
+	 */
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG);
+	val |= LDBG_WE;
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val);
+
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG);
+	val |= LDBG_SR;
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val);
+
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG);
+	val &= ~LDBG_SR;
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val);
+
+	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG);
+	val &= ~LDBG_WE;
+	ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val);
+
+	return 0;
+}
+
 static const struct dw_pcie_host_ops ls_pcie_host_ops = {
 	.host_init = ls_pcie_host_init,
 	.pme_turn_off = ls_pcie_send_turnoff_msg,
@@ -244,6 +315,19 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
 	.flags = LS_PCIE_DRV_SCFG,
 };
 
+static const struct dw_pcie_host_ops ls1043a_pcie_host_ops = {
+	.host_init = ls_pcie_host_init,
+	.pme_turn_off = ls1043a_pcie_send_turnoff_msg,
+};
+
+static const struct ls_pcie_drvdata ls1043a_drvdata = {
+	.pf_lut_off = 0x10000,
+	.pm_support = true,
+	.ops = &ls1043a_pcie_host_ops,
+	.exit_from_l2 = ls1043a_pcie_exit_from_l2,
+	.flags = LS_PCIE_DRV_SCFG,
+};
+
 static const struct ls_pcie_drvdata layerscape_drvdata = {
 	.pf_lut_off = 0xc0000,
 	.pm_support = true,
@@ -254,7 +338,7 @@ static const struct of_device_id ls_pcie_of_match[] = {
 	{ .compatible = "fsl,ls1012a-pcie", .data = &layerscape_drvdata },
 	{ .compatible = "fsl,ls1021a-pcie", .data = &ls1021a_drvdata },
 	{ .compatible = "fsl,ls1028a-pcie", .data = &layerscape_drvdata },
-	{ .compatible = "fsl,ls1043a-pcie", .data = &ls1021a_drvdata },
+	{ .compatible = "fsl,ls1043a-pcie", .data = &ls1043a_drvdata },
 	{ .compatible = "fsl,ls1046a-pcie", .data = &layerscape_drvdata },
 	{ .compatible = "fsl,ls2080a-pcie", .data = &layerscape_drvdata },
 	{ .compatible = "fsl,ls2085a-pcie", .data = &layerscape_drvdata },
-- 
2.34.1

Re: [PATCH v3 0/4] PCI: layerscape: Add suspend/resume support for ls1043 and ls1021

[Frank Li]

On Tue, Oct 17, 2023 at 03:31:41PM -0400, Frank Li wrote:
> Add suspend/resume support for ls1043 and ls1021.
> Change log see each patch
> 
> Frank Li (4):
>   PCI: layerscape: Add function pointer for exit_from_l2()
>   PCI: layerscape: Add suspend/resume for ls1021a
>   PCI: layerscape: Rename pf_* as pf_lut_*
>   PCI: layerscape: Add suspend/resume for ls1043a
> 

@mani:
	Do you have any additional comments for these?

Frank

>  drivers/pci/controller/dwc/pci-layerscape.c | 217 ++++++++++++++++++--
>  1 file changed, 196 insertions(+), 21 deletions(-)
> 
> -- 
> 2.34.1
> 

Re: [PATCH v3 1/4] PCI: layerscape: Add function pointer for exit_from_l2()

[Manivannan Sadhasivam]

On Tue, Oct 17, 2023 at 03:31:42PM -0400, Frank Li wrote:
> Since difference SoCs require different sequence for exiting L2, let's add
> a separate "exit_from_l2()" callback. This callback can be used to execute
> SoC specific sequence.
> 

I missed the fact that this patch honors the return value of the callback (which
was ignored previously). So this should be added to the description as well.

> Signed-off-by: Frank Li <Frank.Li@nxp.com>

With that,

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

- Mani

> ---
> 
> Notes:
>     Change from v2 to v3
>     - fixed according to mani's feedback
>       1. update commit message
>       2. move dw_pcie_host_ops to next patch
>       3. check return value from exit_from_l2()
>     Change from v1 to v2
>     - change subject 'a' to 'A'
>     
>     Change from v1 to v2
>     - change subject 'a' to 'A'
> 
>  drivers/pci/controller/dwc/pci-layerscape.c | 11 +++++++++--
>  1 file changed, 9 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> index 37956e09c65bd..aea89926bcc4f 100644
> --- a/drivers/pci/controller/dwc/pci-layerscape.c
> +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> @@ -39,6 +39,7 @@
>  
>  struct ls_pcie_drvdata {
>  	const u32 pf_off;
> +	int (*exit_from_l2)(struct dw_pcie_rp *pp);
>  	bool pm_support;
>  };
>  
> @@ -125,7 +126,7 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
>  		dev_err(pcie->pci->dev, "PME_Turn_off timeout\n");
>  }
>  
> -static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> +static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
>  {
>  	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
>  	struct ls_pcie *pcie = to_ls_pcie(pci);
> @@ -150,6 +151,8 @@ static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
>  				 10000);
>  	if (ret)
>  		dev_err(pcie->pci->dev, "L2 exit timeout\n");
> +
> +	return ret;
>  }
>  
>  static int ls_pcie_host_init(struct dw_pcie_rp *pp)
> @@ -180,6 +183,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
>  static const struct ls_pcie_drvdata layerscape_drvdata = {
>  	.pf_off = 0xc0000,
>  	.pm_support = true,
> +	.exit_from_l2 = ls_pcie_exit_from_l2,
>  };
>  
>  static const struct of_device_id ls_pcie_of_match[] = {
> @@ -247,11 +251,14 @@ static int ls_pcie_suspend_noirq(struct device *dev)
>  static int ls_pcie_resume_noirq(struct device *dev)
>  {
>  	struct ls_pcie *pcie = dev_get_drvdata(dev);
> +	int ret;
>  
>  	if (!pcie->drvdata->pm_support)
>  		return 0;
>  
> -	ls_pcie_exit_from_l2(&pcie->pci->pp);
> +	ret = pcie->drvdata->exit_from_l2(&pcie->pci->pp);
> +	if (ret)
> +		return ret;
>  
>  	return dw_pcie_resume_noirq(pcie->pci);
>  }
> -- 
> 2.34.1
> 

-- 
மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 2/4] PCI: layerscape: Add suspend/resume for ls1021a

[Manivannan Sadhasivam]

On Tue, Oct 17, 2023 at 03:31:43PM -0400, Frank Li wrote:
> ls1021a add suspend/resume support.
> 
> Implement callback ls1021a_pcie_send_turnoff_msg(), which write scfg's
> SCFG_PEXPMWRCR to issue PME_Turn_off message.
> 
> Implement ls1021a_pcie_exit_from_l2() to let controller exit L2 state.
> 

I'd like to reword it to better reflect what the patch does:

"In the suspend path, PME_Turn_Off message is sent to the endpoint to transition
the link to L2/L3_Ready state. In this SoC, there is no way to check if the
controller has received the PME_To_Ack from the endpoint or not. So to be on the
safer side, the driver just waits for PCIE_PME_TO_L2_TIMEOUT_US before asserting
the SoC specific PMXMTTURNOFF bit to complete the PME_Turn_Off handshake. This
link would then enter L2/L3 state depending on the VAUX supply.

In the resume path, the link is brought back from L2 to L0 by doing a software
reset."

Although I do have questions on the resume behavior below.

> Signed-off-by: Frank Li <Frank.Li@nxp.com>
> ---
> 
> Notes:
>     Change from v2 to v3
>     - update according to mani's feedback
>     change from v1 to v2
>     - change subject 'a' to 'A'
> 
>  drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
>  1 file changed, 85 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> index aea89926bcc4f..6f47cfe146c44 100644
> --- a/drivers/pci/controller/dwc/pci-layerscape.c
> +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> @@ -35,11 +35,21 @@
>  #define PF_MCR_PTOMR		BIT(0)
>  #define PF_MCR_EXL2S		BIT(1)
>  
> +/* LS1021A PEXn PM Write Control Register */
> +#define SCFG_PEXPMWRCR(idx)	(0x5c + (idx) * 0x64)
> +#define PMXMTTURNOFF		BIT(31)
> +#define SCFG_PEXSFTRSTCR	0x190
> +#define PEXSR(idx)		BIT(idx)
> +
>  #define PCIE_IATU_NUM		6
>  
> +#define LS_PCIE_DRV_SCFG	BIT(0)
> +
>  struct ls_pcie_drvdata {
>  	const u32 pf_off;
> +	const struct dw_pcie_host_ops *ops;
>  	int (*exit_from_l2)(struct dw_pcie_rp *pp);
> +	int flags;

Why not "bool scfg_support"?

>  	bool pm_support;
>  };
>  
> @@ -47,6 +57,8 @@ struct ls_pcie {
>  	struct dw_pcie *pci;
>  	const struct ls_pcie_drvdata *drvdata;
>  	void __iomem *pf_base;
> +	struct regmap *scfg;
> +	int index;
>  	bool big_endian;
>  };
>  
> @@ -171,13 +183,65 @@ static int ls_pcie_host_init(struct dw_pcie_rp *pp)
>  	return 0;
>  }
>  
> +static void ls1021a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct ls_pcie *pcie = to_ls_pcie(pci);
> +	u32 val;
> +
> +	/* Send PME_Turn_Off message */
> +	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
> +	val |= PMXMTTURNOFF;
> +	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
> +
> +	/*
> +	 * There is no specific register to check for PME_To_Ack from endpoint.
> +	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
> +	 */
> +	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
> +
> +	/*
> +	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
> +	 * to complete the PME_Turn_Off handshake.
> +	 */
> +	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
> +	val &= ~PMXMTTURNOFF;
> +	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
> +}
> +
> +static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct ls_pcie *pcie = to_ls_pcie(pci);
> +	u32 val;
> +
> +	/* Only way exit from l2 is that do software reset */

So, what does exactly "software reset" mean? Are you resetting the endpoint or
some specific registers/blocks in the controller?

Also, what if the link goes to L3 in the case of no VAUX?

> +	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
> +	val |= PEXSR(pcie->index);
> +	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
> +
> +	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
> +	val &= ~PEXSR(pcie->index);
> +	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
> +
> +	return 0;
> +}
> +
>  static const struct dw_pcie_host_ops ls_pcie_host_ops = {
>  	.host_init = ls_pcie_host_init,
>  	.pme_turn_off = ls_pcie_send_turnoff_msg,
>  };
>  
> +static const struct dw_pcie_host_ops ls1021a_pcie_host_ops = {
> +	.host_init = ls_pcie_host_init,
> +	.pme_turn_off = ls1021a_pcie_send_turnoff_msg,
> +};
> +
>  static const struct ls_pcie_drvdata ls1021a_drvdata = {
> -	.pm_support = false,
> +	.pm_support = true,
> +	.ops = &ls1021a_pcie_host_ops,
> +	.exit_from_l2 = ls1021a_pcie_exit_from_l2,
> +	.flags = LS_PCIE_DRV_SCFG,
>  };
>  
>  static const struct ls_pcie_drvdata layerscape_drvdata = {
> @@ -205,6 +269,8 @@ static int ls_pcie_probe(struct platform_device *pdev)
>  	struct dw_pcie *pci;
>  	struct ls_pcie *pcie;
>  	struct resource *dbi_base;
> +	u32 index[2];
> +	int ret;
>  
>  	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
>  	if (!pcie)
> @@ -220,6 +286,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
>  	pci->pp.ops = &ls_pcie_host_ops;
>  
>  	pcie->pci = pci;
> +	pci->pp.ops = pcie->drvdata->ops ? pcie->drvdata->ops : &ls_pcie_host_ops;
>  
>  	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
>  	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
> @@ -230,6 +297,23 @@ static int ls_pcie_probe(struct platform_device *pdev)
>  
>  	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
>  
> +	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
> +

Remove extra newline.

- Mani

-- 
மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 3/4] PCI: layerscape: Rename pf_* as pf_lut_*

[Manivannan Sadhasivam]

On Tue, Oct 17, 2023 at 03:31:44PM -0400, Frank Li wrote:
> 'pf' and 'lut' is just difference name in difference chips, but basic it is
> a MMIO base address plus an offset.
> 
> Rename it to avoid duplicate pf_* and lut_* in driver.
> 

"pci-layerscape-ep.c" uses "ls_lut_" prefix and now you are using "pf_lut_". May
I know the difference between these two? Can we just use a common name?

- Mani

> Signed-off-by: Frank Li <Frank.Li@nxp.com>
> ---
> 
> Notes:
>     change from v1 to v3
>     - new patch at v3
> 
>  drivers/pci/controller/dwc/pci-layerscape.c | 34 ++++++++++-----------
>  1 file changed, 17 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> index 6f47cfe146c44..4b663b20d8612 100644
> --- a/drivers/pci/controller/dwc/pci-layerscape.c
> +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> @@ -46,7 +46,7 @@
>  #define LS_PCIE_DRV_SCFG	BIT(0)
>  
>  struct ls_pcie_drvdata {
> -	const u32 pf_off;
> +	const u32 pf_lut_off;
>  	const struct dw_pcie_host_ops *ops;
>  	int (*exit_from_l2)(struct dw_pcie_rp *pp);
>  	int flags;
> @@ -56,13 +56,13 @@ struct ls_pcie_drvdata {
>  struct ls_pcie {
>  	struct dw_pcie *pci;
>  	const struct ls_pcie_drvdata *drvdata;
> -	void __iomem *pf_base;
> +	void __iomem *pf_lut_base;
>  	struct regmap *scfg;
>  	int index;
>  	bool big_endian;
>  };
>  
> -#define ls_pcie_pf_readl_addr(addr)	ls_pcie_pf_readl(pcie, addr)
> +#define ls_pcie_pf_lut_readl_addr(addr)	ls_pcie_pf_lut_readl(pcie, addr)
>  #define to_ls_pcie(x)	dev_get_drvdata((x)->dev)
>  
>  static bool ls_pcie_is_bridge(struct ls_pcie *pcie)
> @@ -103,20 +103,20 @@ static void ls_pcie_fix_error_response(struct ls_pcie *pcie)
>  	iowrite32(PCIE_ABSERR_SETTING, pci->dbi_base + PCIE_ABSERR);
>  }
>  
> -static u32 ls_pcie_pf_readl(struct ls_pcie *pcie, u32 off)
> +static u32 ls_pcie_pf_lut_readl(struct ls_pcie *pcie, u32 off)
>  {
>  	if (pcie->big_endian)
> -		return ioread32be(pcie->pf_base + off);
> +		return ioread32be(pcie->pf_lut_base + off);
>  
> -	return ioread32(pcie->pf_base + off);
> +	return ioread32(pcie->pf_lut_base + off);
>  }
>  
> -static void ls_pcie_pf_writel(struct ls_pcie *pcie, u32 off, u32 val)
> +static void ls_pcie_pf_lut_writel(struct ls_pcie *pcie, u32 off, u32 val)
>  {
>  	if (pcie->big_endian)
> -		iowrite32be(val, pcie->pf_base + off);
> +		iowrite32be(val, pcie->pf_lut_base + off);
>  	else
> -		iowrite32(val, pcie->pf_base + off);
> +		iowrite32(val, pcie->pf_lut_base + off);
>  }
>  
>  static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> @@ -126,11 +126,11 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
>  	u32 val;
>  	int ret;
>  
> -	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
> +	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
>  	val |= PF_MCR_PTOMR;
> -	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
> +	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
>  
> -	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
> +	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
>  				 val, !(val & PF_MCR_PTOMR),
>  				 PCIE_PME_TO_L2_TIMEOUT_US/10,
>  				 PCIE_PME_TO_L2_TIMEOUT_US);
> @@ -149,15 +149,15 @@ static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
>  	 * Set PF_MCR_EXL2S bit in LS_PCIE_PF_MCR register for the link
>  	 * to exit L2 state.
>  	 */
> -	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
> +	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
>  	val |= PF_MCR_EXL2S;
> -	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
> +	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
>  
>  	/*
>  	 * L2 exit timeout of 10ms is not defined in the specifications,
>  	 * it was chosen based on empirical observations.
>  	 */
> -	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
> +	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
>  				 val, !(val & PF_MCR_EXL2S),
>  				 1000,
>  				 10000);
> @@ -245,7 +245,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
>  };
>  
>  static const struct ls_pcie_drvdata layerscape_drvdata = {
> -	.pf_off = 0xc0000,
> +	.pf_lut_off = 0xc0000,
>  	.pm_support = true,
>  	.exit_from_l2 = ls_pcie_exit_from_l2,
>  };
> @@ -295,7 +295,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
>  
>  	pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian");
>  
> -	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
> +	pcie->pf_lut_base = pci->dbi_base + pcie->drvdata->pf_lut_off;
>  
>  	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
>  
> -- 
> 2.34.1
> 

-- 
மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 4/4] PCI: layerscape: Add suspend/resume for ls1043a

[Manivannan Sadhasivam]

On Tue, Oct 17, 2023 at 03:31:45PM -0400, Frank Li wrote:
> ls1043a add suspend/resume support.
> Implement ls1043a_pcie_send_turnoff_msg() to send PME_Turn_Off message.
> Implement ls1043a_pcie_exit_from_l2() to exit from L2 state.
> 

Please use the suggestion I gave in patch 2/4.

> Signed-off-by: Frank Li <Frank.Li@nxp.com>
> ---
> 
> Notes:
>     Change from v2 to v3
>     - Remove ls_pcie_lut_readl(writel) function
>     
>     Change from v1 to v2
>     - Update subject 'a' to 'A'
> 
>  drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
>  1 file changed, 85 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> index 4b663b20d8612..9656224960b0c 100644
> --- a/drivers/pci/controller/dwc/pci-layerscape.c
> +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> @@ -41,6 +41,15 @@
>  #define SCFG_PEXSFTRSTCR	0x190
>  #define PEXSR(idx)		BIT(idx)
>  
> +/* LS1043A PEX PME control register */
> +#define SCFG_PEXPMECR		0x144
> +#define PEXPME(idx)		BIT(31 - (idx) * 4)
> +
> +/* LS1043A PEX LUT debug register */
> +#define LS_PCIE_LDBG	0x7fc
> +#define LDBG_SR		BIT(30)
> +#define LDBG_WE		BIT(31)
> +
>  #define PCIE_IATU_NUM		6
>  
>  #define LS_PCIE_DRV_SCFG	BIT(0)
> @@ -227,6 +236,68 @@ static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
>  	return 0;
>  }
>  
> +static void ls1043a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct ls_pcie *pcie = to_ls_pcie(pci);
> +	u32 val;
> +
> +	if (!pcie->scfg) {
> +		dev_dbg(pcie->pci->dev, "SYSCFG is NULL\n");
> +		return;
> +	}

Why scfg is optional for this SoC and not for the other one added in patch 2/4?

> +
> +	/* Send Turn_off message */
> +	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
> +	val |= PEXPME(pcie->index);
> +	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
> +

In my previous review, I asked you to use a common function and just pass the
offsets, as the sequence is same for both the SoCs. But you ignored it :/

> +	/*
> +	 * There is no specific register to check for PME_To_Ack from endpoint.
> +	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
> +	 */
> +	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
> +
> +	/*
> +	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
> +	 * to complete the PME_Turn_Off handshake.
> +	 */
> +	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
> +	val &= ~PEXPME(pcie->index);
> +	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
> +}
> +
> +static int ls1043a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> +	struct ls_pcie *pcie = to_ls_pcie(pci);
> +	u32 val;
> +
> +	/*
> +	 * Only way let PEX module exit L2 is do a software reset.

Same comment applies as patch 2/4.

- Mani

-- 
மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 1/4] PCI: layerscape: Add function pointer for exit_from_l2()

[Frank Li]

On Thu, Nov 02, 2023 at 10:28:08PM +0530, Manivannan Sadhasivam wrote:
> On Tue, Oct 17, 2023 at 03:31:42PM -0400, Frank Li wrote:
> > Since difference SoCs require different sequence for exiting L2, let's add
> > a separate "exit_from_l2()" callback. This callback can be used to execute
> > SoC specific sequence.
> > 
> 
> I missed the fact that this patch honors the return value of the callback (which
> was ignored previously). So this should be added to the description as well.

How about add below?

"Change ls_pcie_exit_from_l2() return value from void to int. Return error
when exit_from_l2() failure to exit suspend flow."

Frank


> 
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> 
> With that,
> 
> Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
> 
> - Mani
> 
> > ---
> > 
> > Notes:
> >     Change from v2 to v3
> >     - fixed according to mani's feedback
> >       1. update commit message
> >       2. move dw_pcie_host_ops to next patch
> >       3. check return value from exit_from_l2()
> >     Change from v1 to v2
> >     - change subject 'a' to 'A'
> >     
> >     Change from v1 to v2
> >     - change subject 'a' to 'A'
> > 
> >  drivers/pci/controller/dwc/pci-layerscape.c | 11 +++++++++--
> >  1 file changed, 9 insertions(+), 2 deletions(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> > index 37956e09c65bd..aea89926bcc4f 100644
> > --- a/drivers/pci/controller/dwc/pci-layerscape.c
> > +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> > @@ -39,6 +39,7 @@
> >  
> >  struct ls_pcie_drvdata {
> >  	const u32 pf_off;
> > +	int (*exit_from_l2)(struct dw_pcie_rp *pp);
> >  	bool pm_support;
> >  };
> >  
> > @@ -125,7 +126,7 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> >  		dev_err(pcie->pci->dev, "PME_Turn_off timeout\n");
> >  }
> >  
> > -static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> > +static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> >  {
> >  	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> >  	struct ls_pcie *pcie = to_ls_pcie(pci);
> > @@ -150,6 +151,8 @@ static void ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> >  				 10000);
> >  	if (ret)
> >  		dev_err(pcie->pci->dev, "L2 exit timeout\n");
> > +
> > +	return ret;
> >  }
> >  
> >  static int ls_pcie_host_init(struct dw_pcie_rp *pp)
> > @@ -180,6 +183,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
> >  static const struct ls_pcie_drvdata layerscape_drvdata = {
> >  	.pf_off = 0xc0000,
> >  	.pm_support = true,
> > +	.exit_from_l2 = ls_pcie_exit_from_l2,
> >  };
> >  
> >  static const struct of_device_id ls_pcie_of_match[] = {
> > @@ -247,11 +251,14 @@ static int ls_pcie_suspend_noirq(struct device *dev)
> >  static int ls_pcie_resume_noirq(struct device *dev)
> >  {
> >  	struct ls_pcie *pcie = dev_get_drvdata(dev);
> > +	int ret;
> >  
> >  	if (!pcie->drvdata->pm_support)
> >  		return 0;
> >  
> > -	ls_pcie_exit_from_l2(&pcie->pci->pp);
> > +	ret = pcie->drvdata->exit_from_l2(&pcie->pci->pp);
> > +	if (ret)
> > +		return ret;
> >  
> >  	return dw_pcie_resume_noirq(pcie->pci);
> >  }
> > -- 
> > 2.34.1
> > 
> 
> -- 
> மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 2/4] PCI: layerscape: Add suspend/resume for ls1021a

[Frank Li]

On Thu, Nov 02, 2023 at 10:58:09PM +0530, Manivannan Sadhasivam wrote:
> On Tue, Oct 17, 2023 at 03:31:43PM -0400, Frank Li wrote:
> > ls1021a add suspend/resume support.
> > 
> > Implement callback ls1021a_pcie_send_turnoff_msg(), which write scfg's
> > SCFG_PEXPMWRCR to issue PME_Turn_off message.
> > 
> > Implement ls1021a_pcie_exit_from_l2() to let controller exit L2 state.
> > 
> 
> I'd like to reword it to better reflect what the patch does:
> 
> "In the suspend path, PME_Turn_Off message is sent to the endpoint to transition
> the link to L2/L3_Ready state. In this SoC, there is no way to check if the
> controller has received the PME_To_Ack from the endpoint or not. So to be on the
> safer side, the driver just waits for PCIE_PME_TO_L2_TIMEOUT_US before asserting
> the SoC specific PMXMTTURNOFF bit to complete the PME_Turn_Off handshake. This
> link would then enter L2/L3 state depending on the VAUX supply.
> 
> In the resume path, the link is brought back from L2 to L0 by doing a software
> reset."
> 
> Although I do have questions on the resume behavior below.
> 
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > ---
> > 
> > Notes:
> >     Change from v2 to v3
> >     - update according to mani's feedback
> >     change from v1 to v2
> >     - change subject 'a' to 'A'
> > 
> >  drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
> >  1 file changed, 85 insertions(+), 1 deletion(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> > index aea89926bcc4f..6f47cfe146c44 100644
> > --- a/drivers/pci/controller/dwc/pci-layerscape.c
> > +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> > @@ -35,11 +35,21 @@
> >  #define PF_MCR_PTOMR		BIT(0)
> >  #define PF_MCR_EXL2S		BIT(1)
> >  
> > +/* LS1021A PEXn PM Write Control Register */
> > +#define SCFG_PEXPMWRCR(idx)	(0x5c + (idx) * 0x64)
> > +#define PMXMTTURNOFF		BIT(31)
> > +#define SCFG_PEXSFTRSTCR	0x190
> > +#define PEXSR(idx)		BIT(idx)
> > +
> >  #define PCIE_IATU_NUM		6
> >  
> > +#define LS_PCIE_DRV_SCFG	BIT(0)
> > +
> >  struct ls_pcie_drvdata {
> >  	const u32 pf_off;
> > +	const struct dw_pcie_host_ops *ops;
> >  	int (*exit_from_l2)(struct dw_pcie_rp *pp);
> > +	int flags;
> 
> Why not "bool scfg_support"?

It will be easy to add new flag if need in future.

> 
> >  	bool pm_support;
> >  };
> >  
> > @@ -47,6 +57,8 @@ struct ls_pcie {
> >  	struct dw_pcie *pci;
> >  	const struct ls_pcie_drvdata *drvdata;
> >  	void __iomem *pf_base;
> > +	struct regmap *scfg;
> > +	int index;
> >  	bool big_endian;
> >  };
> >  
> > @@ -171,13 +183,65 @@ static int ls_pcie_host_init(struct dw_pcie_rp *pp)
> >  	return 0;
> >  }
> >  
> > +static void ls1021a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> > +{
> > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > +	struct ls_pcie *pcie = to_ls_pcie(pci);
> > +	u32 val;
> > +
> > +	/* Send PME_Turn_Off message */
> > +	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
> > +	val |= PMXMTTURNOFF;
> > +	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
> > +
> > +	/*
> > +	 * There is no specific register to check for PME_To_Ack from endpoint.
> > +	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
> > +	 */
> > +	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
> > +
> > +	/*
> > +	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
> > +	 * to complete the PME_Turn_Off handshake.
> > +	 */
> > +	regmap_read(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), &val);
> > +	val &= ~PMXMTTURNOFF;
> > +	regmap_write(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), val);
> > +}
> > +
> > +static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> > +{
> > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > +	struct ls_pcie *pcie = to_ls_pcie(pci);
> > +	u32 val;
> > +
> > +	/* Only way exit from l2 is that do software reset */
> 
> So, what does exactly "software reset" mean? Are you resetting the endpoint or
> some specific registers/blocks in the controlleri?

No, it is PCIe controller reset. Not touch endpoint.

> 
> Also, what if the link goes to L3 in the case of no VAUX?

I am not exactly sure. it should be related with board design. I supposed
not big difference!

We can improve it when we met it.

> 
> > +	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
> > +	val |= PEXSR(pcie->index);
> > +	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
> > +
> > +	regmap_read(pcie->scfg, SCFG_PEXSFTRSTCR, &val);
> > +	val &= ~PEXSR(pcie->index);
> > +	regmap_write(pcie->scfg, SCFG_PEXSFTRSTCR, val);
> > +
> > +	return 0;
> > +}
> > +
> >  static const struct dw_pcie_host_ops ls_pcie_host_ops = {
> >  	.host_init = ls_pcie_host_init,
> >  	.pme_turn_off = ls_pcie_send_turnoff_msg,
> >  };
> >  
> > +static const struct dw_pcie_host_ops ls1021a_pcie_host_ops = {
> > +	.host_init = ls_pcie_host_init,
> > +	.pme_turn_off = ls1021a_pcie_send_turnoff_msg,
> > +};
> > +
> >  static const struct ls_pcie_drvdata ls1021a_drvdata = {
> > -	.pm_support = false,
> > +	.pm_support = true,
> > +	.ops = &ls1021a_pcie_host_ops,
> > +	.exit_from_l2 = ls1021a_pcie_exit_from_l2,
> > +	.flags = LS_PCIE_DRV_SCFG,
> >  };
> >  
> >  static const struct ls_pcie_drvdata layerscape_drvdata = {
> > @@ -205,6 +269,8 @@ static int ls_pcie_probe(struct platform_device *pdev)
> >  	struct dw_pcie *pci;
> >  	struct ls_pcie *pcie;
> >  	struct resource *dbi_base;
> > +	u32 index[2];
> > +	int ret;
> >  
> >  	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
> >  	if (!pcie)
> > @@ -220,6 +286,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
> >  	pci->pp.ops = &ls_pcie_host_ops;
> >  
> >  	pcie->pci = pci;
> > +	pci->pp.ops = pcie->drvdata->ops ? pcie->drvdata->ops : &ls_pcie_host_ops;
> >  
> >  	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
> >  	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
> > @@ -230,6 +297,23 @@ static int ls_pcie_probe(struct platform_device *pdev)
> >  
> >  	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
> >  
> > +	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
> > +
> 
> Remove extra newline.
> 
> - Mani
> 
> -- 
> மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 3/4] PCI: layerscape: Rename pf_* as pf_lut_*

[Frank Li]

On Thu, Nov 02, 2023 at 11:03:14PM +0530, Manivannan Sadhasivam wrote:
> On Tue, Oct 17, 2023 at 03:31:44PM -0400, Frank Li wrote:
> > 'pf' and 'lut' is just difference name in difference chips, but basic it is
> > a MMIO base address plus an offset.
> > 
> > Rename it to avoid duplicate pf_* and lut_* in driver.
> > 
> 
> "pci-layerscape-ep.c" uses "ls_lut_" prefix and now you are using "pf_lut_". May
> I know the difference between these two? Can we just use a common name?

Some chip use name lut, some chip use name pf. I think ls_pcie_pf_lut_*()
is better name then 'ls_lut_' in pci-layerscape-ep.c to align with spec. 

If need, I can rename "ls_lut_" in "pci-layerscape-ep.c" later.

Frank

> 
> - Mani
> 
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > ---
> > 
> > Notes:
> >     change from v1 to v3
> >     - new patch at v3
> > 
> >  drivers/pci/controller/dwc/pci-layerscape.c | 34 ++++++++++-----------
> >  1 file changed, 17 insertions(+), 17 deletions(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> > index 6f47cfe146c44..4b663b20d8612 100644
> > --- a/drivers/pci/controller/dwc/pci-layerscape.c
> > +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> > @@ -46,7 +46,7 @@
> >  #define LS_PCIE_DRV_SCFG	BIT(0)
> >  
> >  struct ls_pcie_drvdata {
> > -	const u32 pf_off;
> > +	const u32 pf_lut_off;
> >  	const struct dw_pcie_host_ops *ops;
> >  	int (*exit_from_l2)(struct dw_pcie_rp *pp);
> >  	int flags;
> > @@ -56,13 +56,13 @@ struct ls_pcie_drvdata {
> >  struct ls_pcie {
> >  	struct dw_pcie *pci;
> >  	const struct ls_pcie_drvdata *drvdata;
> > -	void __iomem *pf_base;
> > +	void __iomem *pf_lut_base;
> >  	struct regmap *scfg;
> >  	int index;
> >  	bool big_endian;
> >  };
> >  
> > -#define ls_pcie_pf_readl_addr(addr)	ls_pcie_pf_readl(pcie, addr)
> > +#define ls_pcie_pf_lut_readl_addr(addr)	ls_pcie_pf_lut_readl(pcie, addr)
> >  #define to_ls_pcie(x)	dev_get_drvdata((x)->dev)
> >  
> >  static bool ls_pcie_is_bridge(struct ls_pcie *pcie)
> > @@ -103,20 +103,20 @@ static void ls_pcie_fix_error_response(struct ls_pcie *pcie)
> >  	iowrite32(PCIE_ABSERR_SETTING, pci->dbi_base + PCIE_ABSERR);
> >  }
> >  
> > -static u32 ls_pcie_pf_readl(struct ls_pcie *pcie, u32 off)
> > +static u32 ls_pcie_pf_lut_readl(struct ls_pcie *pcie, u32 off)
> >  {
> >  	if (pcie->big_endian)
> > -		return ioread32be(pcie->pf_base + off);
> > +		return ioread32be(pcie->pf_lut_base + off);
> >  
> > -	return ioread32(pcie->pf_base + off);
> > +	return ioread32(pcie->pf_lut_base + off);
> >  }
> >  
> > -static void ls_pcie_pf_writel(struct ls_pcie *pcie, u32 off, u32 val)
> > +static void ls_pcie_pf_lut_writel(struct ls_pcie *pcie, u32 off, u32 val)
> >  {
> >  	if (pcie->big_endian)
> > -		iowrite32be(val, pcie->pf_base + off);
> > +		iowrite32be(val, pcie->pf_lut_base + off);
> >  	else
> > -		iowrite32(val, pcie->pf_base + off);
> > +		iowrite32(val, pcie->pf_lut_base + off);
> >  }
> >  
> >  static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> > @@ -126,11 +126,11 @@ static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> >  	u32 val;
> >  	int ret;
> >  
> > -	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
> > +	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
> >  	val |= PF_MCR_PTOMR;
> > -	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
> > +	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
> >  
> > -	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
> > +	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
> >  				 val, !(val & PF_MCR_PTOMR),
> >  				 PCIE_PME_TO_L2_TIMEOUT_US/10,
> >  				 PCIE_PME_TO_L2_TIMEOUT_US);
> > @@ -149,15 +149,15 @@ static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> >  	 * Set PF_MCR_EXL2S bit in LS_PCIE_PF_MCR register for the link
> >  	 * to exit L2 state.
> >  	 */
> > -	val = ls_pcie_pf_readl(pcie, LS_PCIE_PF_MCR);
> > +	val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR);
> >  	val |= PF_MCR_EXL2S;
> > -	ls_pcie_pf_writel(pcie, LS_PCIE_PF_MCR, val);
> > +	ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val);
> >  
> >  	/*
> >  	 * L2 exit timeout of 10ms is not defined in the specifications,
> >  	 * it was chosen based on empirical observations.
> >  	 */
> > -	ret = readx_poll_timeout(ls_pcie_pf_readl_addr, LS_PCIE_PF_MCR,
> > +	ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR,
> >  				 val, !(val & PF_MCR_EXL2S),
> >  				 1000,
> >  				 10000);
> > @@ -245,7 +245,7 @@ static const struct ls_pcie_drvdata ls1021a_drvdata = {
> >  };
> >  
> >  static const struct ls_pcie_drvdata layerscape_drvdata = {
> > -	.pf_off = 0xc0000,
> > +	.pf_lut_off = 0xc0000,
> >  	.pm_support = true,
> >  	.exit_from_l2 = ls_pcie_exit_from_l2,
> >  };
> > @@ -295,7 +295,7 @@ static int ls_pcie_probe(struct platform_device *pdev)
> >  
> >  	pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian");
> >  
> > -	pcie->pf_base = pci->dbi_base + pcie->drvdata->pf_off;
> > +	pcie->pf_lut_base = pci->dbi_base + pcie->drvdata->pf_lut_off;
> >  
> >  	if (pcie->drvdata->flags & LS_PCIE_DRV_SCFG) {
> >  
> > -- 
> > 2.34.1
> > 
> 
> -- 
> மணிவண்ணன் சதாசிவம்

Re: [PATCH v3 4/4] PCI: layerscape: Add suspend/resume for ls1043a

[Frank Li]

On Thu, Nov 02, 2023 at 11:09:00PM +0530, Manivannan Sadhasivam wrote:
> On Tue, Oct 17, 2023 at 03:31:45PM -0400, Frank Li wrote:
> > ls1043a add suspend/resume support.
> > Implement ls1043a_pcie_send_turnoff_msg() to send PME_Turn_Off message.
> > Implement ls1043a_pcie_exit_from_l2() to exit from L2 state.
> > 
> 
> Please use the suggestion I gave in patch 2/4.
> 
> > Signed-off-by: Frank Li <Frank.Li@nxp.com>
> > ---
> > 
> > Notes:
> >     Change from v2 to v3
> >     - Remove ls_pcie_lut_readl(writel) function
> >     
> >     Change from v1 to v2
> >     - Update subject 'a' to 'A'
> > 
> >  drivers/pci/controller/dwc/pci-layerscape.c | 86 ++++++++++++++++++++-
> >  1 file changed, 85 insertions(+), 1 deletion(-)
> > 
> > diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c
> > index 4b663b20d8612..9656224960b0c 100644
> > --- a/drivers/pci/controller/dwc/pci-layerscape.c
> > +++ b/drivers/pci/controller/dwc/pci-layerscape.c
> > @@ -41,6 +41,15 @@
> >  #define SCFG_PEXSFTRSTCR	0x190
> >  #define PEXSR(idx)		BIT(idx)
> >  
> > +/* LS1043A PEX PME control register */
> > +#define SCFG_PEXPMECR		0x144
> > +#define PEXPME(idx)		BIT(31 - (idx) * 4)
> > +
> > +/* LS1043A PEX LUT debug register */
> > +#define LS_PCIE_LDBG	0x7fc
> > +#define LDBG_SR		BIT(30)
> > +#define LDBG_WE		BIT(31)
> > +
> >  #define PCIE_IATU_NUM		6
> >  
> >  #define LS_PCIE_DRV_SCFG	BIT(0)
> > @@ -227,6 +236,68 @@ static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> >  	return 0;
> >  }
> >  
> > +static void ls1043a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp)
> > +{
> > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > +	struct ls_pcie *pcie = to_ls_pcie(pci);
> > +	u32 val;
> > +
> > +	if (!pcie->scfg) {
> > +		dev_dbg(pcie->pci->dev, "SYSCFG is NULL\n");
> > +		return;
> > +	}
> 
> Why scfg is optional for this SoC and not for the other one added in patch 2/4?

No, it is not optional for this SoC. This check can be removed as your
previous comments about 2/4.

> 
> > +
> > +	/* Send Turn_off message */
> > +	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
> > +	val |= PEXPME(pcie->index);
> > +	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
> > +
> 
> In my previous review, I asked you to use a common function and just pass the
> offsets, as the sequence is same for both the SoCs. But you ignored it :/
> 

Sorry, I will fixed it at next version. 

> > +	/*
> > +	 * There is no specific register to check for PME_To_Ack from endpoint.
> > +	 * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US.
> > +	 */
> > +	mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000);
> > +
> > +	/*
> > +	 * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit
> > +	 * to complete the PME_Turn_Off handshake.
> > +	 */
> > +	regmap_read(pcie->scfg, SCFG_PEXPMECR, &val);
> > +	val &= ~PEXPME(pcie->index);
> > +	regmap_write(pcie->scfg, SCFG_PEXPMECR, val);
> > +}
> > +
> > +static int ls1043a_pcie_exit_from_l2(struct dw_pcie_rp *pp)
> > +{
> > +	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> > +	struct ls_pcie *pcie = to_ls_pcie(pci);
> > +	u32 val;
> > +
> > +	/*
> > +	 * Only way let PEX module exit L2 is do a software reset.
> 
> Same comment applies as patch 2/4.
> 
> - Mani
> 
> -- 
> மணிவண்ணன் சதாசிவம்