[PATCH v3 0/5] PCI: qcom: Fix higher MSI vectors handling

[PATCH v3 0/5] PCI: qcom: Fix higher MSI vectors handling

[Dmitry Baryshkov]

I have replied with my Tested-by to the patch at [2], which has landed
in the linux-next as the commit 8ae0117418f3 ("PCI: qcom:
Add support for handling MSIs from 8 endpoints"). However lately I
noticed that during the tests I still had 'pcie_pme=nomsi', so the
device was not forced to use higher MSI vectors.

After removing this option I noticed that hight MSI vectors are not
delivered on tested platforms. After additional research I stumbled upon
a patch in msm-4.14 ([1]), which describes that each group of MSI
vectors is mapped to the separate interrupt. Implement corresponding
mapping.

Patchseries dependecies: [2] (landed in pci-next) and [3] (for the
schema change).

Changes since v2:
 - Fix and rephrase commit message for patch 2.

Changes since v1:
 - Split a huge patch into three patches as suggested by Bjorn Helgaas
 - snps,dw-pcie removal is now part of [3]

[1] https://git.codelinaro.org/clo/la/kernel/msm-4.14/-/commit/671a3d5f129f4bfe477152292ada2194c8440d22
[2] https://lore.kernel.org/linux-arm-msm/20211214101319.25258-1-manivannan.sadhasivam@linaro.org/
[3] https://lore.kernel.org/linux-arm-msm/20220422211002.2012070-1-dmitry.baryshkov@linaro.org/

Dmitry Baryshkov (5):
  PCI: dwc: Convert msi_irq to the array
  PCI: dwc: Teach dwc core to parse additional MSI interrupts
  PCI: qcom: Handle MSI IRQs properly
  dt-bindings: pci/qcom,pcie: support additional MSI interrupts
  arm64: dts: qcom: sm8250: provide additional MSI interrupts

 .../devicetree/bindings/pci/qcom,pcie.yaml    | 12 ++++-
 arch/arm64/boot/dts/qcom/sm8250.dtsi          | 11 +++-
 drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
 drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
 .../pci/controller/dwc/pcie-designware-host.c | 53 ++++++++++++++-----
 drivers/pci/controller/dwc/pcie-designware.h  |  3 +-
 drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
 drivers/pci/controller/dwc/pcie-qcom.c        |  1 +
 drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
 drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
 10 files changed, 69 insertions(+), 21 deletions(-)

-- 
2.35.1

[PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Dmitry Baryshkov]

Qualcomm version of DWC PCIe controller supports more than 32 MSI
interrupts, but they are routed to separate interrupts in groups of 32
vectors. To support such configuration, change the msi_irq field into an
array. Let the DWC core handle all interrupts that were set in this
array.

Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
---
 drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
 drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
 .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
 drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
 drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
 drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
 drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
 7 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index dfcdeb432dc8..0919c96dcdbd 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
 		return pp->irq;
 
 	/* MSI IRQ is muxed */
-	pp->msi_irq = -ENODEV;
+	pp->msi_irq[0] = -ENODEV;
 
 	ret = dra7xx_pcie_init_irq_domain(pp);
 	if (ret < 0)
diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
index 467c8d1cd7e4..4f2010bd9cd7 100644
--- a/drivers/pci/controller/dwc/pci-exynos.c
+++ b/drivers/pci/controller/dwc/pci-exynos.c
@@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
 	}
 
 	pp->ops = &exynos_pcie_host_ops;
-	pp->msi_irq = -ENODEV;
+	pp->msi_irq[0] = -ENODEV;
 
 	ret = dw_pcie_host_init(pp);
 	if (ret) {
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 2fa86f32d964..5d90009a0f73 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
 
 static void dw_pcie_free_msi(struct pcie_port *pp)
 {
-	if (pp->msi_irq)
-		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
+	u32 ctrl;
+
+	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
+		if (pp->msi_irq[ctrl])
+			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
 
 	irq_domain_remove(pp->msi_domain);
 	irq_domain_remove(pp->irq_domain);
@@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
 			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
 				pp->irq_mask[ctrl] = ~0;
 
-			if (!pp->msi_irq) {
-				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
-				if (pp->msi_irq < 0) {
-					pp->msi_irq = platform_get_irq(pdev, 0);
-					if (pp->msi_irq < 0)
-						return pp->msi_irq;
+			if (!pp->msi_irq[0]) {
+				int irq = platform_get_irq_byname_optional(pdev, "msi");
+
+				if (irq < 0) {
+					irq = platform_get_irq(pdev, 0);
+					if (irq < 0)
+						return irq;
 				}
+				pp->msi_irq[0] = irq;
 			}
 
 			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
@@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
 			if (ret)
 				return ret;
 
-			if (pp->msi_irq > 0)
-				irq_set_chained_handler_and_data(pp->msi_irq,
-							    dw_chained_msi_isr,
-							    pp);
+			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
+				if (pp->msi_irq[ctrl] > 0)
+					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
+									 dw_chained_msi_isr,
+									 pp);
 
 			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
 			if (ret)
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 7d6e9b7576be..9c1a38b0a6b3 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -187,7 +187,7 @@ struct pcie_port {
 	u32			io_size;
 	int			irq;
 	const struct dw_pcie_host_ops *ops;
-	int			msi_irq;
+	int			msi_irq[MAX_MSI_CTRLS];
 	struct irq_domain	*irq_domain;
 	struct irq_domain	*msi_domain;
 	u16			msi_msg;
diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
index 1ac29a6eef22..297e6e926c00 100644
--- a/drivers/pci/controller/dwc/pcie-keembay.c
+++ b/drivers/pci/controller/dwc/pcie-keembay.c
@@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
 	int ret;
 
 	pp->ops = &keembay_pcie_host_ops;
-	pp->msi_irq = -ENODEV;
+	pp->msi_irq[0] = -ENODEV;
 
 	ret = keembay_pcie_setup_msi_irq(pcie);
 	if (ret)
diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
index 1569e82b5568..cc7776833810 100644
--- a/drivers/pci/controller/dwc/pcie-spear13xx.c
+++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
@@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
 	}
 
 	pp->ops = &spear13xx_pcie_host_ops;
-	pp->msi_irq = -ENODEV;
+	pp->msi_irq[0] = -ENODEV;
 
 	ret = dw_pcie_host_init(pp);
 	if (ret) {
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
index b1b5f836a806..e75712db85b0 100644
--- a/drivers/pci/controller/dwc/pcie-tegra194.c
+++ b/drivers/pci/controller/dwc/pcie-tegra194.c
@@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
 
 	disable_irq(pcie->pci.pp.irq);
 	if (IS_ENABLED(CONFIG_PCI_MSI))
-		disable_irq(pcie->pci.pp.msi_irq);
+		disable_irq(pcie->pci.pp.msi_irq[0]);
 
 	tegra194_pcie_pme_turnoff(pcie);
 	tegra_pcie_unconfig_controller(pcie);
-- 
2.35.1

Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Manivannan Sadhasivam]

On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
> Qualcomm version of DWC PCIe controller supports more than 32 MSI
> interrupts, but they are routed to separate interrupts in groups of 32
> vectors. To support such configuration, change the msi_irq field into an
> array. Let the DWC core handle all interrupts that were set in this
> array.
> 

Instead of defining it as an array, can we allocate it dynamically in the
controller drivers instead? This has two benefits:

1. There is no need of using a dedicated flag.
2. Controller drivers that don't support MSIs can pass NULL and in the core we
can use platform_get_irq_byname_optional() to get supported number of MSIs from
devicetree.

Thanks,
Mani

> Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
> ---
>  drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
>  drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
>  .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
>  drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
>  drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
>  drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
>  drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
>  7 files changed, 24 insertions(+), 18 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
> index dfcdeb432dc8..0919c96dcdbd 100644
> --- a/drivers/pci/controller/dwc/pci-dra7xx.c
> +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
> @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
>  		return pp->irq;
>  
>  	/* MSI IRQ is muxed */
> -	pp->msi_irq = -ENODEV;
> +	pp->msi_irq[0] = -ENODEV;
>  
>  	ret = dra7xx_pcie_init_irq_domain(pp);
>  	if (ret < 0)
> diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
> index 467c8d1cd7e4..4f2010bd9cd7 100644
> --- a/drivers/pci/controller/dwc/pci-exynos.c
> +++ b/drivers/pci/controller/dwc/pci-exynos.c
> @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
>  	}
>  
>  	pp->ops = &exynos_pcie_host_ops;
> -	pp->msi_irq = -ENODEV;
> +	pp->msi_irq[0] = -ENODEV;
>  
>  	ret = dw_pcie_host_init(pp);
>  	if (ret) {
> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> index 2fa86f32d964..5d90009a0f73 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
>  
>  static void dw_pcie_free_msi(struct pcie_port *pp)
>  {
> -	if (pp->msi_irq)
> -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
> +	u32 ctrl;
> +
> +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
> +		if (pp->msi_irq[ctrl])
> +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
>  
>  	irq_domain_remove(pp->msi_domain);
>  	irq_domain_remove(pp->irq_domain);
> @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
>  			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>  				pp->irq_mask[ctrl] = ~0;
>  
> -			if (!pp->msi_irq) {
> -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
> -				if (pp->msi_irq < 0) {
> -					pp->msi_irq = platform_get_irq(pdev, 0);
> -					if (pp->msi_irq < 0)
> -						return pp->msi_irq;
> +			if (!pp->msi_irq[0]) {
> +				int irq = platform_get_irq_byname_optional(pdev, "msi");
> +
> +				if (irq < 0) {
> +					irq = platform_get_irq(pdev, 0);
> +					if (irq < 0)
> +						return irq;
>  				}
> +				pp->msi_irq[0] = irq;
>  			}
>  
>  			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
> @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
>  			if (ret)
>  				return ret;
>  
> -			if (pp->msi_irq > 0)
> -				irq_set_chained_handler_and_data(pp->msi_irq,
> -							    dw_chained_msi_isr,
> -							    pp);
> +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
> +				if (pp->msi_irq[ctrl] > 0)
> +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
> +									 dw_chained_msi_isr,
> +									 pp);
>  
>  			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
>  			if (ret)
> diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
> index 7d6e9b7576be..9c1a38b0a6b3 100644
> --- a/drivers/pci/controller/dwc/pcie-designware.h
> +++ b/drivers/pci/controller/dwc/pcie-designware.h
> @@ -187,7 +187,7 @@ struct pcie_port {
>  	u32			io_size;
>  	int			irq;
>  	const struct dw_pcie_host_ops *ops;
> -	int			msi_irq;
> +	int			msi_irq[MAX_MSI_CTRLS];
>  	struct irq_domain	*irq_domain;
>  	struct irq_domain	*msi_domain;
>  	u16			msi_msg;
> diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
> index 1ac29a6eef22..297e6e926c00 100644
> --- a/drivers/pci/controller/dwc/pcie-keembay.c
> +++ b/drivers/pci/controller/dwc/pcie-keembay.c
> @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
>  	int ret;
>  
>  	pp->ops = &keembay_pcie_host_ops;
> -	pp->msi_irq = -ENODEV;
> +	pp->msi_irq[0] = -ENODEV;
>  
>  	ret = keembay_pcie_setup_msi_irq(pcie);
>  	if (ret)
> diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
> index 1569e82b5568..cc7776833810 100644
> --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
> +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
> @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
>  	}
>  
>  	pp->ops = &spear13xx_pcie_host_ops;
> -	pp->msi_irq = -ENODEV;
> +	pp->msi_irq[0] = -ENODEV;
>  
>  	ret = dw_pcie_host_init(pp);
>  	if (ret) {
> diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
> index b1b5f836a806..e75712db85b0 100644
> --- a/drivers/pci/controller/dwc/pcie-tegra194.c
> +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
> @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
>  
>  	disable_irq(pcie->pci.pp.irq);
>  	if (IS_ENABLED(CONFIG_PCI_MSI))
> -		disable_irq(pcie->pci.pp.msi_irq);
> +		disable_irq(pcie->pci.pp.msi_irq[0]);
>  
>  	tegra194_pcie_pme_turnoff(pcie);
>  	tegra_pcie_unconfig_controller(pcie);
> -- 
> 2.35.1
> 

Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Dmitry Baryshkov]

On 27/04/2022 17:13, Manivannan Sadhasivam wrote:
> On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
>> Qualcomm version of DWC PCIe controller supports more than 32 MSI
>> interrupts, but they are routed to separate interrupts in groups of 32
>> vectors. To support such configuration, change the msi_irq field into an
>> array. Let the DWC core handle all interrupts that were set in this
>> array.
>>
> 
> Instead of defining it as an array, can we allocate it dynamically in the
> controller drivers instead? This has two benefits:
> 
> 1. There is no need of using a dedicated flag.
> 2. Controller drivers that don't support MSIs can pass NULL and in the core we
> can use platform_get_irq_byname_optional() to get supported number of MSIs from
> devicetree.

I think using dynamic allocation would make code worse. It would add 
additional checks here and there.

If you don't like this design. I have an alternative suggestion: export 
the dw_chained_msi_irq() and move allocation of all MSIs to the 
pcie-qcom code. Would that be better? I'm not sure whether this 
multi-host-IRQ design is used on other DWC platforms or not.

> 
> Thanks,
> Mani
> 
>> Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
>> ---
>>   drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
>>   drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
>>   .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
>>   drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
>>   drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
>>   drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
>>   drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
>>   7 files changed, 24 insertions(+), 18 deletions(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
>> index dfcdeb432dc8..0919c96dcdbd 100644
>> --- a/drivers/pci/controller/dwc/pci-dra7xx.c
>> +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
>> @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
>>   		return pp->irq;
>>   
>>   	/* MSI IRQ is muxed */
>> -	pp->msi_irq = -ENODEV;
>> +	pp->msi_irq[0] = -ENODEV;
>>   
>>   	ret = dra7xx_pcie_init_irq_domain(pp);
>>   	if (ret < 0)
>> diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
>> index 467c8d1cd7e4..4f2010bd9cd7 100644
>> --- a/drivers/pci/controller/dwc/pci-exynos.c
>> +++ b/drivers/pci/controller/dwc/pci-exynos.c
>> @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
>>   	}
>>   
>>   	pp->ops = &exynos_pcie_host_ops;
>> -	pp->msi_irq = -ENODEV;
>> +	pp->msi_irq[0] = -ENODEV;
>>   
>>   	ret = dw_pcie_host_init(pp);
>>   	if (ret) {
>> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
>> index 2fa86f32d964..5d90009a0f73 100644
>> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
>> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
>> @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
>>   
>>   static void dw_pcie_free_msi(struct pcie_port *pp)
>>   {
>> -	if (pp->msi_irq)
>> -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
>> +	u32 ctrl;
>> +
>> +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
>> +		if (pp->msi_irq[ctrl])
>> +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
>>   
>>   	irq_domain_remove(pp->msi_domain);
>>   	irq_domain_remove(pp->irq_domain);
>> @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>   			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>>   				pp->irq_mask[ctrl] = ~0;
>>   
>> -			if (!pp->msi_irq) {
>> -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
>> -				if (pp->msi_irq < 0) {
>> -					pp->msi_irq = platform_get_irq(pdev, 0);
>> -					if (pp->msi_irq < 0)
>> -						return pp->msi_irq;
>> +			if (!pp->msi_irq[0]) {
>> +				int irq = platform_get_irq_byname_optional(pdev, "msi");
>> +
>> +				if (irq < 0) {
>> +					irq = platform_get_irq(pdev, 0);
>> +					if (irq < 0)
>> +						return irq;
>>   				}
>> +				pp->msi_irq[0] = irq;
>>   			}
>>   
>>   			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
>> @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>   			if (ret)
>>   				return ret;
>>   
>> -			if (pp->msi_irq > 0)
>> -				irq_set_chained_handler_and_data(pp->msi_irq,
>> -							    dw_chained_msi_isr,
>> -							    pp);
>> +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>> +				if (pp->msi_irq[ctrl] > 0)
>> +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
>> +									 dw_chained_msi_isr,
>> +									 pp);
>>   
>>   			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
>>   			if (ret)
>> diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
>> index 7d6e9b7576be..9c1a38b0a6b3 100644
>> --- a/drivers/pci/controller/dwc/pcie-designware.h
>> +++ b/drivers/pci/controller/dwc/pcie-designware.h
>> @@ -187,7 +187,7 @@ struct pcie_port {
>>   	u32			io_size;
>>   	int			irq;
>>   	const struct dw_pcie_host_ops *ops;
>> -	int			msi_irq;
>> +	int			msi_irq[MAX_MSI_CTRLS];
>>   	struct irq_domain	*irq_domain;
>>   	struct irq_domain	*msi_domain;
>>   	u16			msi_msg;
>> diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
>> index 1ac29a6eef22..297e6e926c00 100644
>> --- a/drivers/pci/controller/dwc/pcie-keembay.c
>> +++ b/drivers/pci/controller/dwc/pcie-keembay.c
>> @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
>>   	int ret;
>>   
>>   	pp->ops = &keembay_pcie_host_ops;
>> -	pp->msi_irq = -ENODEV;
>> +	pp->msi_irq[0] = -ENODEV;
>>   
>>   	ret = keembay_pcie_setup_msi_irq(pcie);
>>   	if (ret)
>> diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
>> index 1569e82b5568..cc7776833810 100644
>> --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
>> +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
>> @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
>>   	}
>>   
>>   	pp->ops = &spear13xx_pcie_host_ops;
>> -	pp->msi_irq = -ENODEV;
>> +	pp->msi_irq[0] = -ENODEV;
>>   
>>   	ret = dw_pcie_host_init(pp);
>>   	if (ret) {
>> diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
>> index b1b5f836a806..e75712db85b0 100644
>> --- a/drivers/pci/controller/dwc/pcie-tegra194.c
>> +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
>> @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
>>   
>>   	disable_irq(pcie->pci.pp.irq);
>>   	if (IS_ENABLED(CONFIG_PCI_MSI))
>> -		disable_irq(pcie->pci.pp.msi_irq);
>> +		disable_irq(pcie->pci.pp.msi_irq[0]);
>>   
>>   	tegra194_pcie_pme_turnoff(pcie);
>>   	tegra_pcie_unconfig_controller(pcie);
>> -- 
>> 2.35.1
>>


-- 
With best wishes
Dmitry

Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Manivannan Sadhasivam]

On Wed, Apr 27, 2022 at 07:59:57PM +0300, Dmitry Baryshkov wrote:
> On 27/04/2022 17:13, Manivannan Sadhasivam wrote:
> > On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
> > > Qualcomm version of DWC PCIe controller supports more than 32 MSI
> > > interrupts, but they are routed to separate interrupts in groups of 32
> > > vectors. To support such configuration, change the msi_irq field into an
> > > array. Let the DWC core handle all interrupts that were set in this
> > > array.
> > > 
> > 
> > Instead of defining it as an array, can we allocate it dynamically in the
> > controller drivers instead? This has two benefits:
> > 
> > 1. There is no need of using a dedicated flag.
> > 2. Controller drivers that don't support MSIs can pass NULL and in the core we
> > can use platform_get_irq_byname_optional() to get supported number of MSIs from
> > devicetree.
> 
> I think using dynamic allocation would make code worse. It would add
> additional checks here and there.
> 

I take back my suggestion of allocating the memory for msi_irq in controller
drivers. It should be done in the designware-host instead.

We already know how many MSIs are supported by the platform using num_vectors.
So we should just allocate msi_irqs of num_vectors length in
dw_pcie_host_init() and populate it using platform_get_irq_byname_optional().

I don't think this can make the code worse.

> If you don't like this design. I have an alternative suggestion: export the
> dw_chained_msi_irq() and move allocation of all MSIs to the pcie-qcom code.
> Would that be better? I'm not sure whether this multi-host-IRQ design is
> used on other DWC platforms or not.
> 

No, I think the allocation should belong to designware-host.

Thanks,
Mani

> > 
> > Thanks,
> > Mani
> > 
> > > Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
> > > ---
> > >   drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
> > >   drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
> > >   .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
> > >   drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
> > >   drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
> > >   drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
> > >   drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
> > >   7 files changed, 24 insertions(+), 18 deletions(-)
> > > 
> > > diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
> > > index dfcdeb432dc8..0919c96dcdbd 100644
> > > --- a/drivers/pci/controller/dwc/pci-dra7xx.c
> > > +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
> > > @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
> > >   		return pp->irq;
> > >   	/* MSI IRQ is muxed */
> > > -	pp->msi_irq = -ENODEV;
> > > +	pp->msi_irq[0] = -ENODEV;
> > >   	ret = dra7xx_pcie_init_irq_domain(pp);
> > >   	if (ret < 0)
> > > diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
> > > index 467c8d1cd7e4..4f2010bd9cd7 100644
> > > --- a/drivers/pci/controller/dwc/pci-exynos.c
> > > +++ b/drivers/pci/controller/dwc/pci-exynos.c
> > > @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
> > >   	}
> > >   	pp->ops = &exynos_pcie_host_ops;
> > > -	pp->msi_irq = -ENODEV;
> > > +	pp->msi_irq[0] = -ENODEV;
> > >   	ret = dw_pcie_host_init(pp);
> > >   	if (ret) {
> > > diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> > > index 2fa86f32d964..5d90009a0f73 100644
> > > --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> > > +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> > > @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
> > >   static void dw_pcie_free_msi(struct pcie_port *pp)
> > >   {
> > > -	if (pp->msi_irq)
> > > -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
> > > +	u32 ctrl;
> > > +
> > > +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
> > > +		if (pp->msi_irq[ctrl])
> > > +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
> > >   	irq_domain_remove(pp->msi_domain);
> > >   	irq_domain_remove(pp->irq_domain);
> > > @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
> > >   			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
> > >   				pp->irq_mask[ctrl] = ~0;
> > > -			if (!pp->msi_irq) {
> > > -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
> > > -				if (pp->msi_irq < 0) {
> > > -					pp->msi_irq = platform_get_irq(pdev, 0);
> > > -					if (pp->msi_irq < 0)
> > > -						return pp->msi_irq;
> > > +			if (!pp->msi_irq[0]) {
> > > +				int irq = platform_get_irq_byname_optional(pdev, "msi");
> > > +
> > > +				if (irq < 0) {
> > > +					irq = platform_get_irq(pdev, 0);
> > > +					if (irq < 0)
> > > +						return irq;
> > >   				}
> > > +				pp->msi_irq[0] = irq;
> > >   			}
> > >   			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
> > > @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
> > >   			if (ret)
> > >   				return ret;
> > > -			if (pp->msi_irq > 0)
> > > -				irq_set_chained_handler_and_data(pp->msi_irq,
> > > -							    dw_chained_msi_isr,
> > > -							    pp);
> > > +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
> > > +				if (pp->msi_irq[ctrl] > 0)
> > > +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
> > > +									 dw_chained_msi_isr,
> > > +									 pp);
> > >   			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
> > >   			if (ret)
> > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
> > > index 7d6e9b7576be..9c1a38b0a6b3 100644
> > > --- a/drivers/pci/controller/dwc/pcie-designware.h
> > > +++ b/drivers/pci/controller/dwc/pcie-designware.h
> > > @@ -187,7 +187,7 @@ struct pcie_port {
> > >   	u32			io_size;
> > >   	int			irq;
> > >   	const struct dw_pcie_host_ops *ops;
> > > -	int			msi_irq;
> > > +	int			msi_irq[MAX_MSI_CTRLS];
> > >   	struct irq_domain	*irq_domain;
> > >   	struct irq_domain	*msi_domain;
> > >   	u16			msi_msg;
> > > diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
> > > index 1ac29a6eef22..297e6e926c00 100644
> > > --- a/drivers/pci/controller/dwc/pcie-keembay.c
> > > +++ b/drivers/pci/controller/dwc/pcie-keembay.c
> > > @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
> > >   	int ret;
> > >   	pp->ops = &keembay_pcie_host_ops;
> > > -	pp->msi_irq = -ENODEV;
> > > +	pp->msi_irq[0] = -ENODEV;
> > >   	ret = keembay_pcie_setup_msi_irq(pcie);
> > >   	if (ret)
> > > diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > index 1569e82b5568..cc7776833810 100644
> > > --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
> > >   	}
> > >   	pp->ops = &spear13xx_pcie_host_ops;
> > > -	pp->msi_irq = -ENODEV;
> > > +	pp->msi_irq[0] = -ENODEV;
> > >   	ret = dw_pcie_host_init(pp);
> > >   	if (ret) {
> > > diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
> > > index b1b5f836a806..e75712db85b0 100644
> > > --- a/drivers/pci/controller/dwc/pcie-tegra194.c
> > > +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
> > > @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
> > >   	disable_irq(pcie->pci.pp.irq);
> > >   	if (IS_ENABLED(CONFIG_PCI_MSI))
> > > -		disable_irq(pcie->pci.pp.msi_irq);
> > > +		disable_irq(pcie->pci.pp.msi_irq[0]);
> > >   	tegra194_pcie_pme_turnoff(pcie);
> > >   	tegra_pcie_unconfig_controller(pcie);
> > > -- 
> > > 2.35.1
> > > 
> 
> 
> -- 
> With best wishes
> Dmitry

Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Dmitry Baryshkov]

On 28/04/2022 09:06, Manivannan Sadhasivam wrote:
> On Wed, Apr 27, 2022 at 07:59:57PM +0300, Dmitry Baryshkov wrote:
>> On 27/04/2022 17:13, Manivannan Sadhasivam wrote:
>>> On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
>>>> Qualcomm version of DWC PCIe controller supports more than 32 MSI
>>>> interrupts, but they are routed to separate interrupts in groups of 32
>>>> vectors. To support such configuration, change the msi_irq field into an
>>>> array. Let the DWC core handle all interrupts that were set in this
>>>> array.
>>>>
>>>
>>> Instead of defining it as an array, can we allocate it dynamically in the
>>> controller drivers instead? This has two benefits:
>>>
>>> 1. There is no need of using a dedicated flag.
>>> 2. Controller drivers that don't support MSIs can pass NULL and in the core we
>>> can use platform_get_irq_byname_optional() to get supported number of MSIs from
>>> devicetree.
>>
>> I think using dynamic allocation would make code worse. It would add
>> additional checks here and there.
>>
> 
> I take back my suggestion of allocating the memory for msi_irq in controller
> drivers. It should be done in the designware-host instead.
> 
> We already know how many MSIs are supported by the platform using num_vectors.
> So we should just allocate msi_irqs of num_vectors length in
> dw_pcie_host_init() and populate it using platform_get_irq_byname_optional().
> 
> I don't think this can make the code worse.
> 
>> If you don't like this design. I have an alternative suggestion: export the
>> dw_chained_msi_irq() and move allocation of all MSIs to the pcie-qcom code.
>> Would that be better? I'm not sure whether this multi-host-IRQ design is
>> used on other DWC platforms or not.
>>
> 
> No, I think the allocation should belong to designware-host.

Granted that at least tegra and iMX tie all MSI vectors to a single host 
interrupt, I think it's impractical to assume that other hosts would 
benefit from such allocation. Let's move support for split IRQs into the 
pcie-qcom.c. We can make this generic later if any other host would use 
a separate per-"endpoint" (per-group) IRQ.

> 
> Thanks,
> Mani
> 
>>>
>>> Thanks,
>>> Mani
>>>
>>>> Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
>>>> ---
>>>>    drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
>>>>    drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
>>>>    .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
>>>>    drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
>>>>    7 files changed, 24 insertions(+), 18 deletions(-)
>>>>
>>>> diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> index dfcdeb432dc8..0919c96dcdbd 100644
>>>> --- a/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
>>>>    		return pp->irq;
>>>>    	/* MSI IRQ is muxed */
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dra7xx_pcie_init_irq_domain(pp);
>>>>    	if (ret < 0)
>>>> diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
>>>> index 467c8d1cd7e4..4f2010bd9cd7 100644
>>>> --- a/drivers/pci/controller/dwc/pci-exynos.c
>>>> +++ b/drivers/pci/controller/dwc/pci-exynos.c
>>>> @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
>>>>    	}
>>>>    	pp->ops = &exynos_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dw_pcie_host_init(pp);
>>>>    	if (ret) {
>>>> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> index 2fa86f32d964..5d90009a0f73 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
>>>>    static void dw_pcie_free_msi(struct pcie_port *pp)
>>>>    {
>>>> -	if (pp->msi_irq)
>>>> -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
>>>> +	u32 ctrl;
>>>> +
>>>> +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
>>>> +		if (pp->msi_irq[ctrl])
>>>> +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
>>>>    	irq_domain_remove(pp->msi_domain);
>>>>    	irq_domain_remove(pp->irq_domain);
>>>> @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>>>    			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>>>>    				pp->irq_mask[ctrl] = ~0;
>>>> -			if (!pp->msi_irq) {
>>>> -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
>>>> -				if (pp->msi_irq < 0) {
>>>> -					pp->msi_irq = platform_get_irq(pdev, 0);
>>>> -					if (pp->msi_irq < 0)
>>>> -						return pp->msi_irq;
>>>> +			if (!pp->msi_irq[0]) {
>>>> +				int irq = platform_get_irq_byname_optional(pdev, "msi");
>>>> +
>>>> +				if (irq < 0) {
>>>> +					irq = platform_get_irq(pdev, 0);
>>>> +					if (irq < 0)
>>>> +						return irq;
>>>>    				}
>>>> +				pp->msi_irq[0] = irq;
>>>>    			}
>>>>    			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
>>>> @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>>>    			if (ret)
>>>>    				return ret;
>>>> -			if (pp->msi_irq > 0)
>>>> -				irq_set_chained_handler_and_data(pp->msi_irq,
>>>> -							    dw_chained_msi_isr,
>>>> -							    pp);
>>>> +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>>>> +				if (pp->msi_irq[ctrl] > 0)
>>>> +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
>>>> +									 dw_chained_msi_isr,
>>>> +									 pp);
>>>>    			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
>>>>    			if (ret)
>>>> diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
>>>> index 7d6e9b7576be..9c1a38b0a6b3 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-designware.h
>>>> +++ b/drivers/pci/controller/dwc/pcie-designware.h
>>>> @@ -187,7 +187,7 @@ struct pcie_port {
>>>>    	u32			io_size;
>>>>    	int			irq;
>>>>    	const struct dw_pcie_host_ops *ops;
>>>> -	int			msi_irq;
>>>> +	int			msi_irq[MAX_MSI_CTRLS];
>>>>    	struct irq_domain	*irq_domain;
>>>>    	struct irq_domain	*msi_domain;
>>>>    	u16			msi_msg;
>>>> diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
>>>> index 1ac29a6eef22..297e6e926c00 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-keembay.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-keembay.c
>>>> @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
>>>>    	int ret;
>>>>    	pp->ops = &keembay_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = keembay_pcie_setup_msi_irq(pcie);
>>>>    	if (ret)
>>>> diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> index 1569e82b5568..cc7776833810 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
>>>>    	}
>>>>    	pp->ops = &spear13xx_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dw_pcie_host_init(pp);
>>>>    	if (ret) {
>>>> diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> index b1b5f836a806..e75712db85b0 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
>>>>    	disable_irq(pcie->pci.pp.irq);
>>>>    	if (IS_ENABLED(CONFIG_PCI_MSI))
>>>> -		disable_irq(pcie->pci.pp.msi_irq);
>>>> +		disable_irq(pcie->pci.pp.msi_irq[0]);
>>>>    	tegra194_pcie_pme_turnoff(pcie);
>>>>    	tegra_pcie_unconfig_controller(pcie);
>>>> -- 
>>>> 2.35.1
>>>>
>>
>>
>> -- 
>> With best wishes
>> Dmitry


-- 
With best wishes
Dmitry

Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Manivannan Sadhasivam]

On Thu, Apr 28, 2022 at 10:43:54AM +0300, Dmitry Baryshkov wrote:
> On 28/04/2022 09:06, Manivannan Sadhasivam wrote:
> > On Wed, Apr 27, 2022 at 07:59:57PM +0300, Dmitry Baryshkov wrote:
> > > On 27/04/2022 17:13, Manivannan Sadhasivam wrote:
> > > > On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
> > > > > Qualcomm version of DWC PCIe controller supports more than 32 MSI
> > > > > interrupts, but they are routed to separate interrupts in groups of 32
> > > > > vectors. To support such configuration, change the msi_irq field into an
> > > > > array. Let the DWC core handle all interrupts that were set in this
> > > > > array.
> > > > > 
> > > > 
> > > > Instead of defining it as an array, can we allocate it dynamically in the
> > > > controller drivers instead? This has two benefits:
> > > > 
> > > > 1. There is no need of using a dedicated flag.
> > > > 2. Controller drivers that don't support MSIs can pass NULL and in the core we
> > > > can use platform_get_irq_byname_optional() to get supported number of MSIs from
> > > > devicetree.
> > > 
> > > I think using dynamic allocation would make code worse. It would add
> > > additional checks here and there.
> > > 
> > 
> > I take back my suggestion of allocating the memory for msi_irq in controller
> > drivers. It should be done in the designware-host instead.
> > 
> > We already know how many MSIs are supported by the platform using num_vectors.
> > So we should just allocate msi_irqs of num_vectors length in
> > dw_pcie_host_init() and populate it using platform_get_irq_byname_optional().
> > 
> > I don't think this can make the code worse.
> > 
> > > If you don't like this design. I have an alternative suggestion: export the
> > > dw_chained_msi_irq() and move allocation of all MSIs to the pcie-qcom code.
> > > Would that be better? I'm not sure whether this multi-host-IRQ design is
> > > used on other DWC platforms or not.
> > > 
> > 
> > No, I think the allocation should belong to designware-host.
> 
> Granted that at least tegra and iMX tie all MSI vectors to a single host
> interrupt, I think it's impractical to assume that other hosts would benefit
> from such allocation. Let's move support for split IRQs into the
> pcie-qcom.c. We can make this generic later if any other host would use a
> separate per-"endpoint" (per-group) IRQ.
> 

Okay, I'm fine with moving just the split irq handling to Qcom driver.

Thanks,
Mani

> > 
> > Thanks,
> > Mani
> > 
> > > > 
> > > > Thanks,
> > > > Mani
> > > > 
> > > > > Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
> > > > > ---
> > > > >    drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
> > > > >    drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
> > > > >    .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
> > > > >    drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
> > > > >    drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
> > > > >    drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
> > > > >    drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
> > > > >    7 files changed, 24 insertions(+), 18 deletions(-)
> > > > > 
> > > > > diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
> > > > > index dfcdeb432dc8..0919c96dcdbd 100644
> > > > > --- a/drivers/pci/controller/dwc/pci-dra7xx.c
> > > > > +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
> > > > > @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
> > > > >    		return pp->irq;
> > > > >    	/* MSI IRQ is muxed */
> > > > > -	pp->msi_irq = -ENODEV;
> > > > > +	pp->msi_irq[0] = -ENODEV;
> > > > >    	ret = dra7xx_pcie_init_irq_domain(pp);
> > > > >    	if (ret < 0)
> > > > > diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
> > > > > index 467c8d1cd7e4..4f2010bd9cd7 100644
> > > > > --- a/drivers/pci/controller/dwc/pci-exynos.c
> > > > > +++ b/drivers/pci/controller/dwc/pci-exynos.c
> > > > > @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
> > > > >    	}
> > > > >    	pp->ops = &exynos_pcie_host_ops;
> > > > > -	pp->msi_irq = -ENODEV;
> > > > > +	pp->msi_irq[0] = -ENODEV;
> > > > >    	ret = dw_pcie_host_init(pp);
> > > > >    	if (ret) {
> > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> > > > > index 2fa86f32d964..5d90009a0f73 100644
> > > > > --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> > > > > +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> > > > > @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
> > > > >    static void dw_pcie_free_msi(struct pcie_port *pp)
> > > > >    {
> > > > > -	if (pp->msi_irq)
> > > > > -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
> > > > > +	u32 ctrl;
> > > > > +
> > > > > +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
> > > > > +		if (pp->msi_irq[ctrl])
> > > > > +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
> > > > >    	irq_domain_remove(pp->msi_domain);
> > > > >    	irq_domain_remove(pp->irq_domain);
> > > > > @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
> > > > >    			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
> > > > >    				pp->irq_mask[ctrl] = ~0;
> > > > > -			if (!pp->msi_irq) {
> > > > > -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
> > > > > -				if (pp->msi_irq < 0) {
> > > > > -					pp->msi_irq = platform_get_irq(pdev, 0);
> > > > > -					if (pp->msi_irq < 0)
> > > > > -						return pp->msi_irq;
> > > > > +			if (!pp->msi_irq[0]) {
> > > > > +				int irq = platform_get_irq_byname_optional(pdev, "msi");
> > > > > +
> > > > > +				if (irq < 0) {
> > > > > +					irq = platform_get_irq(pdev, 0);
> > > > > +					if (irq < 0)
> > > > > +						return irq;
> > > > >    				}
> > > > > +				pp->msi_irq[0] = irq;
> > > > >    			}
> > > > >    			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
> > > > > @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
> > > > >    			if (ret)
> > > > >    				return ret;
> > > > > -			if (pp->msi_irq > 0)
> > > > > -				irq_set_chained_handler_and_data(pp->msi_irq,
> > > > > -							    dw_chained_msi_isr,
> > > > > -							    pp);
> > > > > +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
> > > > > +				if (pp->msi_irq[ctrl] > 0)
> > > > > +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
> > > > > +									 dw_chained_msi_isr,
> > > > > +									 pp);
> > > > >    			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
> > > > >    			if (ret)
> > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
> > > > > index 7d6e9b7576be..9c1a38b0a6b3 100644
> > > > > --- a/drivers/pci/controller/dwc/pcie-designware.h
> > > > > +++ b/drivers/pci/controller/dwc/pcie-designware.h
> > > > > @@ -187,7 +187,7 @@ struct pcie_port {
> > > > >    	u32			io_size;
> > > > >    	int			irq;
> > > > >    	const struct dw_pcie_host_ops *ops;
> > > > > -	int			msi_irq;
> > > > > +	int			msi_irq[MAX_MSI_CTRLS];
> > > > >    	struct irq_domain	*irq_domain;
> > > > >    	struct irq_domain	*msi_domain;
> > > > >    	u16			msi_msg;
> > > > > diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
> > > > > index 1ac29a6eef22..297e6e926c00 100644
> > > > > --- a/drivers/pci/controller/dwc/pcie-keembay.c
> > > > > +++ b/drivers/pci/controller/dwc/pcie-keembay.c
> > > > > @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
> > > > >    	int ret;
> > > > >    	pp->ops = &keembay_pcie_host_ops;
> > > > > -	pp->msi_irq = -ENODEV;
> > > > > +	pp->msi_irq[0] = -ENODEV;
> > > > >    	ret = keembay_pcie_setup_msi_irq(pcie);
> > > > >    	if (ret)
> > > > > diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > > > index 1569e82b5568..cc7776833810 100644
> > > > > --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > > > +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
> > > > > @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
> > > > >    	}
> > > > >    	pp->ops = &spear13xx_pcie_host_ops;
> > > > > -	pp->msi_irq = -ENODEV;
> > > > > +	pp->msi_irq[0] = -ENODEV;
> > > > >    	ret = dw_pcie_host_init(pp);
> > > > >    	if (ret) {
> > > > > diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
> > > > > index b1b5f836a806..e75712db85b0 100644
> > > > > --- a/drivers/pci/controller/dwc/pcie-tegra194.c
> > > > > +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
> > > > > @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
> > > > >    	disable_irq(pcie->pci.pp.irq);
> > > > >    	if (IS_ENABLED(CONFIG_PCI_MSI))
> > > > > -		disable_irq(pcie->pci.pp.msi_irq);
> > > > > +		disable_irq(pcie->pci.pp.msi_irq[0]);
> > > > >    	tegra194_pcie_pme_turnoff(pcie);
> > > > >    	tegra_pcie_unconfig_controller(pcie);
> > > > > -- 
> > > > > 2.35.1
> > > > > 
> > > 
> > > 
> > > -- 
> > > With best wishes
> > > Dmitry
> 
> 
> -- 
> With best wishes
> Dmitry

[PATCH v3 2/5] PCI: dwc: Teach dwc core to parse additional MSI interrupts

[Dmitry Baryshkov]

DWC driver parses a single "msi" interrupt which gets fired when the EP
sends an MSI interrupt, however for some devices (Qualcomm) MSI vectors
are handled in groups by 32 vectors in each group. Add support for
parsing "split" MSI interrupts.

In addition to the "msi" interrupt, the code will lookup the "msi2",
"msi3", etc. IRQs and use them for the MSI group interrupts. For
backwards compatibility with existing DTS files, the code will not error
out if these interrupts are missing. Instead it will limit itself
to the number of MSI group IRQs declared in the DT file.

Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
---
 .../pci/controller/dwc/pcie-designware-host.c | 23 +++++++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h  |  1 +
 2 files changed, 24 insertions(+)

diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 5d90009a0f73..ce7071095006 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -382,6 +382,29 @@ int dw_pcie_host_init(struct pcie_port *pp)
 				pp->msi_irq[0] = irq;
 			}
 
+			if (pp->has_split_msi_irq) {
+				char irq_name[] = "msiXXX";
+				int irq;
+
+				for (ctrl = 1; ctrl < num_ctrls; ctrl++) {
+					if (pp->msi_irq[ctrl])
+						continue;
+
+					snprintf(irq_name, sizeof(irq_name), "msi%d", ctrl + 1);
+					irq = platform_get_irq_byname_optional(pdev, irq_name);
+					if (irq == -ENXIO) {
+						num_ctrls = ctrl;
+						pp->num_vectors = num_ctrls * MAX_MSI_IRQS_PER_CTRL;
+						dev_warn(dev, "Limiting amount of MSI irqs to %d\n", pp->num_vectors);
+						break;
+					}
+					if (irq < 0)
+						return irq;
+
+					pp->msi_irq[ctrl] = irq;
+				}
+			}
+
 			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
 
 			ret = dw_pcie_allocate_domains(pp);
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 9c1a38b0a6b3..3aa840a5b19c 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -179,6 +179,7 @@ struct dw_pcie_host_ops {
 
 struct pcie_port {
 	bool			has_msi_ctrl:1;
+	bool			has_split_msi_irq:1;
 	u64			cfg0_base;
 	void __iomem		*va_cfg0_base;
 	u32			cfg0_size;
-- 
2.35.1

[PATCH v3 3/5] PCI: qcom: Handle MSI IRQs properly

[Dmitry Baryshkov]

On Qualcomm platforms each group of 32 MSI vectors is routed to the
separate GIC interrupt. Thus to receive higher MSI vectors properly,
enable has_split_msi_irq support.

Note, that if DT doesn't list extra MSI interrupts, DWC core will limit
the amount of supported MSI vectors accordingly (to 32).

Fixes: 20f1bfb8dd62 ("PCI: qcom: Add support for handling MSIs from 8 endpoints")
Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
---
 drivers/pci/controller/dwc/pcie-qcom.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index 45631c0aa468..78c4e2bcf38a 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -1587,6 +1587,7 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	pci->ops = &dw_pcie_ops;
 	pp = &pci->pp;
 	pp->num_vectors = MAX_MSI_IRQS;
+	pp->has_split_msi_irq = true;
 
 	pcie->pci = pci;
 
-- 
2.35.1

[PATCH v3 4/5] dt-bindings: pci/qcom,pcie: support additional MSI interrupts

[Dmitry Baryshkov]

On Qualcomm platforms each group of 32 MSI vectors is routed to the
separate GIC interrupt. Document mapping of additional interrupts.

Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
---
 Documentation/devicetree/bindings/pci/qcom,pcie.yaml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
index 48d56b073564..8447076bef97 100644
--- a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
@@ -42,11 +42,21 @@ properties:
     maxItems: 5
 
   interrupts:
-    maxItems: 1
+    minItems: 1
+    maxItems: 8
 
   interrupt-names:
+    minItems: 1
+    maxItems: 8
     items:
       - const: msi
+      - const: msi2
+      - const: msi3
+      - const: msi4
+      - const: msi5
+      - const: msi6
+      - const: msi7
+      - const: msi8
 
   # Common definitions for clocks, clock-names and reset.
   # Platform constraints are described later.
-- 
2.35.1

Re: [PATCH v3 4/5] dt-bindings: pci/qcom,pcie: support additional MSI interrupts

[Krzysztof Kozlowski]

On 27/04/2022 14:16, Dmitry Baryshkov wrote:
> On Qualcomm platforms each group of 32 MSI vectors is routed to the
> separate GIC interrupt. Document mapping of additional interrupts.
> 
> Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
> ---
>  Documentation/devicetree/bindings/pci/qcom,pcie.yaml | 12 +++++++++++-
>  1 file changed, 11 insertions(+), 1 deletion(-)


Acked-by: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>


Best regards,
Krzysztof

[PATCH v3 5/5] arm64: dts: qcom: sm8250: provide additional MSI interrupts

[Dmitry Baryshkov]

On SM8250 each group of MSI interrupts is mapped to the separate host
interrupt. Describe each of interrupts in the device tree for PCIe0
host.

Tested on Qualcomm RB5 platform with first group of MSI interrupts being
used by the PME and attached ath11k WiFi chip using second group of MSI
interrupts.

Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
---
 arch/arm64/boot/dts/qcom/sm8250.dtsi | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
index 410272a1e19b..0659ac45c651 100644
--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
@@ -1807,8 +1807,15 @@ pcie0: pci@1c00000 {
 			ranges = <0x01000000 0x0 0x60200000 0 0x60200000 0x0 0x100000>,
 				 <0x02000000 0x0 0x60300000 0 0x60300000 0x0 0x3d00000>;
 
-			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "msi";
+			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi", "msi2", "msi3", "msi4", "msi5", "msi6", "msi7", "msi8";
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
 			interrupt-map = <0 0 0 1 &intc 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
-- 
2.35.1