[PATCH v10] PCI: tango: Add MSI controller support

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

The MSI controller in Tango supports 256 message-signaled interrupts
and a single doorbell address.

Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
---
Changes from v9 to v10
- Start from Bjorn's cleanup branch
- Clean up the MSI init and unused statements
- Based on top of v4.13-rc6

Hello Bjorn,

This patch is almost identical to the patch reviewed by Marc Zyngier
on June 14 (10 weeks ago). I'm not sure he could review this patch
again in time for 4.14 (given his work load these past few weeks).

The host bridge part landed in 4.13, but the driver is useless
without MSI support (legacy interrupts are not supported).
Can you take it for 4.14?

Regards.
---
 drivers/pci/host/pcie-tango.c | 191 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 189 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/host/pcie-tango.c b/drivers/pci/host/pcie-tango.c
index 6bbb81f06a53..d672271ad719 100644
--- a/drivers/pci/host/pcie-tango.c
+++ b/drivers/pci/host/pcie-tango.c
@@ -1,12 +1,170 @@
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
 #include <linux/pci-ecam.h>
 #include <linux/delay.h>
+#include <linux/msi.h>
 #include <linux/of.h>
 
+#define MSI_MAX			256
+
 #define SMP8759_MUX		0x48
 #define SMP8759_TEST_OUT	0x74
+#define SMP8759_STATUS		0x80
+#define SMP8759_ENABLE		0xa0
+#define SMP8759_DOORBELL	0xa002e07c
 
 struct tango_pcie {
-	void __iomem *base;
+	DECLARE_BITMAP(used_msi, MSI_MAX);
+	spinlock_t		used_msi_lock;
+	void __iomem		*base;
+	struct irq_domain	*dom;
+};
+
+static void tango_msi_isr(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct tango_pcie *pcie = irq_desc_get_handler_data(desc);
+	unsigned long status, base, virq, idx, pos = 0;
+
+	chained_irq_enter(chip, desc);
+	spin_lock(&pcie->used_msi_lock);
+
+	while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) {
+		base = round_down(pos, 32);
+		status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8);
+		for_each_set_bit(idx, &status, 32) {
+			virq = irq_find_mapping(pcie->dom, base + idx);
+			generic_handle_irq(virq);
+		}
+		pos = base + 32;
+	}
+
+	spin_unlock(&pcie->used_msi_lock);
+	chained_irq_exit(chip, desc);
+}
+
+static void tango_ack(struct irq_data *d)
+{
+	struct tango_pcie *pcie = d->chip_data;
+	u32 offset = (d->hwirq / 32) * 4;
+	u32 bit = BIT(d->hwirq % 32);
+
+	writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset);
+}
+
+static void update_msi_enable(struct irq_data *d, bool unmask)
+{
+	unsigned long flags;
+	struct tango_pcie *pcie = d->chip_data;
+	u32 offset = (d->hwirq / 32) * 4;
+	u32 bit = BIT(d->hwirq % 32);
+	u32 val;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset);
+	val = unmask ? val | bit : val & ~bit;
+	writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+}
+
+static void tango_mask(struct irq_data *d)
+{
+	update_msi_enable(d, false);
+}
+
+static void tango_unmask(struct irq_data *d)
+{
+	update_msi_enable(d, true);
+}
+
+static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask,
+			      bool force)
+{
+	return -EINVAL;
+}
+
+static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
+{
+	msg->address_lo = lower_32_bits(SMP8759_DOORBELL);
+	msg->address_hi = upper_32_bits(SMP8759_DOORBELL);
+	msg->data = d->hwirq;
+}
+
+static struct irq_chip tango_chip = {
+	.irq_ack		= tango_ack,
+	.irq_mask		= tango_mask,
+	.irq_unmask		= tango_unmask,
+	.irq_set_affinity	= tango_set_affinity,
+	.irq_compose_msi_msg	= tango_compose_msi_msg,
+};
+
+static void msi_ack(struct irq_data *d)
+{
+	irq_chip_ack_parent(d);
+}
+
+static void msi_mask(struct irq_data *d)
+{
+	pci_msi_mask_irq(d);
+	irq_chip_mask_parent(d);
+}
+
+static void msi_unmask(struct irq_data *d)
+{
+	pci_msi_unmask_irq(d);
+	irq_chip_unmask_parent(d);
+}
+
+static struct irq_chip msi_chip = {
+	.name = "MSI",
+	.irq_ack = msi_ack,
+	.irq_mask = msi_mask,
+	.irq_unmask = msi_unmask,
+};
+
+static struct msi_domain_info msi_dom_info = {
+	.flags	= MSI_FLAG_PCI_MSIX
+		| MSI_FLAG_USE_DEF_DOM_OPS
+		| MSI_FLAG_USE_DEF_CHIP_OPS,
+	.chip	= &msi_chip,
+};
+
+static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq,
+				  unsigned int nr_irqs, void *args)
+{
+	struct tango_pcie *pcie = dom->host_data;
+	unsigned long flags;
+	int pos;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	pos = find_first_zero_bit(pcie->used_msi, MSI_MAX);
+	if (pos >= MSI_MAX) {
+		spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+		return -ENOSPC;
+	}
+	__set_bit(pos, pcie->used_msi);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+	irq_domain_set_info(dom, virq, pos, &tango_chip,
+			pcie, handle_edge_irq, NULL, NULL);
+
+	return 0;
+}
+
+static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq,
+				  unsigned int nr_irqs)
+{
+	unsigned long flags;
+	struct irq_data *d = irq_domain_get_irq_data(dom, virq);
+	struct tango_pcie *pcie = d->chip_data;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	__clear_bit(d->hwirq, pcie->used_msi);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+}
+
+static const struct irq_domain_ops dom_ops = {
+	.alloc	= tango_irq_domain_alloc,
+	.free	= tango_irq_domain_free,
 };
 
 static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn,
@@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct tango_pcie *pcie;
 	struct resource *res;
-	int ret;
+	struct irq_domain *msi_dom, *irq_dom;
+	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
+	int ret, reg, virq;
 
 	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
@@ -95,6 +255,33 @@ static int tango_pcie_probe(struct platform_device *pdev)
 	if (!tango_pcie_link_up(pcie))
 		return -ENODEV;
 
+	for (reg = 0; reg < MSI_MAX / 8; reg += 4)
+		writel_relaxed(0, pcie->base + SMP8759_ENABLE + reg);
+
+	virq = platform_get_irq(pdev, 1);
+	if (virq <= 0) {
+		dev_err(dev, "Failed to map IRQ\n");
+		return -ENXIO;
+	}
+
+	irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie);
+	if (!irq_dom) {
+		dev_err(dev, "Failed to create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom);
+	if (!msi_dom) {
+		dev_err(dev, "Failed to create MSI domain\n");
+		irq_domain_remove(irq_dom);
+		return -ENOMEM;
+	}
+
+	pcie->dom = irq_dom;
+	spin_lock_init(&pcie->used_msi_lock);
+
+	irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie);
+
 	return pci_host_common_probe(pdev, &smp8759_ecam_ops);
 }
 
-- 
2.11.0


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

The MSI controller in Tango supports 256 message-signaled interrupts
and a single doorbell address.

Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
---
Changes from v9 to v10
- Start from Bjorn's cleanup branch
- Clean up the MSI init and unused statements
- Based on top of v4.13-rc6

Hello Bjorn,

This patch is almost identical to the patch reviewed by Marc Zyngier
on June 14 (10 weeks ago). I'm not sure he could review this patch
again in time for 4.14 (given his work load these past few weeks).

The host bridge part landed in 4.13, but the driver is useless
without MSI support (legacy interrupts are not supported).
Can you take it for 4.14?

Regards.
---
 drivers/pci/host/pcie-tango.c | 191 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 189 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/host/pcie-tango.c b/drivers/pci/host/pcie-tango.c
index 6bbb81f06a53..d672271ad719 100644
--- a/drivers/pci/host/pcie-tango.c
+++ b/drivers/pci/host/pcie-tango.c
@@ -1,12 +1,170 @@
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
 #include <linux/pci-ecam.h>
 #include <linux/delay.h>
+#include <linux/msi.h>
 #include <linux/of.h>
 
+#define MSI_MAX			256
+
 #define SMP8759_MUX		0x48
 #define SMP8759_TEST_OUT	0x74
+#define SMP8759_STATUS		0x80
+#define SMP8759_ENABLE		0xa0
+#define SMP8759_DOORBELL	0xa002e07c
 
 struct tango_pcie {
-	void __iomem *base;
+	DECLARE_BITMAP(used_msi, MSI_MAX);
+	spinlock_t		used_msi_lock;
+	void __iomem		*base;
+	struct irq_domain	*dom;
+};
+
+static void tango_msi_isr(struct irq_desc *desc)
+{
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct tango_pcie *pcie = irq_desc_get_handler_data(desc);
+	unsigned long status, base, virq, idx, pos = 0;
+
+	chained_irq_enter(chip, desc);
+	spin_lock(&pcie->used_msi_lock);
+
+	while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) {
+		base = round_down(pos, 32);
+		status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8);
+		for_each_set_bit(idx, &status, 32) {
+			virq = irq_find_mapping(pcie->dom, base + idx);
+			generic_handle_irq(virq);
+		}
+		pos = base + 32;
+	}
+
+	spin_unlock(&pcie->used_msi_lock);
+	chained_irq_exit(chip, desc);
+}
+
+static void tango_ack(struct irq_data *d)
+{
+	struct tango_pcie *pcie = d->chip_data;
+	u32 offset = (d->hwirq / 32) * 4;
+	u32 bit = BIT(d->hwirq % 32);
+
+	writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset);
+}
+
+static void update_msi_enable(struct irq_data *d, bool unmask)
+{
+	unsigned long flags;
+	struct tango_pcie *pcie = d->chip_data;
+	u32 offset = (d->hwirq / 32) * 4;
+	u32 bit = BIT(d->hwirq % 32);
+	u32 val;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset);
+	val = unmask ? val | bit : val & ~bit;
+	writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+}
+
+static void tango_mask(struct irq_data *d)
+{
+	update_msi_enable(d, false);
+}
+
+static void tango_unmask(struct irq_data *d)
+{
+	update_msi_enable(d, true);
+}
+
+static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask,
+			      bool force)
+{
+	return -EINVAL;
+}
+
+static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
+{
+	msg->address_lo = lower_32_bits(SMP8759_DOORBELL);
+	msg->address_hi = upper_32_bits(SMP8759_DOORBELL);
+	msg->data = d->hwirq;
+}
+
+static struct irq_chip tango_chip = {
+	.irq_ack		= tango_ack,
+	.irq_mask		= tango_mask,
+	.irq_unmask		= tango_unmask,
+	.irq_set_affinity	= tango_set_affinity,
+	.irq_compose_msi_msg	= tango_compose_msi_msg,
+};
+
+static void msi_ack(struct irq_data *d)
+{
+	irq_chip_ack_parent(d);
+}
+
+static void msi_mask(struct irq_data *d)
+{
+	pci_msi_mask_irq(d);
+	irq_chip_mask_parent(d);
+}
+
+static void msi_unmask(struct irq_data *d)
+{
+	pci_msi_unmask_irq(d);
+	irq_chip_unmask_parent(d);
+}
+
+static struct irq_chip msi_chip = {
+	.name = "MSI",
+	.irq_ack = msi_ack,
+	.irq_mask = msi_mask,
+	.irq_unmask = msi_unmask,
+};
+
+static struct msi_domain_info msi_dom_info = {
+	.flags	= MSI_FLAG_PCI_MSIX
+		| MSI_FLAG_USE_DEF_DOM_OPS
+		| MSI_FLAG_USE_DEF_CHIP_OPS,
+	.chip	= &msi_chip,
+};
+
+static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq,
+				  unsigned int nr_irqs, void *args)
+{
+	struct tango_pcie *pcie = dom->host_data;
+	unsigned long flags;
+	int pos;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	pos = find_first_zero_bit(pcie->used_msi, MSI_MAX);
+	if (pos >= MSI_MAX) {
+		spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+		return -ENOSPC;
+	}
+	__set_bit(pos, pcie->used_msi);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+	irq_domain_set_info(dom, virq, pos, &tango_chip,
+			pcie, handle_edge_irq, NULL, NULL);
+
+	return 0;
+}
+
+static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq,
+				  unsigned int nr_irqs)
+{
+	unsigned long flags;
+	struct irq_data *d = irq_domain_get_irq_data(dom, virq);
+	struct tango_pcie *pcie = d->chip_data;
+
+	spin_lock_irqsave(&pcie->used_msi_lock, flags);
+	__clear_bit(d->hwirq, pcie->used_msi);
+	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
+}
+
+static const struct irq_domain_ops dom_ops = {
+	.alloc	= tango_irq_domain_alloc,
+	.free	= tango_irq_domain_free,
 };
 
 static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn,
@@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct tango_pcie *pcie;
 	struct resource *res;
-	int ret;
+	struct irq_domain *msi_dom, *irq_dom;
+	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
+	int ret, reg, virq;
 
 	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
 	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
@@ -95,6 +255,33 @@ static int tango_pcie_probe(struct platform_device *pdev)
 	if (!tango_pcie_link_up(pcie))
 		return -ENODEV;
 
+	for (reg = 0; reg < MSI_MAX / 8; reg += 4)
+		writel_relaxed(0, pcie->base + SMP8759_ENABLE + reg);
+
+	virq = platform_get_irq(pdev, 1);
+	if (virq <= 0) {
+		dev_err(dev, "Failed to map IRQ\n");
+		return -ENXIO;
+	}
+
+	irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie);
+	if (!irq_dom) {
+		dev_err(dev, "Failed to create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom);
+	if (!msi_dom) {
+		dev_err(dev, "Failed to create MSI domain\n");
+		irq_domain_remove(irq_dom);
+		return -ENOMEM;
+	}
+
+	pcie->dom = irq_dom;
+	spin_lock_init(&pcie->used_msi_lock);
+
+	irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie);
+
 	return pci_host_common_probe(pdev, &smp8759_ecam_ops);
 }
 
-- 
2.11.0

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On 22/08/17 15:56, Marc Gonzalez wrote:
> The MSI controller in Tango supports 256 message-signaled interrupts
> and a single doorbell address.
> 
> Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
> ---
> Changes from v9 to v10
> - Start from Bjorn's cleanup branch
> - Clean up the MSI init and unused statements
> - Based on top of v4.13-rc6
> 
> Hello Bjorn,
> 
> This patch is almost identical to the patch reviewed by Marc Zyngier
> on June 14 (10 weeks ago). I'm not sure he could review this patch
> again in time for 4.14 (given his work load these past few weeks).

Thanks for worrying about my workload.

> 
> The host bridge part landed in 4.13, but the driver is useless
> without MSI support (legacy interrupts are not supported).
> Can you take it for 4.14?
> 
> Regards.
> ---
>  drivers/pci/host/pcie-tango.c | 191 +++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 189 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/pci/host/pcie-tango.c b/drivers/pci/host/pcie-tango.c
> index 6bbb81f06a53..d672271ad719 100644
> --- a/drivers/pci/host/pcie-tango.c
> +++ b/drivers/pci/host/pcie-tango.c
> @@ -1,12 +1,170 @@
> +#include <linux/irqchip/chained_irq.h>
> +#include <linux/irqdomain.h>
>  #include <linux/pci-ecam.h>
>  #include <linux/delay.h>
> +#include <linux/msi.h>
>  #include <linux/of.h>
>  
> +#define MSI_MAX			256
> +
>  #define SMP8759_MUX		0x48
>  #define SMP8759_TEST_OUT	0x74
> +#define SMP8759_STATUS		0x80
> +#define SMP8759_ENABLE		0xa0
> +#define SMP8759_DOORBELL	0xa002e07c

Why is this hardcoded and not coming from the device-tree, just like any
other address property?

>  
>  struct tango_pcie {
> -	void __iomem *base;
> +	DECLARE_BITMAP(used_msi, MSI_MAX);
> +	spinlock_t		used_msi_lock;
> +	void __iomem		*base;
> +	struct irq_domain	*dom;
> +};
> +
> +static void tango_msi_isr(struct irq_desc *desc)
> +{
> +	struct irq_chip *chip = irq_desc_get_chip(desc);
> +	struct tango_pcie *pcie = irq_desc_get_handler_data(desc);
> +	unsigned long status, base, virq, idx, pos = 0;
> +
> +	chained_irq_enter(chip, desc);
> +	spin_lock(&pcie->used_msi_lock);
> +
> +	while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) {
> +		base = round_down(pos, 32);
> +		status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8);
> +		for_each_set_bit(idx, &status, 32) {
> +			virq = irq_find_mapping(pcie->dom, base + idx);
> +			generic_handle_irq(virq);
> +		}
> +		pos = base + 32;
> +	}
> +
> +	spin_unlock(&pcie->used_msi_lock);
> +	chained_irq_exit(chip, desc);
> +}
> +
> +static void tango_ack(struct irq_data *d)
> +{
> +	struct tango_pcie *pcie = d->chip_data;
> +	u32 offset = (d->hwirq / 32) * 4;
> +	u32 bit = BIT(d->hwirq % 32);
> +
> +	writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset);
> +}
> +
> +static void update_msi_enable(struct irq_data *d, bool unmask)
> +{
> +	unsigned long flags;
> +	struct tango_pcie *pcie = d->chip_data;
> +	u32 offset = (d->hwirq / 32) * 4;
> +	u32 bit = BIT(d->hwirq % 32);
> +	u32 val;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset);
> +	val = unmask ? val | bit : val & ~bit;
> +	writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +}
> +
> +static void tango_mask(struct irq_data *d)
> +{
> +	update_msi_enable(d, false);
> +}
> +
> +static void tango_unmask(struct irq_data *d)
> +{
> +	update_msi_enable(d, true);
> +}
> +
> +static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask,
> +			      bool force)
> +{
> +	return -EINVAL;
> +}
> +
> +static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
> +{
> +	msg->address_lo = lower_32_bits(SMP8759_DOORBELL);
> +	msg->address_hi = upper_32_bits(SMP8759_DOORBELL);
> +	msg->data = d->hwirq;
> +}
> +
> +static struct irq_chip tango_chip = {
> +	.irq_ack		= tango_ack,
> +	.irq_mask		= tango_mask,
> +	.irq_unmask		= tango_unmask,
> +	.irq_set_affinity	= tango_set_affinity,
> +	.irq_compose_msi_msg	= tango_compose_msi_msg,
> +};
> +
> +static void msi_ack(struct irq_data *d)
> +{
> +	irq_chip_ack_parent(d);
> +}
> +
> +static void msi_mask(struct irq_data *d)
> +{
> +	pci_msi_mask_irq(d);
> +	irq_chip_mask_parent(d);
> +}
> +
> +static void msi_unmask(struct irq_data *d)
> +{
> +	pci_msi_unmask_irq(d);
> +	irq_chip_unmask_parent(d);
> +}
> +
> +static struct irq_chip msi_chip = {
> +	.name = "MSI",
> +	.irq_ack = msi_ack,
> +	.irq_mask = msi_mask,
> +	.irq_unmask = msi_unmask,
> +};
> +
> +static struct msi_domain_info msi_dom_info = {
> +	.flags	= MSI_FLAG_PCI_MSIX
> +		| MSI_FLAG_USE_DEF_DOM_OPS
> +		| MSI_FLAG_USE_DEF_CHIP_OPS,
> +	.chip	= &msi_chip,
> +};
> +
> +static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq,
> +				  unsigned int nr_irqs, void *args)
> +{
> +	struct tango_pcie *pcie = dom->host_data;
> +	unsigned long flags;
> +	int pos;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	pos = find_first_zero_bit(pcie->used_msi, MSI_MAX);
> +	if (pos >= MSI_MAX) {
> +		spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +		return -ENOSPC;
> +	}
> +	__set_bit(pos, pcie->used_msi);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +	irq_domain_set_info(dom, virq, pos, &tango_chip,
> +			pcie, handle_edge_irq, NULL, NULL);
> +
> +	return 0;
> +}
> +
> +static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq,
> +				  unsigned int nr_irqs)
> +{
> +	unsigned long flags;
> +	struct irq_data *d = irq_domain_get_irq_data(dom, virq);
> +	struct tango_pcie *pcie = d->chip_data;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	__clear_bit(d->hwirq, pcie->used_msi);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +}
> +
> +static const struct irq_domain_ops dom_ops = {
> +	.alloc	= tango_irq_domain_alloc,
> +	.free	= tango_irq_domain_free,
>  };
>  
>  static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn,
> @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
>  	struct device *dev = &pdev->dev;
>  	struct tango_pcie *pcie;
>  	struct resource *res;
> -	int ret;
> +	struct irq_domain *msi_dom, *irq_dom;
> +	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> +	int ret, reg, virq;
>  
>  	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
>  	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> @@ -95,6 +255,33 @@ static int tango_pcie_probe(struct platform_device *pdev)
>  	if (!tango_pcie_link_up(pcie))
>  		return -ENODEV;
>  
> +	for (reg = 0; reg < MSI_MAX / 8; reg += 4)
> +		writel_relaxed(0, pcie->base + SMP8759_ENABLE + reg);
> +
> +	virq = platform_get_irq(pdev, 1);
> +	if (virq <= 0) {
> +		dev_err(dev, "Failed to map IRQ\n");
> +		return -ENXIO;
> +	}
> +
> +	irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie);
> +	if (!irq_dom) {
> +		dev_err(dev, "Failed to create IRQ domain\n");
> +		return -ENOMEM;
> +	}
> +
> +	msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom);
> +	if (!msi_dom) {
> +		dev_err(dev, "Failed to create MSI domain\n");
> +		irq_domain_remove(irq_dom);
> +		return -ENOMEM;
> +	}
> +
> +	pcie->dom = irq_dom;
> +	spin_lock_init(&pcie->used_msi_lock);
> +
> +	irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie);
> +
>  	return pci_host_common_probe(pdev, &smp8759_ecam_ops);
>  }
>  
> 

The remark above notwithstanding:

Acked-by: Marc Zyngier <marc.zyngier@arm.com>

	M.
-- 
Jazz is not dead. It just smells funny...

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On 22/08/17 15:56, Marc Gonzalez wrote:
> The MSI controller in Tango supports 256 message-signaled interrupts
> and a single doorbell address.
> 
> Signed-off-by: Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
> ---
> Changes from v9 to v10
> - Start from Bjorn's cleanup branch
> - Clean up the MSI init and unused statements
> - Based on top of v4.13-rc6
> 
> Hello Bjorn,
> 
> This patch is almost identical to the patch reviewed by Marc Zyngier
> on June 14 (10 weeks ago). I'm not sure he could review this patch
> again in time for 4.14 (given his work load these past few weeks).

Thanks for worrying about my workload.

> 
> The host bridge part landed in 4.13, but the driver is useless
> without MSI support (legacy interrupts are not supported).
> Can you take it for 4.14?
> 
> Regards.
> ---
>  drivers/pci/host/pcie-tango.c | 191 +++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 189 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/pci/host/pcie-tango.c b/drivers/pci/host/pcie-tango.c
> index 6bbb81f06a53..d672271ad719 100644
> --- a/drivers/pci/host/pcie-tango.c
> +++ b/drivers/pci/host/pcie-tango.c
> @@ -1,12 +1,170 @@
> +#include <linux/irqchip/chained_irq.h>
> +#include <linux/irqdomain.h>
>  #include <linux/pci-ecam.h>
>  #include <linux/delay.h>
> +#include <linux/msi.h>
>  #include <linux/of.h>
>  
> +#define MSI_MAX			256
> +
>  #define SMP8759_MUX		0x48
>  #define SMP8759_TEST_OUT	0x74
> +#define SMP8759_STATUS		0x80
> +#define SMP8759_ENABLE		0xa0
> +#define SMP8759_DOORBELL	0xa002e07c

Why is this hardcoded and not coming from the device-tree, just like any
other address property?

>  
>  struct tango_pcie {
> -	void __iomem *base;
> +	DECLARE_BITMAP(used_msi, MSI_MAX);
> +	spinlock_t		used_msi_lock;
> +	void __iomem		*base;
> +	struct irq_domain	*dom;
> +};
> +
> +static void tango_msi_isr(struct irq_desc *desc)
> +{
> +	struct irq_chip *chip = irq_desc_get_chip(desc);
> +	struct tango_pcie *pcie = irq_desc_get_handler_data(desc);
> +	unsigned long status, base, virq, idx, pos = 0;
> +
> +	chained_irq_enter(chip, desc);
> +	spin_lock(&pcie->used_msi_lock);
> +
> +	while ((pos = find_next_bit(pcie->used_msi, MSI_MAX, pos)) < MSI_MAX) {
> +		base = round_down(pos, 32);
> +		status = readl_relaxed(pcie->base + SMP8759_STATUS + base / 8);
> +		for_each_set_bit(idx, &status, 32) {
> +			virq = irq_find_mapping(pcie->dom, base + idx);
> +			generic_handle_irq(virq);
> +		}
> +		pos = base + 32;
> +	}
> +
> +	spin_unlock(&pcie->used_msi_lock);
> +	chained_irq_exit(chip, desc);
> +}
> +
> +static void tango_ack(struct irq_data *d)
> +{
> +	struct tango_pcie *pcie = d->chip_data;
> +	u32 offset = (d->hwirq / 32) * 4;
> +	u32 bit = BIT(d->hwirq % 32);
> +
> +	writel_relaxed(bit, pcie->base + SMP8759_STATUS + offset);
> +}
> +
> +static void update_msi_enable(struct irq_data *d, bool unmask)
> +{
> +	unsigned long flags;
> +	struct tango_pcie *pcie = d->chip_data;
> +	u32 offset = (d->hwirq / 32) * 4;
> +	u32 bit = BIT(d->hwirq % 32);
> +	u32 val;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	val = readl_relaxed(pcie->base + SMP8759_ENABLE + offset);
> +	val = unmask ? val | bit : val & ~bit;
> +	writel_relaxed(val, pcie->base + SMP8759_ENABLE + offset);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +}
> +
> +static void tango_mask(struct irq_data *d)
> +{
> +	update_msi_enable(d, false);
> +}
> +
> +static void tango_unmask(struct irq_data *d)
> +{
> +	update_msi_enable(d, true);
> +}
> +
> +static int tango_set_affinity(struct irq_data *d, const struct cpumask *mask,
> +			      bool force)
> +{
> +	return -EINVAL;
> +}
> +
> +static void tango_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
> +{
> +	msg->address_lo = lower_32_bits(SMP8759_DOORBELL);
> +	msg->address_hi = upper_32_bits(SMP8759_DOORBELL);
> +	msg->data = d->hwirq;
> +}
> +
> +static struct irq_chip tango_chip = {
> +	.irq_ack		= tango_ack,
> +	.irq_mask		= tango_mask,
> +	.irq_unmask		= tango_unmask,
> +	.irq_set_affinity	= tango_set_affinity,
> +	.irq_compose_msi_msg	= tango_compose_msi_msg,
> +};
> +
> +static void msi_ack(struct irq_data *d)
> +{
> +	irq_chip_ack_parent(d);
> +}
> +
> +static void msi_mask(struct irq_data *d)
> +{
> +	pci_msi_mask_irq(d);
> +	irq_chip_mask_parent(d);
> +}
> +
> +static void msi_unmask(struct irq_data *d)
> +{
> +	pci_msi_unmask_irq(d);
> +	irq_chip_unmask_parent(d);
> +}
> +
> +static struct irq_chip msi_chip = {
> +	.name = "MSI",
> +	.irq_ack = msi_ack,
> +	.irq_mask = msi_mask,
> +	.irq_unmask = msi_unmask,
> +};
> +
> +static struct msi_domain_info msi_dom_info = {
> +	.flags	= MSI_FLAG_PCI_MSIX
> +		| MSI_FLAG_USE_DEF_DOM_OPS
> +		| MSI_FLAG_USE_DEF_CHIP_OPS,
> +	.chip	= &msi_chip,
> +};
> +
> +static int tango_irq_domain_alloc(struct irq_domain *dom, unsigned int virq,
> +				  unsigned int nr_irqs, void *args)
> +{
> +	struct tango_pcie *pcie = dom->host_data;
> +	unsigned long flags;
> +	int pos;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	pos = find_first_zero_bit(pcie->used_msi, MSI_MAX);
> +	if (pos >= MSI_MAX) {
> +		spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +		return -ENOSPC;
> +	}
> +	__set_bit(pos, pcie->used_msi);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +	irq_domain_set_info(dom, virq, pos, &tango_chip,
> +			pcie, handle_edge_irq, NULL, NULL);
> +
> +	return 0;
> +}
> +
> +static void tango_irq_domain_free(struct irq_domain *dom, unsigned int virq,
> +				  unsigned int nr_irqs)
> +{
> +	unsigned long flags;
> +	struct irq_data *d = irq_domain_get_irq_data(dom, virq);
> +	struct tango_pcie *pcie = d->chip_data;
> +
> +	spin_lock_irqsave(&pcie->used_msi_lock, flags);
> +	__clear_bit(d->hwirq, pcie->used_msi);
> +	spin_unlock_irqrestore(&pcie->used_msi_lock, flags);
> +}
> +
> +static const struct irq_domain_ops dom_ops = {
> +	.alloc	= tango_irq_domain_alloc,
> +	.free	= tango_irq_domain_free,
>  };
>  
>  static int smp8759_config_read(struct pci_bus *bus, unsigned int devfn,
> @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
>  	struct device *dev = &pdev->dev;
>  	struct tango_pcie *pcie;
>  	struct resource *res;
> -	int ret;
> +	struct irq_domain *msi_dom, *irq_dom;
> +	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> +	int ret, reg, virq;
>  
>  	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
>  	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> @@ -95,6 +255,33 @@ static int tango_pcie_probe(struct platform_device *pdev)
>  	if (!tango_pcie_link_up(pcie))
>  		return -ENODEV;
>  
> +	for (reg = 0; reg < MSI_MAX / 8; reg += 4)
> +		writel_relaxed(0, pcie->base + SMP8759_ENABLE + reg);
> +
> +	virq = platform_get_irq(pdev, 1);
> +	if (virq <= 0) {
> +		dev_err(dev, "Failed to map IRQ\n");
> +		return -ENXIO;
> +	}
> +
> +	irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie);
> +	if (!irq_dom) {
> +		dev_err(dev, "Failed to create IRQ domain\n");
> +		return -ENOMEM;
> +	}
> +
> +	msi_dom = pci_msi_create_irq_domain(fwnode, &msi_dom_info, irq_dom);
> +	if (!msi_dom) {
> +		dev_err(dev, "Failed to create MSI domain\n");
> +		irq_domain_remove(irq_dom);
> +		return -ENOMEM;
> +	}
> +
> +	pcie->dom = irq_dom;
> +	spin_lock_init(&pcie->used_msi_lock);
> +
> +	irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie);
> +
>  	return pci_host_common_probe(pdev, &smp8759_ecam_ops);
>  }
>  
> 

The remark above notwithstanding:

Acked-by: Marc Zyngier <marc.zyngier@arm.com>

	M.
-- 
Jazz is not dead. It just smells funny...

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 22/08/2017 18:29, Marc Zyngier wrote:

> On 22/08/17 15:56, Marc Gonzalez wrote:
>
>>  #define SMP8759_MUX=09=090x48
>>  #define SMP8759_TEST_OUT=090x74
>> +#define SMP8759_STATUS=09=090x80
>> +#define SMP8759_ENABLE=09=090xa0
>> +#define SMP8759_DOORBELL=090xa002e07c
>=20
> Why is this hardcoded and not coming from the device-tree, just like any
> other address property?

Since this bus address is software-configurable, I didn't think
it belonged in the DT. Also, I didn't see anything similar in
other binding docs, especially

Documentation/devicetree/bindings/interrupt-controller/msi.txt

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 22/08/2017 18:29, Marc Zyngier wrote:

> On 22/08/17 15:56, Marc Gonzalez wrote:
>
>>  #define SMP8759_MUX		0x48
>>  #define SMP8759_TEST_OUT	0x74
>> +#define SMP8759_STATUS		0x80
>> +#define SMP8759_ENABLE		0xa0
>> +#define SMP8759_DOORBELL	0xa002e07c
> 
> Why is this hardcoded and not coming from the device-tree, just like any
> other address property?

Since this bus address is software-configurable, I didn't think
it belonged in the DT. Also, I didn't see anything similar in
other binding docs, especially

Documentation/devicetree/bindings/interrupt-controller/msi.txt

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Tue, Aug 22 2017 at  8:02:18 pm BST, Marc Gonzalez <marc_gonzalez@sigmadesigns.com> wrote:
> On 22/08/2017 18:29, Marc Zyngier wrote:
>
>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>
>>>  #define SMP8759_MUX		0x48
>>>  #define SMP8759_TEST_OUT	0x74
>>> +#define SMP8759_STATUS		0x80
>>> +#define SMP8759_ENABLE		0xa0
>>> +#define SMP8759_DOORBELL	0xa002e07c
>> 
>> Why is this hardcoded and not coming from the device-tree, just like any
>> other address property?
>
> Since this bus address is software-configurable, I didn't think
> it belonged in the DT. Also, I didn't see anything similar in
> other binding docs, especially
>
> Documentation/devicetree/bindings/interrupt-controller/msi.txt

If that's software configurable, how on Earth did you pick the address?
How do you ensure that it doesn't conflict with DMA? How is it
configured into the RC?

	M.
-- 
Jazz is not dead. It just smells funny.

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Tue, Aug 22 2017 at  8:02:18 pm BST, Marc Gonzalez <marc_gonzalez@sigmadesigns.com> wrote:
> On 22/08/2017 18:29, Marc Zyngier wrote:
>
>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>
>>>  #define SMP8759_MUX		0x48
>>>  #define SMP8759_TEST_OUT	0x74
>>> +#define SMP8759_STATUS		0x80
>>> +#define SMP8759_ENABLE		0xa0
>>> +#define SMP8759_DOORBELL	0xa002e07c
>> 
>> Why is this hardcoded and not coming from the device-tree, just like any
>> other address property?
>
> Since this bus address is software-configurable, I didn't think
> it belonged in the DT. Also, I didn't see anything similar in
> other binding docs, especially
>
> Documentation/devicetree/bindings/interrupt-controller/msi.txt

If that's software configurable, how on Earth did you pick the address?
How do you ensure that it doesn't conflict with DMA? How is it
configured into the RC?

	M.
-- 
Jazz is not dead. It just smells funny.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 22/08/2017 16:56, Marc Gonzalez wrote:

> @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pd=
ev)
>  =09struct device *dev =3D &pdev->dev;
>  =09struct tango_pcie *pcie;
>  =09struct resource *res;
> -=09int ret;
> +=09struct irq_domain *msi_dom, *irq_dom;
> +=09struct fwnode_handle *fwnode =3D of_node_to_fwnode(dev->of_node);
> +=09int ret, reg, virq;
> =20
>  =09dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause da=
ta corruption\n");
>  =09add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);

  CC      drivers/pci/host/pcie-tango.o
drivers/pci/host/pcie-tango.c: In function 'tango_pcie_probe':
drivers/pci/host/pcie-tango.c:257:6: warning: unused variable 'ret' [-Wunus=
ed-variable]
  int ret, reg, virq;
      ^~~

Hmmm, dunno how I managed to miss that...
Is the kbuild test robot enjoying a well-deserved vacation?

Bjorn, if/when you take the patch, can you first apply this fixup:

@@ -254,7 +254,7 @@ static int tango_pcie_probe(struct platform_device *pde=
v)
        struct resource *res;
        struct irq_domain *msi_dom, *irq_dom;
        struct fwnode_handle *fwnode =3D of_node_to_fwnode(dev->of_node);
-       int ret, reg, virq;
+       int reg, virq;
=20
        dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause =
data corruption\n");
        add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);


Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 22/08/2017 16:56, Marc Gonzalez wrote:

> @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
>  	struct device *dev = &pdev->dev;
>  	struct tango_pcie *pcie;
>  	struct resource *res;
> -	int ret;
> +	struct irq_domain *msi_dom, *irq_dom;
> +	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> +	int ret, reg, virq;
>  
>  	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
>  	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);

  CC      drivers/pci/host/pcie-tango.o
drivers/pci/host/pcie-tango.c: In function 'tango_pcie_probe':
drivers/pci/host/pcie-tango.c:257:6: warning: unused variable 'ret' [-Wunused-variable]
  int ret, reg, virq;
      ^~~

Hmmm, dunno how I managed to miss that...
Is the kbuild test robot enjoying a well-deserved vacation?

Bjorn, if/when you take the patch, can you first apply this fixup:

@@ -254,7 +254,7 @@ static int tango_pcie_probe(struct platform_device *pdev)
        struct resource *res;
        struct irq_domain *msi_dom, *irq_dom;
        struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
-       int ret, reg, virq;
+       int reg, virq;
 
        dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
        add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);


Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Bjorn Helgaas]

On Wed, Aug 23, 2017 at 02:59:42PM +0200, Marc Gonzalez wrote:
> On 22/08/2017 16:56, Marc Gonzalez wrote:
> 
> > @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
> >  	struct device *dev = &pdev->dev;
> >  	struct tango_pcie *pcie;
> >  	struct resource *res;
> > -	int ret;
> > +	struct irq_domain *msi_dom, *irq_dom;
> > +	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> > +	int ret, reg, virq;
> >  
> >  	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
> >  	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> 
>   CC      drivers/pci/host/pcie-tango.o
> drivers/pci/host/pcie-tango.c: In function 'tango_pcie_probe':
> drivers/pci/host/pcie-tango.c:257:6: warning: unused variable 'ret' [-Wunused-variable]
>   int ret, reg, virq;
>       ^~~
> 
> Hmmm, dunno how I managed to miss that...
> Is the kbuild test robot enjoying a well-deserved vacation?
> 
> Bjorn, if/when you take the patch, can you first apply this fixup:

Sure, no problem.

> @@ -254,7 +254,7 @@ static int tango_pcie_probe(struct platform_device *pdev)
>         struct resource *res;
>         struct irq_domain *msi_dom, *irq_dom;
>         struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> -       int ret, reg, virq;
> +       int reg, virq;
>  
>         dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
>         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> 
> 
> Regards.
> 

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v10] PCI: tango: Add MSI controller support

[Bjorn Helgaas]

On Wed, Aug 23, 2017 at 02:59:42PM +0200, Marc Gonzalez wrote:
> On 22/08/2017 16:56, Marc Gonzalez wrote:
> 
> > @@ -76,7 +234,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
> >  	struct device *dev = &pdev->dev;
> >  	struct tango_pcie *pcie;
> >  	struct resource *res;
> > -	int ret;
> > +	struct irq_domain *msi_dom, *irq_dom;
> > +	struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> > +	int ret, reg, virq;
> >  
> >  	dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
> >  	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> 
>   CC      drivers/pci/host/pcie-tango.o
> drivers/pci/host/pcie-tango.c: In function 'tango_pcie_probe':
> drivers/pci/host/pcie-tango.c:257:6: warning: unused variable 'ret' [-Wunused-variable]
>   int ret, reg, virq;
>       ^~~
> 
> Hmmm, dunno how I managed to miss that...
> Is the kbuild test robot enjoying a well-deserved vacation?
> 
> Bjorn, if/when you take the patch, can you first apply this fixup:

Sure, no problem.

> @@ -254,7 +254,7 @@ static int tango_pcie_probe(struct platform_device *pdev)
>         struct resource *res;
>         struct irq_domain *msi_dom, *irq_dom;
>         struct fwnode_handle *fwnode = of_node_to_fwnode(dev->of_node);
> -       int ret, reg, virq;
> +       int reg, virq;
>  
>         dev_warn(dev, "simultaneous PCI config and MMIO accesses may cause data corruption\n");
>         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
> 
> 
> Regards.
> 

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Bjorn Helgaas]

On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
> On Tue, Aug 22 2017 at  8:02:18 pm BST, Marc Gonzalez <marc_gonzalez@sigmadesigns.com> wrote:
> > On 22/08/2017 18:29, Marc Zyngier wrote:
> >
> >> On 22/08/17 15:56, Marc Gonzalez wrote:
> >>
> >>>  #define SMP8759_MUX		0x48
> >>>  #define SMP8759_TEST_OUT	0x74
> >>> +#define SMP8759_STATUS		0x80
> >>> +#define SMP8759_ENABLE		0xa0
> >>> +#define SMP8759_DOORBELL	0xa002e07c
> >> 
> >> Why is this hardcoded and not coming from the device-tree, just like any
> >> other address property?
> >
> > Since this bus address is software-configurable, I didn't think
> > it belonged in the DT. Also, I didn't see anything similar in
> > other binding docs, especially
> >
> > Documentation/devicetree/bindings/interrupt-controller/msi.txt
> 
> If that's software configurable, how on Earth did you pick the address?
> How do you ensure that it doesn't conflict with DMA? How is it
> configured into the RC?

But we *do* need to resolve this.  This does seem like an address that
shouldn't be hard-coded into the driver.  Since this driver is
programming the address into an MSI message, but not into the receiver
of that message, there's a coordination issue between this driver and
whatever other software does that receiver configuration.

Bjorn

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v10] PCI: tango: Add MSI controller support

[Bjorn Helgaas]

On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
> On Tue, Aug 22 2017 at  8:02:18 pm BST, Marc Gonzalez <marc_gonzalez@sigmadesigns.com> wrote:
> > On 22/08/2017 18:29, Marc Zyngier wrote:
> >
> >> On 22/08/17 15:56, Marc Gonzalez wrote:
> >>
> >>>  #define SMP8759_MUX		0x48
> >>>  #define SMP8759_TEST_OUT	0x74
> >>> +#define SMP8759_STATUS		0x80
> >>> +#define SMP8759_ENABLE		0xa0
> >>> +#define SMP8759_DOORBELL	0xa002e07c
> >> 
> >> Why is this hardcoded and not coming from the device-tree, just like any
> >> other address property?
> >
> > Since this bus address is software-configurable, I didn't think
> > it belonged in the DT. Also, I didn't see anything similar in
> > other binding docs, especially
> >
> > Documentation/devicetree/bindings/interrupt-controller/msi.txt
> 
> If that's software configurable, how on Earth did you pick the address?
> How do you ensure that it doesn't conflict with DMA? How is it
> configured into the RC?

But we *do* need to resolve this.  This does seem like an address that
shouldn't be hard-coded into the driver.  Since this driver is
programming the address into an MSI message, but not into the receiver
of that message, there's a coordination issue between this driver and
whatever other software does that receiver configuration.

Bjorn

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 24/08/2017 19:04, Bjorn Helgaas wrote:
> On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
>> Marc Gonzalez wrote:
>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>
>>>>>  #define SMP8759_MUX		0x48
>>>>>  #define SMP8759_TEST_OUT	0x74
>>>>> +#define SMP8759_STATUS		0x80
>>>>> +#define SMP8759_ENABLE		0xa0
>>>>> +#define SMP8759_DOORBELL	0xa002e07c
>>>>
>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>> other address property?
>>>
>>> Since this bus address is software-configurable, I didn't think
>>> it belonged in the DT. Also, I didn't see anything similar in
>>> other binding docs, especially
>>>
>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>
>> If that's software configurable, how on Earth did you pick the address?
>> How do you ensure that it doesn't conflict with DMA? How is it
>> configured into the RC?
> 
> But we *do* need to resolve this.  This does seem like an address that
> shouldn't be hard-coded into the driver.  Since this driver is
> programming the address into an MSI message, but not into the receiver
> of that message, there's a coordination issue between this driver and
> whatever other software does that receiver configuration.

OK. I'll move the doorbell address to the DT for v11.

What property should be used for this address?

sigma,doorbell ?

Or maybe I can put it in reg, since I have a 1:1 mapping
between bus and cpu addresses?

git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
returns nothing.

Regards.


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Gonzalez]

On 24/08/2017 19:04, Bjorn Helgaas wrote:
> On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
>> Marc Gonzalez wrote:
>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>
>>>>>  #define SMP8759_MUX		0x48
>>>>>  #define SMP8759_TEST_OUT	0x74
>>>>> +#define SMP8759_STATUS		0x80
>>>>> +#define SMP8759_ENABLE		0xa0
>>>>> +#define SMP8759_DOORBELL	0xa002e07c
>>>>
>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>> other address property?
>>>
>>> Since this bus address is software-configurable, I didn't think
>>> it belonged in the DT. Also, I didn't see anything similar in
>>> other binding docs, especially
>>>
>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>
>> If that's software configurable, how on Earth did you pick the address?
>> How do you ensure that it doesn't conflict with DMA? How is it
>> configured into the RC?
> 
> But we *do* need to resolve this.  This does seem like an address that
> shouldn't be hard-coded into the driver.  Since this driver is
> programming the address into an MSI message, but not into the receiver
> of that message, there's a coordination issue between this driver and
> whatever other software does that receiver configuration.

OK. I'll move the doorbell address to the DT for v11.

What property should be used for this address?

sigma,doorbell ?

Or maybe I can put it in reg, since I have a 1:1 mapping
between bus and cpu addresses?

git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
returns nothing.

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Ard Biesheuvel]

On 24 August 2017 at 18:51, Marc Gonzalez
<marc_gonzalez@sigmadesigns.com> wrote:
> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>> On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
>>> Marc Gonzalez wrote:
>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>
>>>>>>  #define SMP8759_MUX              0x48
>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>> +#define SMP8759_STATUS           0x80
>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>
>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>> other address property?
>>>>
>>>> Since this bus address is software-configurable, I didn't think
>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>> other binding docs, especially
>>>>
>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>
>>> If that's software configurable, how on Earth did you pick the address?
>>> How do you ensure that it doesn't conflict with DMA? How is it
>>> configured into the RC?
>>
>> But we *do* need to resolve this.  This does seem like an address that
>> shouldn't be hard-coded into the driver.  Since this driver is
>> programming the address into an MSI message, but not into the receiver
>> of that message, there's a coordination issue between this driver and
>> whatever other software does that receiver configuration.
>
> OK. I'll move the doorbell address to the DT for v11.
>
> What property should be used for this address?
>
> sigma,doorbell ?
>
> Or maybe I can put it in reg, since I have a 1:1 mapping
> between bus and cpu addresses?
>
> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
> returns nothing.
>

You haven't answered the question yet: you stated that the doorbell
address is software configurable, yet your code does not seem to
configure it. It only returns the doorbell address so that it gets
communicated to the downstream devices.

So how does the RC know which address is special, so it can trigger on
inbound writes hitting that address and assert the SPI ?

[PATCH v10] PCI: tango: Add MSI controller support

[Ard Biesheuvel]

On 24 August 2017 at 18:51, Marc Gonzalez
<marc_gonzalez@sigmadesigns.com> wrote:
> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>> On Tue, Aug 22, 2017 at 09:03:41PM +0100, Marc Zyngier wrote:
>>> Marc Gonzalez wrote:
>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>
>>>>>>  #define SMP8759_MUX              0x48
>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>> +#define SMP8759_STATUS           0x80
>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>
>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>> other address property?
>>>>
>>>> Since this bus address is software-configurable, I didn't think
>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>> other binding docs, especially
>>>>
>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>
>>> If that's software configurable, how on Earth did you pick the address?
>>> How do you ensure that it doesn't conflict with DMA? How is it
>>> configured into the RC?
>>
>> But we *do* need to resolve this.  This does seem like an address that
>> shouldn't be hard-coded into the driver.  Since this driver is
>> programming the address into an MSI message, but not into the receiver
>> of that message, there's a coordination issue between this driver and
>> whatever other software does that receiver configuration.
>
> OK. I'll move the doorbell address to the DT for v11.
>
> What property should be used for this address?
>
> sigma,doorbell ?
>
> Or maybe I can put it in reg, since I have a 1:1 mapping
> between bus and cpu addresses?
>
> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
> returns nothing.
>

You haven't answered the question yet: you stated that the doorbell
address is software configurable, yet your code does not seem to
configure it. It only returns the doorbell address so that it gets
communicated to the downstream devices.

So how does the RC know which address is special, so it can trigger on
inbound writes hitting that address and assert the SPI ?

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 24/08/2017 20:35, Ard Biesheuvel wrote:
> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>> Marc Gonzalez wrote:
>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>
>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>
>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>> other address property?
>>>>>
>>>>> Since this bus address is software-configurable, I didn't think
>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>> other binding docs, especially
>>>>>
>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>
>>>> If that's software configurable, how on Earth did you pick the address?
>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>> configured into the RC?
>>>
>>> But we *do* need to resolve this.  This does seem like an address that
>>> shouldn't be hard-coded into the driver.  Since this driver is
>>> programming the address into an MSI message, but not into the receiver
>>> of that message, there's a coordination issue between this driver and
>>> whatever other software does that receiver configuration.
>>
>> OK. I'll move the doorbell address to the DT for v11.
>>
>> What property should be used for this address?
>>
>> sigma,doorbell ?
>>
>> Or maybe I can put it in reg, since I have a 1:1 mapping
>> between bus and cpu addresses?
>>
>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>> returns nothing.
> 
> You haven't answered the question yet: you stated that the doorbell
> address is software configurable, yet your code does not seem to
> configure it. It only returns the doorbell address so that it gets
> communicated to the downstream devices.
> 
> So how does the RC know which address is special, so it can trigger on
> inbound writes hitting that address and assert the SPI ?

The CPU address of the MSI doorbell address is 0x2e07c
i.e. within the reg space of the PCIe controller block.

As I discussed back in March, the RC implements an odd
bus-to-system mapping.

RC BAR0 defines a window in PCI address space (max 1GB).
Accesses outside this window are silently ignored.
The window is divided into 8 "regions" and there are 8
registers defining the offset into CPU space.

In pseudo code, assuming pci_address is within the
window defined by BAR0:

cpu_address map_bus_to_system(pci_address)
{
    temp = pci_address - BAR0.base
    region = temp / region_size
    offset = temp % region_size
    cpu_address = region_reg[region] + offset
    return cpu_address
}

The current setup is:

DRAM at 0x80000000-0xa0000000
BAR0.base = 0x80000000
REGION[0] = 0x80000000
REGION[1] = 0x88000000
REGION[2] = 0x90000000
REGION[3] = 0x98000000
REGION[4] = 0x0

(This map means 1:1 identity for DRAM addresses.)

Thus when a device writes to 0xa002e07c (region 4)
the write is forwarded to 0x2e07c.

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 24/08/2017 20:35, Ard Biesheuvel wrote:
> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>> Marc Gonzalez wrote:
>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>
>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>
>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>> other address property?
>>>>>
>>>>> Since this bus address is software-configurable, I didn't think
>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>> other binding docs, especially
>>>>>
>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>
>>>> If that's software configurable, how on Earth did you pick the address?
>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>> configured into the RC?
>>>
>>> But we *do* need to resolve this.  This does seem like an address that
>>> shouldn't be hard-coded into the driver.  Since this driver is
>>> programming the address into an MSI message, but not into the receiver
>>> of that message, there's a coordination issue between this driver and
>>> whatever other software does that receiver configuration.
>>
>> OK. I'll move the doorbell address to the DT for v11.
>>
>> What property should be used for this address?
>>
>> sigma,doorbell ?
>>
>> Or maybe I can put it in reg, since I have a 1:1 mapping
>> between bus and cpu addresses?
>>
>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>> returns nothing.
> 
> You haven't answered the question yet: you stated that the doorbell
> address is software configurable, yet your code does not seem to
> configure it. It only returns the doorbell address so that it gets
> communicated to the downstream devices.
> 
> So how does the RC know which address is special, so it can trigger on
> inbound writes hitting that address and assert the SPI ?

The CPU address of the MSI doorbell address is 0x2e07c
i.e. within the reg space of the PCIe controller block.

As I discussed back in March, the RC implements an odd
bus-to-system mapping.

RC BAR0 defines a window in PCI address space (max 1GB).
Accesses outside this window are silently ignored.
The window is divided into 8 "regions" and there are 8
registers defining the offset into CPU space.

In pseudo code, assuming pci_address is within the
window defined by BAR0:

cpu_address map_bus_to_system(pci_address)
{
    temp = pci_address - BAR0.base
    region = temp / region_size
    offset = temp % region_size
    cpu_address = region_reg[region] + offset
    return cpu_address
}

The current setup is:

DRAM at 0x80000000-0xa0000000
BAR0.base = 0x80000000
REGION[0] = 0x80000000
REGION[1] = 0x88000000
REGION[2] = 0x90000000
REGION[3] = 0x98000000
REGION[4] = 0x0

(This map means 1:1 identity for DRAM addresses.)

Thus when a device writes to 0xa002e07c (region 4)
the write is forwarded to 0x2e07c.

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>> Marc Gonzalez wrote:
>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>
>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>
>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>> other address property?
>>>>>>
>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>> other binding docs, especially
>>>>>>
>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>
>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>> configured into the RC?
>>>>
>>>> But we *do* need to resolve this.  This does seem like an address that
>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>> programming the address into an MSI message, but not into the receiver
>>>> of that message, there's a coordination issue between this driver and
>>>> whatever other software does that receiver configuration.
>>>
>>> OK. I'll move the doorbell address to the DT for v11.
>>>
>>> What property should be used for this address?
>>>
>>> sigma,doorbell ?
>>>
>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>> between bus and cpu addresses?
>>>
>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>> returns nothing.
>> 
>> You haven't answered the question yet: you stated that the doorbell
>> address is software configurable, yet your code does not seem to
>> configure it. It only returns the doorbell address so that it gets
>> communicated to the downstream devices.
>> 
>> So how does the RC know which address is special, so it can trigger on
>> inbound writes hitting that address and assert the SPI ?
>
> The CPU address of the MSI doorbell address is 0x2e07c
> i.e. within the reg space of the PCIe controller block.

Which you describe in DT already, right? So why aren't you using an
offset in this region as your MSI ddorbell (potentially applying an
offset, see below)?

>
> As I discussed back in March, the RC implements an odd
> bus-to-system mapping.
>
> RC BAR0 defines a window in PCI address space (max 1GB).
> Accesses outside this window are silently ignored.
> The window is divided into 8 "regions" and there are 8
> registers defining the offset into CPU space.
>
> In pseudo code, assuming pci_address is within the
> window defined by BAR0:
>
> cpu_address map_bus_to_system(pci_address)
> {
>     temp = pci_address - BAR0.base
>     region = temp / region_size
>     offset = temp % region_size
>     cpu_address = region_reg[region] + offset
>     return cpu_address
> }
>
> The current setup is:
>
> DRAM at 0x80000000-0xa0000000
> BAR0.base = 0x80000000
> REGION[0] = 0x80000000
> REGION[1] = 0x88000000
> REGION[2] = 0x90000000
> REGION[3] = 0x98000000
> REGION[4] = 0x0
>
> (This map means 1:1 identity for DRAM addresses.)
>
> Thus when a device writes to 0xa002e07c (region 4)
> the write is forwarded to 0x2e07c.

But how do you find out about the 0xa0000000 offset? You must make sure
that the provided address is outside of RAM, should you end-up on a
system more than 1GB of RAM.

	M.
-- 
Jazz is not dead, it just smell funny.

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>> Marc Gonzalez wrote:
>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>
>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>
>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>> other address property?
>>>>>>
>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>> other binding docs, especially
>>>>>>
>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>
>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>> configured into the RC?
>>>>
>>>> But we *do* need to resolve this.  This does seem like an address that
>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>> programming the address into an MSI message, but not into the receiver
>>>> of that message, there's a coordination issue between this driver and
>>>> whatever other software does that receiver configuration.
>>>
>>> OK. I'll move the doorbell address to the DT for v11.
>>>
>>> What property should be used for this address?
>>>
>>> sigma,doorbell ?
>>>
>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>> between bus and cpu addresses?
>>>
>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>> returns nothing.
>> 
>> You haven't answered the question yet: you stated that the doorbell
>> address is software configurable, yet your code does not seem to
>> configure it. It only returns the doorbell address so that it gets
>> communicated to the downstream devices.
>> 
>> So how does the RC know which address is special, so it can trigger on
>> inbound writes hitting that address and assert the SPI ?
>
> The CPU address of the MSI doorbell address is 0x2e07c
> i.e. within the reg space of the PCIe controller block.

Which you describe in DT already, right? So why aren't you using an
offset in this region as your MSI ddorbell (potentially applying an
offset, see below)?

>
> As I discussed back in March, the RC implements an odd
> bus-to-system mapping.
>
> RC BAR0 defines a window in PCI address space (max 1GB).
> Accesses outside this window are silently ignored.
> The window is divided into 8 "regions" and there are 8
> registers defining the offset into CPU space.
>
> In pseudo code, assuming pci_address is within the
> window defined by BAR0:
>
> cpu_address map_bus_to_system(pci_address)
> {
>     temp = pci_address - BAR0.base
>     region = temp / region_size
>     offset = temp % region_size
>     cpu_address = region_reg[region] + offset
>     return cpu_address
> }
>
> The current setup is:
>
> DRAM at 0x80000000-0xa0000000
> BAR0.base = 0x80000000
> REGION[0] = 0x80000000
> REGION[1] = 0x88000000
> REGION[2] = 0x90000000
> REGION[3] = 0x98000000
> REGION[4] = 0x0
>
> (This map means 1:1 identity for DRAM addresses.)
>
> Thus when a device writes to 0xa002e07c (region 4)
> the write is forwarded to 0x2e07c.

But how do you find out about the 0xa0000000 offset? You must make sure
that the provided address is outside of RAM, should you end-up on a
system more than 1GB of RAM.

	M.
-- 
Jazz is not dead, it just smell funny.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 09:54, Marc Zyngier wrote:
> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>> Marc Gonzalez wrote:
>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>
>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>
>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>> other address property?
>>>>>>>
>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>> other binding docs, especially
>>>>>>>
>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>
>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>> configured into the RC?
>>>>>
>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>> programming the address into an MSI message, but not into the receiver
>>>>> of that message, there's a coordination issue between this driver and
>>>>> whatever other software does that receiver configuration.
>>>>
>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>
>>>> What property should be used for this address?
>>>>
>>>> sigma,doorbell ?
>>>>
>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>> between bus and cpu addresses?
>>>>
>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>> returns nothing.
>>>
>>> You haven't answered the question yet: you stated that the doorbell
>>> address is software configurable, yet your code does not seem to
>>> configure it. It only returns the doorbell address so that it gets
>>> communicated to the downstream devices.
>>>
>>> So how does the RC know which address is special, so it can trigger on
>>> inbound writes hitting that address and assert the SPI ?
>>
>> The CPU address of the MSI doorbell address is 0x2e07c
>> i.e. within the reg space of the PCIe controller block.
> 
> Which you describe in DT already, right? So why aren't you using an
> offset in this region as your MSI doorbell (potentially applying an
> offset, see below)?

Yes, the controller is described in DT:

	pcie@2e000 {
		compatible = "sigma,smp8759-pcie";
		reg = <0x50000000 0x400000>, <0x2e000 0x100>;

IIUC, you're saying I don't need the doorbell address
explicitly in the DT, because I can compute:

0x2e000 (from the DT) + 0x7c (offset within the block)

OK, that sounds right.

Then there is the matter of the region offset,
i.e. 0xa0000000 in my current code.

It might also be worth keeping in mind that there is
a second revision of the PCIe controller that handles
the doorbell differently. In rev2, I just pick an
arbitrary address within the window, program that
bus address into the controller, and the controller
knows to forward that single address to the right
place. See "[RFC PATCH v0.2] PCI: Add support for
tango PCIe host bridge" for an example of this.

Typically, I pick the first address of the window,
BAR0.base, since I assume no device will ever need
to read/write the first word of RAM.


>> As I discussed back in March, the RC implements an odd
>> bus-to-system mapping.
>>
>> RC BAR0 defines a window in PCI address space (max 1GB).
>> Accesses outside this window are silently ignored.
>> The window is divided into 8 "regions" and there are 8
>> registers defining the offset into CPU space.
>>
>> In pseudo code, assuming pci_address is within the
>> window defined by BAR0:
>>
>> cpu_address map_bus_to_system(pci_address)
>> {
>>     temp = pci_address - BAR0.base
>>     region = temp / region_size
>>     offset = temp % region_size
>>     cpu_address = region_reg[region] + offset
>>     return cpu_address
>> }
>>
>> The current setup is:
>>
>> DRAM at 0x80000000-0xa0000000
>> BAR0.base = 0x80000000
>> REGION[0] = 0x80000000
>> REGION[1] = 0x88000000
>> REGION[2] = 0x90000000
>> REGION[3] = 0x98000000
>> REGION[4] = 0x0
>>
>> (This map means 1:1 identity for DRAM addresses.)
>>
>> Thus when a device writes to 0xa002e07c (region 4)
>> the write is forwarded to 0x2e07c.
> 
> But how do you find out about the 0xa0000000 offset? You must make sure
> that the provided address is outside of RAM, should you end-up on a
> system more than 1GB of RAM.

You're right, I've swept this issue under the rug so far.

The boards typically come with either
- two 512MB DIMMs
- two 1GB DIMMs
(there may be other setups I'm not aware of, e.g. with
a single memory module).

In the DT for my dev board, I describe *all* of the RAM.

	memory@80000000 {
		device_type = "memory";
		reg = <0x80000000 0x80000000>; /* 2 GB */
	};

But actually, Linux is only given to manage a fraction
of this memory, as some of it is for other processors
and DSPs, and a large part is for video decoder buffers.

In the end, Linux manages, typically
128MB, 256MB, 512MB, 2x128MB, or 2x256MB
(there may be other setups I'm not aware of)
And the actual config is passed to Linux through
a mem= command-line directive.

It is not clear to me, as I discussed with Ard, how the
Linux driver is supposed to make this all work.

Maybe I can have some platform code that walks the
different RAM areas available to Linux, and sets up
the appropriate physical-to-dma mappings?

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 09:54, Marc Zyngier wrote:
> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>> Marc Gonzalez wrote:
>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>
>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>
>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>> other address property?
>>>>>>>
>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>> other binding docs, especially
>>>>>>>
>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>
>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>> configured into the RC?
>>>>>
>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>> programming the address into an MSI message, but not into the receiver
>>>>> of that message, there's a coordination issue between this driver and
>>>>> whatever other software does that receiver configuration.
>>>>
>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>
>>>> What property should be used for this address?
>>>>
>>>> sigma,doorbell ?
>>>>
>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>> between bus and cpu addresses?
>>>>
>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>> returns nothing.
>>>
>>> You haven't answered the question yet: you stated that the doorbell
>>> address is software configurable, yet your code does not seem to
>>> configure it. It only returns the doorbell address so that it gets
>>> communicated to the downstream devices.
>>>
>>> So how does the RC know which address is special, so it can trigger on
>>> inbound writes hitting that address and assert the SPI ?
>>
>> The CPU address of the MSI doorbell address is 0x2e07c
>> i.e. within the reg space of the PCIe controller block.
> 
> Which you describe in DT already, right? So why aren't you using an
> offset in this region as your MSI doorbell (potentially applying an
> offset, see below)?

Yes, the controller is described in DT:

	pcie at 2e000 {
		compatible = "sigma,smp8759-pcie";
		reg = <0x50000000 0x400000>, <0x2e000 0x100>;

IIUC, you're saying I don't need the doorbell address
explicitly in the DT, because I can compute:

0x2e000 (from the DT) + 0x7c (offset within the block)

OK, that sounds right.

Then there is the matter of the region offset,
i.e. 0xa0000000 in my current code.

It might also be worth keeping in mind that there is
a second revision of the PCIe controller that handles
the doorbell differently. In rev2, I just pick an
arbitrary address within the window, program that
bus address into the controller, and the controller
knows to forward that single address to the right
place. See "[RFC PATCH v0.2] PCI: Add support for
tango PCIe host bridge" for an example of this.

Typically, I pick the first address of the window,
BAR0.base, since I assume no device will ever need
to read/write the first word of RAM.


>> As I discussed back in March, the RC implements an odd
>> bus-to-system mapping.
>>
>> RC BAR0 defines a window in PCI address space (max 1GB).
>> Accesses outside this window are silently ignored.
>> The window is divided into 8 "regions" and there are 8
>> registers defining the offset into CPU space.
>>
>> In pseudo code, assuming pci_address is within the
>> window defined by BAR0:
>>
>> cpu_address map_bus_to_system(pci_address)
>> {
>>     temp = pci_address - BAR0.base
>>     region = temp / region_size
>>     offset = temp % region_size
>>     cpu_address = region_reg[region] + offset
>>     return cpu_address
>> }
>>
>> The current setup is:
>>
>> DRAM at 0x80000000-0xa0000000
>> BAR0.base = 0x80000000
>> REGION[0] = 0x80000000
>> REGION[1] = 0x88000000
>> REGION[2] = 0x90000000
>> REGION[3] = 0x98000000
>> REGION[4] = 0x0
>>
>> (This map means 1:1 identity for DRAM addresses.)
>>
>> Thus when a device writes to 0xa002e07c (region 4)
>> the write is forwarded to 0x2e07c.
> 
> But how do you find out about the 0xa0000000 offset? You must make sure
> that the provided address is outside of RAM, should you end-up on a
> system more than 1GB of RAM.

You're right, I've swept this issue under the rug so far.

The boards typically come with either
- two 512MB DIMMs
- two 1GB DIMMs
(there may be other setups I'm not aware of, e.g. with
a single memory module).

In the DT for my dev board, I describe *all* of the RAM.

	memory at 80000000 {
		device_type = "memory";
		reg = <0x80000000 0x80000000>; /* 2 GB */
	};

But actually, Linux is only given to manage a fraction
of this memory, as some of it is for other processors
and DSPs, and a large part is for video decoder buffers.

In the end, Linux manages, typically
128MB, 256MB, 512MB, 2x128MB, or 2x256MB
(there may be other setups I'm not aware of)
And the actual config is passed to Linux through
a mem= command-line directive.

It is not clear to me, as I discussed with Ard, how the
Linux driver is supposed to make this all work.

Maybe I can have some platform code that walks the
different RAM areas available to Linux, and sets up
the appropriate physical-to-dma mappings?

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 09:54, Marc Zyngier wrote:
> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>> Marc Gonzalez wrote:
>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>
>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>
>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>> other address property?
>>>>>>>
>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>> other binding docs, especially
>>>>>>>
>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>
>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>> configured into the RC?
>>>>>
>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>> programming the address into an MSI message, but not into the receiver
>>>>> of that message, there's a coordination issue between this driver and
>>>>> whatever other software does that receiver configuration.
>>>>
>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>
>>>> What property should be used for this address?
>>>>
>>>> sigma,doorbell ?
>>>>
>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>> between bus and cpu addresses?
>>>>
>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>> returns nothing.
>>>
>>> You haven't answered the question yet: you stated that the doorbell
>>> address is software configurable, yet your code does not seem to
>>> configure it. It only returns the doorbell address so that it gets
>>> communicated to the downstream devices.
>>>
>>> So how does the RC know which address is special, so it can trigger on
>>> inbound writes hitting that address and assert the SPI ?
>>
>> The CPU address of the MSI doorbell address is 0x2e07c
>> i.e. within the reg space of the PCIe controller block.
> 
> Which you describe in DT already, right? So why aren't you using an
> offset in this region as your MSI ddorbell (potentially applying an
> offset, see below)?
> 
>
>> As I discussed back in March, the RC implements an odd
>> bus-to-system mapping.
>>
>> RC BAR0 defines a window in PCI address space (max 1GB).
>> Accesses outside this window are silently ignored.
>> The window is divided into 8 "regions" and there are 8
>> registers defining the offset into CPU space.
>>
>> In pseudo code, assuming pci_address is within the
>> window defined by BAR0:
>>
>> cpu_address map_bus_to_system(pci_address)
>> {
>>     temp = pci_address - BAR0.base
>>     region = temp / region_size
>>     offset = temp % region_size
>>     cpu_address = region_reg[region] + offset
>>     return cpu_address
>> }
>>
>> The current setup is:
>>
>> DRAM at 0x80000000-0xa0000000
>> BAR0.base = 0x80000000
>> REGION[0] = 0x80000000
>> REGION[1] = 0x88000000
>> REGION[2] = 0x90000000
>> REGION[3] = 0x98000000
>> REGION[4] = 0x0
>>
>> (This map means 1:1 identity for DRAM addresses.)
>>
>> Thus when a device writes to 0xa002e07c (region 4)
>> the write is forwarded to 0x2e07c.
> 
> But how do you find out about the 0xa0000000 offset? You must make sure
> that the provided address is outside of RAM, should you end-up on a
> system more than 1GB of RAM.

Robin wrote a prophetic post back in March:
http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html

> The appropriate DT property would be "dma-ranges", i.e.
> 
> pci@... {
> 	...
> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
> }

The dma-ranges property seems to be exactly what I'm looking for:

Restrict DMA to the first X MB of RAM (use a bounce buffer
for other physical addresses).

I added the following property to my PCIe node

  dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;

with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[

But it does not work. Arg!

My PCIe controller driver seems to be correctly calling of_dma_get_range:

[    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
[    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
[    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
[    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
[    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
[    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
[    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
[    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
[    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
[    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)

of_dma_get_range() is called on the pcie node (which is expected)
but after parsing n_addr_cells and n_size_cells in the while loop,
the code jumps to the parent node ("soc")... while my property is
attached to the pcie node...

[    0.507754] of_dma_get_range: node=dfbf74cc np=dfbf74cc name=/soc/pcie@2e000 name2=/soc/pcie@2e000
...
[    0.509162] __of_find_property: node=soc find=#address-cells prop=compatible
[    0.509168] __of_find_property: node=soc find=#address-cells prop=interrupt-parent
[    0.509173] __of_find_property: node=soc find=#address-cells prop=#address-cells
[    0.509178] __of_find_property: node=soc find=#size-cells prop=compatible
[    0.509182] __of_find_property: node=soc find=#size-cells prop=interrupt-parent
[    0.509186] __of_find_property: node=soc find=#size-cells prop=#address-cells
[    0.509190] __of_find_property: node=soc find=#size-cells prop=#size-cells
[    0.509195] __of_find_property: node=soc find=dma-ranges prop=compatible
[    0.509199] __of_find_property: node=soc find=dma-ranges prop=interrupt-parent
[    0.509203] __of_find_property: node=soc find=dma-ranges prop=#address-cells
[    0.509207] __of_find_property: node=soc find=dma-ranges prop=#size-cells
[    0.509211] __of_find_property: node=soc find=dma-ranges prop=ranges
[    0.509215] __of_find_property: node=soc find=dma-ranges prop=name
[    0.509219] dma-ranges=  (null)

http://elixir.free-electrons.com/linux/latest/source/drivers/of/address.c#L838

What am I missing?

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 09:54, Marc Zyngier wrote:
> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>> Marc Gonzalez wrote:
>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>
>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>
>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>> other address property?
>>>>>>>
>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>> other binding docs, especially
>>>>>>>
>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>
>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>> configured into the RC?
>>>>>
>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>> programming the address into an MSI message, but not into the receiver
>>>>> of that message, there's a coordination issue between this driver and
>>>>> whatever other software does that receiver configuration.
>>>>
>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>
>>>> What property should be used for this address?
>>>>
>>>> sigma,doorbell ?
>>>>
>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>> between bus and cpu addresses?
>>>>
>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>> returns nothing.
>>>
>>> You haven't answered the question yet: you stated that the doorbell
>>> address is software configurable, yet your code does not seem to
>>> configure it. It only returns the doorbell address so that it gets
>>> communicated to the downstream devices.
>>>
>>> So how does the RC know which address is special, so it can trigger on
>>> inbound writes hitting that address and assert the SPI ?
>>
>> The CPU address of the MSI doorbell address is 0x2e07c
>> i.e. within the reg space of the PCIe controller block.
> 
> Which you describe in DT already, right? So why aren't you using an
> offset in this region as your MSI ddorbell (potentially applying an
> offset, see below)?
> 
>
>> As I discussed back in March, the RC implements an odd
>> bus-to-system mapping.
>>
>> RC BAR0 defines a window in PCI address space (max 1GB).
>> Accesses outside this window are silently ignored.
>> The window is divided into 8 "regions" and there are 8
>> registers defining the offset into CPU space.
>>
>> In pseudo code, assuming pci_address is within the
>> window defined by BAR0:
>>
>> cpu_address map_bus_to_system(pci_address)
>> {
>>     temp = pci_address - BAR0.base
>>     region = temp / region_size
>>     offset = temp % region_size
>>     cpu_address = region_reg[region] + offset
>>     return cpu_address
>> }
>>
>> The current setup is:
>>
>> DRAM at 0x80000000-0xa0000000
>> BAR0.base = 0x80000000
>> REGION[0] = 0x80000000
>> REGION[1] = 0x88000000
>> REGION[2] = 0x90000000
>> REGION[3] = 0x98000000
>> REGION[4] = 0x0
>>
>> (This map means 1:1 identity for DRAM addresses.)
>>
>> Thus when a device writes to 0xa002e07c (region 4)
>> the write is forwarded to 0x2e07c.
> 
> But how do you find out about the 0xa0000000 offset? You must make sure
> that the provided address is outside of RAM, should you end-up on a
> system more than 1GB of RAM.

Robin wrote a prophetic post back in March:
http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html

> The appropriate DT property would be "dma-ranges", i.e.
> 
> pci at ... {
> 	...
> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
> }

The dma-ranges property seems to be exactly what I'm looking for:

Restrict DMA to the first X MB of RAM (use a bounce buffer
for other physical addresses).

I added the following property to my PCIe node

  dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;

with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[

But it does not work. Arg!

My PCIe controller driver seems to be correctly calling of_dma_get_range:

[    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
[    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
[    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
[    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
[    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
[    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
[    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
[    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
[    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
[    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)

of_dma_get_range() is called on the pcie node (which is expected)
but after parsing n_addr_cells and n_size_cells in the while loop,
the code jumps to the parent node ("soc")... while my property is
attached to the pcie node...

[    0.507754] of_dma_get_range: node=dfbf74cc np=dfbf74cc name=/soc/pcie at 2e000 name2=/soc/pcie at 2e000
...
[    0.509162] __of_find_property: node=soc find=#address-cells prop=compatible
[    0.509168] __of_find_property: node=soc find=#address-cells prop=interrupt-parent
[    0.509173] __of_find_property: node=soc find=#address-cells prop=#address-cells
[    0.509178] __of_find_property: node=soc find=#size-cells prop=compatible
[    0.509182] __of_find_property: node=soc find=#size-cells prop=interrupt-parent
[    0.509186] __of_find_property: node=soc find=#size-cells prop=#address-cells
[    0.509190] __of_find_property: node=soc find=#size-cells prop=#size-cells
[    0.509195] __of_find_property: node=soc find=dma-ranges prop=compatible
[    0.509199] __of_find_property: node=soc find=dma-ranges prop=interrupt-parent
[    0.509203] __of_find_property: node=soc find=dma-ranges prop=#address-cells
[    0.509207] __of_find_property: node=soc find=dma-ranges prop=#size-cells
[    0.509211] __of_find_property: node=soc find=dma-ranges prop=ranges
[    0.509215] __of_find_property: node=soc find=dma-ranges prop=name
[    0.509219] dma-ranges=  (null)

http://elixir.free-electrons.com/linux/latest/source/drivers/of/address.c#L838

What am I missing?

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Robin Murphy]

On 25/08/17 16:01, Mason wrote:
> On 25/08/2017 09:54, Marc Zyngier wrote:
>> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>>> Marc Gonzalez wrote:
>>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>>
>>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>>
>>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>>> other address property?
>>>>>>>>
>>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>>> other binding docs, especially
>>>>>>>>
>>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>>
>>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>>> configured into the RC?
>>>>>>
>>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>>> programming the address into an MSI message, but not into the receiver
>>>>>> of that message, there's a coordination issue between this driver and
>>>>>> whatever other software does that receiver configuration.
>>>>>
>>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>>
>>>>> What property should be used for this address?
>>>>>
>>>>> sigma,doorbell ?
>>>>>
>>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>>> between bus and cpu addresses?
>>>>>
>>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>>> returns nothing.
>>>>
>>>> You haven't answered the question yet: you stated that the doorbell
>>>> address is software configurable, yet your code does not seem to
>>>> configure it. It only returns the doorbell address so that it gets
>>>> communicated to the downstream devices.
>>>>
>>>> So how does the RC know which address is special, so it can trigger on
>>>> inbound writes hitting that address and assert the SPI ?
>>>
>>> The CPU address of the MSI doorbell address is 0x2e07c
>>> i.e. within the reg space of the PCIe controller block.
>>
>> Which you describe in DT already, right? So why aren't you using an
>> offset in this region as your MSI ddorbell (potentially applying an
>> offset, see below)?
>>
>>
>>> As I discussed back in March, the RC implements an odd
>>> bus-to-system mapping.
>>>
>>> RC BAR0 defines a window in PCI address space (max 1GB).
>>> Accesses outside this window are silently ignored.
>>> The window is divided into 8 "regions" and there are 8
>>> registers defining the offset into CPU space.
>>>
>>> In pseudo code, assuming pci_address is within the
>>> window defined by BAR0:
>>>
>>> cpu_address map_bus_to_system(pci_address)
>>> {
>>>     temp = pci_address - BAR0.base
>>>     region = temp / region_size
>>>     offset = temp % region_size
>>>     cpu_address = region_reg[region] + offset
>>>     return cpu_address
>>> }
>>>
>>> The current setup is:
>>>
>>> DRAM at 0x80000000-0xa0000000
>>> BAR0.base = 0x80000000
>>> REGION[0] = 0x80000000
>>> REGION[1] = 0x88000000
>>> REGION[2] = 0x90000000
>>> REGION[3] = 0x98000000
>>> REGION[4] = 0x0
>>>
>>> (This map means 1:1 identity for DRAM addresses.)
>>>
>>> Thus when a device writes to 0xa002e07c (region 4)
>>> the write is forwarded to 0x2e07c.
>>
>> But how do you find out about the 0xa0000000 offset? You must make sure
>> that the provided address is outside of RAM, should you end-up on a
>> system more than 1GB of RAM.
> 
> Robin wrote a prophetic post back in March:
> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
> 
>> The appropriate DT property would be "dma-ranges", i.e.
>>
>> pci@... {
>> 	...
>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>> }
> 
> The dma-ranges property seems to be exactly what I'm looking for:
> 
> Restrict DMA to the first X MB of RAM (use a bounce buffer
> for other physical addresses).
> 
> I added the following property to my PCIe node
> 
>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
> 
> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
> 
> But it does not work. Arg!
> 
> My PCIe controller driver seems to be correctly calling of_dma_get_range:
> 
> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
> 
> of_dma_get_range() is called on the pcie node (which is expected)
> but after parsing n_addr_cells and n_size_cells in the while loop,
> the code jumps to the parent node ("soc")... while my property is
> attached to the pcie node...

This is not your driver calling of_dma_get_range(), this is the PCI core
doing so in the act of DMA master configuration for a discovered
*endpoint*. The fact that the "pass the host controller's OF node
because we don't have one for the endpoint" bodge only works properly
for dma-coherent and not dma-ranges is a known, but irrelevant, problem.

If your host controller driver needs to discover its windows from DT to
configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
pcie-racar, pcie-xgene, etc. for examples.

Robin.

> 
> [    0.507754] of_dma_get_range: node=dfbf74cc np=dfbf74cc name=/soc/pcie@2e000 name2=/soc/pcie@2e000
> ...
> [    0.509162] __of_find_property: node=soc find=#address-cells prop=compatible
> [    0.509168] __of_find_property: node=soc find=#address-cells prop=interrupt-parent
> [    0.509173] __of_find_property: node=soc find=#address-cells prop=#address-cells
> [    0.509178] __of_find_property: node=soc find=#size-cells prop=compatible
> [    0.509182] __of_find_property: node=soc find=#size-cells prop=interrupt-parent
> [    0.509186] __of_find_property: node=soc find=#size-cells prop=#address-cells
> [    0.509190] __of_find_property: node=soc find=#size-cells prop=#size-cells
> [    0.509195] __of_find_property: node=soc find=dma-ranges prop=compatible
> [    0.509199] __of_find_property: node=soc find=dma-ranges prop=interrupt-parent
> [    0.509203] __of_find_property: node=soc find=dma-ranges prop=#address-cells
> [    0.509207] __of_find_property: node=soc find=dma-ranges prop=#size-cells
> [    0.509211] __of_find_property: node=soc find=dma-ranges prop=ranges
> [    0.509215] __of_find_property: node=soc find=dma-ranges prop=name
> [    0.509219] dma-ranges=  (null)
> 
> http://elixir.free-electrons.com/linux/latest/source/drivers/of/address.c#L838
> 
> What am I missing?
> 
> Regards.
> 

[PATCH v10] PCI: tango: Add MSI controller support

[Robin Murphy]

On 25/08/17 16:01, Mason wrote:
> On 25/08/2017 09:54, Marc Zyngier wrote:
>> On Thu, Aug 24 2017 at 10:53:16 pm BST, Mason <slash.tmp@free.fr> wrote:
>>> On 24/08/2017 20:35, Ard Biesheuvel wrote:
>>>> On 24 August 2017 at 18:51, Marc Gonzalez wrote:
>>>>> On 24/08/2017 19:04, Bjorn Helgaas wrote:
>>>>>> On Tue, Aug 22, 2017 Marc Zyngier wrote:
>>>>>>> Marc Gonzalez wrote:
>>>>>>>> On 22/08/2017 18:29, Marc Zyngier wrote:
>>>>>>>>> On 22/08/17 15:56, Marc Gonzalez wrote:
>>>>>>>>>
>>>>>>>>>>  #define SMP8759_MUX              0x48
>>>>>>>>>>  #define SMP8759_TEST_OUT 0x74
>>>>>>>>>> +#define SMP8759_STATUS           0x80
>>>>>>>>>> +#define SMP8759_ENABLE           0xa0
>>>>>>>>>> +#define SMP8759_DOORBELL 0xa002e07c
>>>>>>>>>
>>>>>>>>> Why is this hardcoded and not coming from the device-tree, just like any
>>>>>>>>> other address property?
>>>>>>>>
>>>>>>>> Since this bus address is software-configurable, I didn't think
>>>>>>>> it belonged in the DT. Also, I didn't see anything similar in
>>>>>>>> other binding docs, especially
>>>>>>>>
>>>>>>>> Documentation/devicetree/bindings/interrupt-controller/msi.txt
>>>>>>>
>>>>>>> If that's software configurable, how on Earth did you pick the address?
>>>>>>> How do you ensure that it doesn't conflict with DMA? How is it
>>>>>>> configured into the RC?
>>>>>>
>>>>>> But we *do* need to resolve this.  This does seem like an address that
>>>>>> shouldn't be hard-coded into the driver.  Since this driver is
>>>>>> programming the address into an MSI message, but not into the receiver
>>>>>> of that message, there's a coordination issue between this driver and
>>>>>> whatever other software does that receiver configuration.
>>>>>
>>>>> OK. I'll move the doorbell address to the DT for v11.
>>>>>
>>>>> What property should be used for this address?
>>>>>
>>>>> sigma,doorbell ?
>>>>>
>>>>> Or maybe I can put it in reg, since I have a 1:1 mapping
>>>>> between bus and cpu addresses?
>>>>>
>>>>> git grep -i doorbell arch/arm/boot/dts/ arch/arm64/boot/dts/
>>>>> returns nothing.
>>>>
>>>> You haven't answered the question yet: you stated that the doorbell
>>>> address is software configurable, yet your code does not seem to
>>>> configure it. It only returns the doorbell address so that it gets
>>>> communicated to the downstream devices.
>>>>
>>>> So how does the RC know which address is special, so it can trigger on
>>>> inbound writes hitting that address and assert the SPI ?
>>>
>>> The CPU address of the MSI doorbell address is 0x2e07c
>>> i.e. within the reg space of the PCIe controller block.
>>
>> Which you describe in DT already, right? So why aren't you using an
>> offset in this region as your MSI ddorbell (potentially applying an
>> offset, see below)?
>>
>>
>>> As I discussed back in March, the RC implements an odd
>>> bus-to-system mapping.
>>>
>>> RC BAR0 defines a window in PCI address space (max 1GB).
>>> Accesses outside this window are silently ignored.
>>> The window is divided into 8 "regions" and there are 8
>>> registers defining the offset into CPU space.
>>>
>>> In pseudo code, assuming pci_address is within the
>>> window defined by BAR0:
>>>
>>> cpu_address map_bus_to_system(pci_address)
>>> {
>>>     temp = pci_address - BAR0.base
>>>     region = temp / region_size
>>>     offset = temp % region_size
>>>     cpu_address = region_reg[region] + offset
>>>     return cpu_address
>>> }
>>>
>>> The current setup is:
>>>
>>> DRAM at 0x80000000-0xa0000000
>>> BAR0.base = 0x80000000
>>> REGION[0] = 0x80000000
>>> REGION[1] = 0x88000000
>>> REGION[2] = 0x90000000
>>> REGION[3] = 0x98000000
>>> REGION[4] = 0x0
>>>
>>> (This map means 1:1 identity for DRAM addresses.)
>>>
>>> Thus when a device writes to 0xa002e07c (region 4)
>>> the write is forwarded to 0x2e07c.
>>
>> But how do you find out about the 0xa0000000 offset? You must make sure
>> that the provided address is outside of RAM, should you end-up on a
>> system more than 1GB of RAM.
> 
> Robin wrote a prophetic post back in March:
> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
> 
>> The appropriate DT property would be "dma-ranges", i.e.
>>
>> pci at ... {
>> 	...
>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>> }
> 
> The dma-ranges property seems to be exactly what I'm looking for:
> 
> Restrict DMA to the first X MB of RAM (use a bounce buffer
> for other physical addresses).
> 
> I added the following property to my PCIe node
> 
>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
> 
> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
> 
> But it does not work. Arg!
> 
> My PCIe controller driver seems to be correctly calling of_dma_get_range:
> 
> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
> 
> of_dma_get_range() is called on the pcie node (which is expected)
> but after parsing n_addr_cells and n_size_cells in the while loop,
> the code jumps to the parent node ("soc")... while my property is
> attached to the pcie node...

This is not your driver calling of_dma_get_range(), this is the PCI core
doing so in the act of DMA master configuration for a discovered
*endpoint*. The fact that the "pass the host controller's OF node
because we don't have one for the endpoint" bodge only works properly
for dma-coherent and not dma-ranges is a known, but irrelevant, problem.

If your host controller driver needs to discover its windows from DT to
configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
pcie-racar, pcie-xgene, etc. for examples.

Robin.

> 
> [    0.507754] of_dma_get_range: node=dfbf74cc np=dfbf74cc name=/soc/pcie at 2e000 name2=/soc/pcie at 2e000
> ...
> [    0.509162] __of_find_property: node=soc find=#address-cells prop=compatible
> [    0.509168] __of_find_property: node=soc find=#address-cells prop=interrupt-parent
> [    0.509173] __of_find_property: node=soc find=#address-cells prop=#address-cells
> [    0.509178] __of_find_property: node=soc find=#size-cells prop=compatible
> [    0.509182] __of_find_property: node=soc find=#size-cells prop=interrupt-parent
> [    0.509186] __of_find_property: node=soc find=#size-cells prop=#address-cells
> [    0.509190] __of_find_property: node=soc find=#size-cells prop=#size-cells
> [    0.509195] __of_find_property: node=soc find=dma-ranges prop=compatible
> [    0.509199] __of_find_property: node=soc find=dma-ranges prop=interrupt-parent
> [    0.509203] __of_find_property: node=soc find=dma-ranges prop=#address-cells
> [    0.509207] __of_find_property: node=soc find=dma-ranges prop=#size-cells
> [    0.509211] __of_find_property: node=soc find=dma-ranges prop=ranges
> [    0.509215] __of_find_property: node=soc find=dma-ranges prop=name
> [    0.509219] dma-ranges=  (null)
> 
> http://elixir.free-electrons.com/linux/latest/source/drivers/of/address.c#L838
> 
> What am I missing?
> 
> Regards.
> 

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:25, Robin Murphy wrote:

> On 25/08/17 16:01, Mason wrote:
>
>> Robin wrote a prophetic post back in March:
>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>
>>> The appropriate DT property would be "dma-ranges", i.e.
>>>
>>> pci@... {
>>> 	...
>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>> }
>>
>> The dma-ranges property seems to be exactly what I'm looking for:
>>
>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>> for other physical addresses).
>>
>> I added the following property to my PCIe node
>>
>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>
>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>
>> But it does not work. Arg!
>>
>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>
>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>
>> of_dma_get_range() is called on the pcie node (which is expected)
>> but after parsing n_addr_cells and n_size_cells in the while loop,
>> the code jumps to the parent node ("soc")... while my property is
>> attached to the pcie node...
> 
> This is not your driver calling of_dma_get_range(), this is the PCI core
> doing so in the act of DMA master configuration for a discovered
> *endpoint*. The fact that the "pass the host controller's OF node
> because we don't have one for the endpoint" bodge only works properly
> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
> 
> If your host controller driver needs to discover its windows from DT to
> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
> pcie-racar, pcie-xgene, etc. for examples.

Yes, I'm aware that I need to do my own parsing of dma-ranges.
I can use that information to configure BAR0.base and the
region registers.

But Linux needs to record my settings at some point, right?
Otherwise, how does the DMA framework know that devices can
only reach cpu addresses [0x80000000, 0xa0000000[ and when
to use bounce buffers?

What's preventing the XHCI driver from allocating memory
outside of my "safe" range, and having the DMA framework
blindly map that?

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:25, Robin Murphy wrote:

> On 25/08/17 16:01, Mason wrote:
>
>> Robin wrote a prophetic post back in March:
>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>
>>> The appropriate DT property would be "dma-ranges", i.e.
>>>
>>> pci at ... {
>>> 	...
>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>> }
>>
>> The dma-ranges property seems to be exactly what I'm looking for:
>>
>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>> for other physical addresses).
>>
>> I added the following property to my PCIe node
>>
>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>
>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>
>> But it does not work. Arg!
>>
>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>
>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>
>> of_dma_get_range() is called on the pcie node (which is expected)
>> but after parsing n_addr_cells and n_size_cells in the while loop,
>> the code jumps to the parent node ("soc")... while my property is
>> attached to the pcie node...
> 
> This is not your driver calling of_dma_get_range(), this is the PCI core
> doing so in the act of DMA master configuration for a discovered
> *endpoint*. The fact that the "pass the host controller's OF node
> because we don't have one for the endpoint" bodge only works properly
> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
> 
> If your host controller driver needs to discover its windows from DT to
> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
> pcie-racar, pcie-xgene, etc. for examples.

Yes, I'm aware that I need to do my own parsing of dma-ranges.
I can use that information to configure BAR0.base and the
region registers.

But Linux needs to record my settings at some point, right?
Otherwise, how does the DMA framework know that devices can
only reach cpu addresses [0x80000000, 0xa0000000[ and when
to use bounce buffers?

What's preventing the XHCI driver from allocating memory
outside of my "safe" range, and having the DMA framework
blindly map that?

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Robin Murphy]

On 25/08/17 16:35, Mason wrote:
> On 25/08/2017 17:25, Robin Murphy wrote:
> 
>> On 25/08/17 16:01, Mason wrote:
>>
>>> Robin wrote a prophetic post back in March:
>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>
>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>
>>>> pci@... {
>>>> 	...
>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>> }
>>>
>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>
>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>> for other physical addresses).
>>>
>>> I added the following property to my PCIe node
>>>
>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>
>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>
>>> But it does not work. Arg!
>>>
>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>
>>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>
>>> of_dma_get_range() is called on the pcie node (which is expected)
>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>> the code jumps to the parent node ("soc")... while my property is
>>> attached to the pcie node...
>>
>> This is not your driver calling of_dma_get_range(), this is the PCI core
>> doing so in the act of DMA master configuration for a discovered
>> *endpoint*. The fact that the "pass the host controller's OF node
>> because we don't have one for the endpoint" bodge only works properly
>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>
>> If your host controller driver needs to discover its windows from DT to
>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>> pcie-racar, pcie-xgene, etc. for examples.
> 
> Yes, I'm aware that I need to do my own parsing of dma-ranges.
> I can use that information to configure BAR0.base and the
> region registers.
> 
> But Linux needs to record my settings at some point, right?
> Otherwise, how does the DMA framework know that devices can
> only reach cpu addresses [0x80000000, 0xa0000000[ and when
> to use bounce buffers?
> 
> What's preventing the XHCI driver from allocating memory
> outside of my "safe" range, and having the DMA framework
> blindly map that?

At the moment, nothing. Systems that have physical memory that is not
visible in PCI mem space are having a bad time and will not go to space
today.

But that bears no relation to your MSI controller getting its doorbell
address set appropriately.

Robin.

[PATCH v10] PCI: tango: Add MSI controller support

[Robin Murphy]

On 25/08/17 16:35, Mason wrote:
> On 25/08/2017 17:25, Robin Murphy wrote:
> 
>> On 25/08/17 16:01, Mason wrote:
>>
>>> Robin wrote a prophetic post back in March:
>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>
>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>
>>>> pci at ... {
>>>> 	...
>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>> }
>>>
>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>
>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>> for other physical addresses).
>>>
>>> I added the following property to my PCIe node
>>>
>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>
>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>
>>> But it does not work. Arg!
>>>
>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>
>>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>
>>> of_dma_get_range() is called on the pcie node (which is expected)
>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>> the code jumps to the parent node ("soc")... while my property is
>>> attached to the pcie node...
>>
>> This is not your driver calling of_dma_get_range(), this is the PCI core
>> doing so in the act of DMA master configuration for a discovered
>> *endpoint*. The fact that the "pass the host controller's OF node
>> because we don't have one for the endpoint" bodge only works properly
>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>
>> If your host controller driver needs to discover its windows from DT to
>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>> pcie-racar, pcie-xgene, etc. for examples.
> 
> Yes, I'm aware that I need to do my own parsing of dma-ranges.
> I can use that information to configure BAR0.base and the
> region registers.
> 
> But Linux needs to record my settings at some point, right?
> Otherwise, how does the DMA framework know that devices can
> only reach cpu addresses [0x80000000, 0xa0000000[ and when
> to use bounce buffers?
> 
> What's preventing the XHCI driver from allocating memory
> outside of my "safe" range, and having the DMA framework
> blindly map that?

At the moment, nothing. Systems that have physical memory that is not
visible in PCI mem space are having a bad time and will not go to space
today.

But that bears no relation to your MSI controller getting its doorbell
address set appropriately.

Robin.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:45, Robin Murphy wrote:

> On 25/08/17 16:35, Mason wrote:
>
>> On 25/08/2017 17:25, Robin Murphy wrote:
>>
>>> On 25/08/17 16:01, Mason wrote:
>>>
>>>> Robin wrote a prophetic post back in March:
>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>
>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>
>>>>> pci@... {
>>>>> 	...
>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>> }
>>>>
>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>
>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>> for other physical addresses).
>>>>
>>>> I added the following property to my PCIe node
>>>>
>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>
>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>
>>>> But it does not work. Arg!
>>>>
>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>
>>>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>>
>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>> the code jumps to the parent node ("soc")... while my property is
>>>> attached to the pcie node...
>>>
>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>> doing so in the act of DMA master configuration for a discovered
>>> *endpoint*. The fact that the "pass the host controller's OF node
>>> because we don't have one for the endpoint" bodge only works properly
>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>
>>> If your host controller driver needs to discover its windows from DT to
>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>> pcie-racar, pcie-xgene, etc. for examples.
>>
>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>> I can use that information to configure BAR0.base and the
>> region registers.
>>
>> But Linux needs to record my settings at some point, right?
>> Otherwise, how does the DMA framework know that devices can
>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>> to use bounce buffers?
>>
>> What's preventing the XHCI driver from allocating memory
>> outside of my "safe" range, and having the DMA framework
>> blindly map that?
> 
> At the moment, nothing. Systems that have physical memory that is not
> visible in PCI mem space are having a bad time and will not go to space
> today.
> 
> But that bears no relation to your MSI controller getting its doorbell
> address set appropriately.

OK, so this is what I propose for v11 in order to not
hard code the MSI doorbell address (e.g. 0xa002e07c)

I add the following property to the pcie node:

	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;

I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000

Then in the PCIe driver, I parse dma-ranges.

Consequently

	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c

Bjorn, Marc, Robin, is that an acceptable solution?


Tangent:

Robin, for my own education, how does one configure the DMA
framework to use bounce buffers for certain addresses?

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:45, Robin Murphy wrote:

> On 25/08/17 16:35, Mason wrote:
>
>> On 25/08/2017 17:25, Robin Murphy wrote:
>>
>>> On 25/08/17 16:01, Mason wrote:
>>>
>>>> Robin wrote a prophetic post back in March:
>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>
>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>
>>>>> pci at ... {
>>>>> 	...
>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>> }
>>>>
>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>
>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>> for other physical addresses).
>>>>
>>>> I added the following property to my PCIe node
>>>>
>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>
>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>
>>>> But it does not work. Arg!
>>>>
>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>
>>>> [    0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>>> [    0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>>> [    0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>>> [    0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>> [    0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>>> [    0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>>> [    0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>> [    0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>>> [    0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>> [    0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>>
>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>> the code jumps to the parent node ("soc")... while my property is
>>>> attached to the pcie node...
>>>
>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>> doing so in the act of DMA master configuration for a discovered
>>> *endpoint*. The fact that the "pass the host controller's OF node
>>> because we don't have one for the endpoint" bodge only works properly
>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>
>>> If your host controller driver needs to discover its windows from DT to
>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>> pcie-racar, pcie-xgene, etc. for examples.
>>
>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>> I can use that information to configure BAR0.base and the
>> region registers.
>>
>> But Linux needs to record my settings at some point, right?
>> Otherwise, how does the DMA framework know that devices can
>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>> to use bounce buffers?
>>
>> What's preventing the XHCI driver from allocating memory
>> outside of my "safe" range, and having the DMA framework
>> blindly map that?
> 
> At the moment, nothing. Systems that have physical memory that is not
> visible in PCI mem space are having a bad time and will not go to space
> today.
> 
> But that bears no relation to your MSI controller getting its doorbell
> address set appropriately.

OK, so this is what I propose for v11 in order to not
hard code the MSI doorbell address (e.g. 0xa002e07c)

I add the following property to the pcie node:

	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;

I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000

Then in the PCIe driver, I parse dma-ranges.

Consequently

	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c

Bjorn, Marc, Robin, is that an acceptable solution?


Tangent:

Robin, for my own education, how does one configure the DMA
framework to use bounce buffers for certain addresses?

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Fri, Aug 25 2017 at  6:44:27 pm BST, Mason <slash.tmp@free.fr> wrote:
> On 25/08/2017 17:45, Robin Murphy wrote:
>
>> On 25/08/17 16:35, Mason wrote:
>>
>>> On 25/08/2017 17:25, Robin Murphy wrote:
>>>
>>>> On 25/08/17 16:01, Mason wrote:
>>>>
>>>>> Robin wrote a prophetic post back in March:
>>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>>
>>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>>
>>>>>> pci@... {
>>>>>> 	...
>>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>>> }
>>>>>
>>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>>
>>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>>> for other physical addresses).
>>>>>
>>>>> I added the following property to my PCIe node
>>>>>
>>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>>
>>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>>
>>>>> But it does not work. Arg!
>>>>>
>>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>>
>>>>> [ 0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>]
>>>>> (of_dma_configure+0x48/0x234)
>>>>> [ 0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>]
>>>>> (pci_device_add+0xac/0x350)
>>>>> [ 0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>]
>>>>> (pci_scan_single_device+0x90/0xb0)
>>>>> [ 0.520501] [<c02fa488>] (pci_scan_single_device) from
>>>>> [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>>> [ 0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>]
>>>>> (pci_scan_child_bus+0x20/0xf8)
>>>>> [ 0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>]
>>>>> (pci_scan_root_bus_msi+0xcc/0xd8)
>>>>> [ 0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from
>>>>> [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>>> [ 0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>]
>>>>> (pci_host_common_probe+0xc8/0x314)
>>>>> [ 0.520546] [<c0310544>] (pci_host_common_probe) from
>>>>> [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>>> [ 0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>]
>>>>> (platform_drv_probe+0x34/0x6c)
>>>>>
>>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>>> the code jumps to the parent node ("soc")... while my property is
>>>>> attached to the pcie node...
>>>>
>>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>>> doing so in the act of DMA master configuration for a discovered
>>>> *endpoint*. The fact that the "pass the host controller's OF node
>>>> because we don't have one for the endpoint" bodge only works properly
>>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>>
>>>> If your host controller driver needs to discover its windows from DT to
>>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>>> pcie-racar, pcie-xgene, etc. for examples.
>>>
>>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>>> I can use that information to configure BAR0.base and the
>>> region registers.
>>>
>>> But Linux needs to record my settings at some point, right?
>>> Otherwise, how does the DMA framework know that devices can
>>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>>> to use bounce buffers?
>>>
>>> What's preventing the XHCI driver from allocating memory
>>> outside of my "safe" range, and having the DMA framework
>>> blindly map that?
>> 
>> At the moment, nothing. Systems that have physical memory that is not
>> visible in PCI mem space are having a bad time and will not go to space
>> today.
>> 
>> But that bears no relation to your MSI controller getting its doorbell
>> address set appropriately.
>
> OK, so this is what I propose for v11 in order to not
> hard code the MSI doorbell address (e.g. 0xa002e07c)
>
> I add the following property to the pcie node:
>
> 	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>
> I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000
>
> Then in the PCIe driver, I parse dma-ranges.
>
> Consequently
>
> 	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c
>
> Bjorn, Marc, Robin, is that an acceptable solution?

It seems to work, but I still have my doubts about this BAR0.base and
the associated regions. Are these regions so hardcoded in HW that the RC
cannot DMA outside of this 1GB region? Or can it be reconfigured by some
SW agent to cover more RAM, should someone decide that 1GB is on the
"too little" side?

If the former is true, the HW is remarkably busted and/or inflexible. If
the latter is true, then the dma-ranges property feels very fragile, as
it must be kept in sync with the amount of memory that the system has.

	M.
-- 
Jazz is not dead. It just smells funny.

[PATCH v10] PCI: tango: Add MSI controller support

[Marc Zyngier]

On Fri, Aug 25 2017 at  6:44:27 pm BST, Mason <slash.tmp@free.fr> wrote:
> On 25/08/2017 17:45, Robin Murphy wrote:
>
>> On 25/08/17 16:35, Mason wrote:
>>
>>> On 25/08/2017 17:25, Robin Murphy wrote:
>>>
>>>> On 25/08/17 16:01, Mason wrote:
>>>>
>>>>> Robin wrote a prophetic post back in March:
>>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>>
>>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>>
>>>>>> pci at ... {
>>>>>> 	...
>>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>>> }
>>>>>
>>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>>
>>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>>> for other physical addresses).
>>>>>
>>>>> I added the following property to my PCIe node
>>>>>
>>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>>
>>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>>
>>>>> But it does not work. Arg!
>>>>>
>>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>>
>>>>> [ 0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>]
>>>>> (of_dma_configure+0x48/0x234)
>>>>> [ 0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>]
>>>>> (pci_device_add+0xac/0x350)
>>>>> [ 0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>]
>>>>> (pci_scan_single_device+0x90/0xb0)
>>>>> [ 0.520501] [<c02fa488>] (pci_scan_single_device) from
>>>>> [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>>> [ 0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>]
>>>>> (pci_scan_child_bus+0x20/0xf8)
>>>>> [ 0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>]
>>>>> (pci_scan_root_bus_msi+0xcc/0xd8)
>>>>> [ 0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from
>>>>> [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>>> [ 0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>]
>>>>> (pci_host_common_probe+0xc8/0x314)
>>>>> [ 0.520546] [<c0310544>] (pci_host_common_probe) from
>>>>> [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>>> [ 0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>]
>>>>> (platform_drv_probe+0x34/0x6c)
>>>>>
>>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>>> the code jumps to the parent node ("soc")... while my property is
>>>>> attached to the pcie node...
>>>>
>>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>>> doing so in the act of DMA master configuration for a discovered
>>>> *endpoint*. The fact that the "pass the host controller's OF node
>>>> because we don't have one for the endpoint" bodge only works properly
>>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>>
>>>> If your host controller driver needs to discover its windows from DT to
>>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>>> pcie-racar, pcie-xgene, etc. for examples.
>>>
>>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>>> I can use that information to configure BAR0.base and the
>>> region registers.
>>>
>>> But Linux needs to record my settings at some point, right?
>>> Otherwise, how does the DMA framework know that devices can
>>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>>> to use bounce buffers?
>>>
>>> What's preventing the XHCI driver from allocating memory
>>> outside of my "safe" range, and having the DMA framework
>>> blindly map that?
>> 
>> At the moment, nothing. Systems that have physical memory that is not
>> visible in PCI mem space are having a bad time and will not go to space
>> today.
>> 
>> But that bears no relation to your MSI controller getting its doorbell
>> address set appropriately.
>
> OK, so this is what I propose for v11 in order to not
> hard code the MSI doorbell address (e.g. 0xa002e07c)
>
> I add the following property to the pcie node:
>
> 	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>
> I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000
>
> Then in the PCIe driver, I parse dma-ranges.
>
> Consequently
>
> 	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c
>
> Bjorn, Marc, Robin, is that an acceptable solution?

It seems to work, but I still have my doubts about this BAR0.base and
the associated regions. Are these regions so hardcoded in HW that the RC
cannot DMA outside of this 1GB region? Or can it be reconfigured by some
SW agent to cover more RAM, should someone decide that 1GB is on the
"too little" side?

If the former is true, the HW is remarkably busted and/or inflexible. If
the latter is true, then the dma-ranges property feels very fragile, as
it must be kept in sync with the amount of memory that the system has.

	M.
-- 
Jazz is not dead. It just smells funny.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 26/08/2017 15:08, Marc Zyngier wrote:

> On Aug 25 2017 at 18:44, Mason wrote:
>
>> On 25/08/2017 17:45, Robin Murphy wrote:
>>
>>> On 25/08/17 16:35, Mason wrote:
>>>
>>>> On 25/08/2017 17:25, Robin Murphy wrote:
>>>>
>>>>> On 25/08/17 16:01, Mason wrote:
>>>>>
>>>>>> Robin wrote a prophetic post back in March:
>>>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>>>
>>>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>>>
>>>>>>> pci@... {
>>>>>>> 	...
>>>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>>>> }
>>>>>>
>>>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>>>
>>>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>>>> for other physical addresses).
>>>>>>
>>>>>> I added the following property to my PCIe node
>>>>>>
>>>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>>>
>>>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>>>
>>>>>> But it does not work. Arg!
>>>>>>
>>>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>>>
>>>>>> [ 0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>>>>> [ 0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>>>>> [ 0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>>>>> [ 0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>>>> [ 0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>>>>> [ 0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>>>>> [ 0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from> [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>>>> [ 0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>>>>> [ 0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>>>> [ 0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>>>>
>>>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>>>> the code jumps to the parent node ("soc")... while my property is
>>>>>> attached to the pcie node...
>>>>>
>>>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>>>> doing so in the act of DMA master configuration for a discovered
>>>>> *endpoint*. The fact that the "pass the host controller's OF node
>>>>> because we don't have one for the endpoint" bodge only works properly
>>>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>>>
>>>>> If your host controller driver needs to discover its windows from DT to
>>>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>>>> pcie-racar, pcie-xgene, etc. for examples.
>>>>
>>>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>>>> I can use that information to configure BAR0.base and the
>>>> region registers.
>>>>
>>>> But Linux needs to record my settings at some point, right?
>>>> Otherwise, how does the DMA framework know that devices can
>>>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>>>> to use bounce buffers?
>>>>
>>>> What's preventing the XHCI driver from allocating memory
>>>> outside of my "safe" range, and having the DMA framework
>>>> blindly map that?
>>>
>>> At the moment, nothing. Systems that have physical memory that is not
>>> visible in PCI mem space are having a bad time and will not go to space
>>> today.
>>>
>>> But that bears no relation to your MSI controller getting its doorbell
>>> address set appropriately.
>>
>> OK, so this is what I propose for v11 in order to not
>> hard code the MSI doorbell address (e.g. 0xa002e07c)
>>
>> I add the following property to the pcie node:
>>
>> 	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>
>> I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000
>>
>> Then in the PCIe driver, I parse dma-ranges.
>>
>> Consequently
>>
>> 	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c
>>
>> Bjorn, Marc, Robin, is that an acceptable solution?
> 
> It seems to work, but I still have my doubts about this BAR0.base and
> the associated regions. Are these regions so hardcoded in HW that the RC
> cannot DMA outside of this 1GB region? Or can it be reconfigured by some
> SW agent to cover more RAM, should someone decide that 1GB is on the
> "too little" side?
> 
> If the former is true, the HW is remarkably busted and/or inflexible.

This HW block has already been deemed insane because
of the muxing of mem and config space... So you're
late to the party :-)

I wouldn't call the regions "hard-coded" since they are
software-configurable to point anywhere in the CPU bus.
(Although I'm not sure if that's any use, since the DMA
framework seems to expect a 1:1 mapping.)

But the other side (PCI bus) is quite inflexible:
accesses to addresses outside the window defined by BAR0
are silently ignored, no working around that :-(

> If the latter is true, then the dma-ranges property feels very fragile, as
> it must be kept in sync with the amount of memory that the system has.

I'm confused.

As I pointed out, the dma-ranges in the pcie node is
ignored by the DMA framework. And Robin confirmed that
"Systems that have physical memory that is not visible
in PCI mem space are having a bad time and will not go
to space today."

So there are several setups where something is bound
to break:

1) Linux manages more than 1 GB (contiguous)
=> because one region needs to point to the doorbell
area, so 128 MB are wasted.

2) Linux manages non-contiguous memory
=> e.g. 128MB@0x80000000 + 128MB@0xc0000000

That's why I've asked about bounce buffers.

The system I test on boots with mem=512MB

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 26/08/2017 15:08, Marc Zyngier wrote:

> On Aug 25 2017 at 18:44, Mason wrote:
>
>> On 25/08/2017 17:45, Robin Murphy wrote:
>>
>>> On 25/08/17 16:35, Mason wrote:
>>>
>>>> On 25/08/2017 17:25, Robin Murphy wrote:
>>>>
>>>>> On 25/08/17 16:01, Mason wrote:
>>>>>
>>>>>> Robin wrote a prophetic post back in March:
>>>>>> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-March/492965.html
>>>>>>
>>>>>>> The appropriate DT property would be "dma-ranges", i.e.
>>>>>>>
>>>>>>> pci at ... {
>>>>>>> 	...
>>>>>>> 	dma-ranges = <(PCI bus address) (CPU phys address) (size)>;
>>>>>>> }
>>>>>>
>>>>>> The dma-ranges property seems to be exactly what I'm looking for:
>>>>>>
>>>>>> Restrict DMA to the first X MB of RAM (use a bounce buffer
>>>>>> for other physical addresses).
>>>>>>
>>>>>> I added the following property to my PCIe node
>>>>>>
>>>>>>   dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>>>>>
>>>>>> with the intent to create a 1:1 mapping for [0x80000000, 0xa0000000[
>>>>>>
>>>>>> But it does not work. Arg!
>>>>>>
>>>>>> My PCIe controller driver seems to be correctly calling of_dma_get_range:
>>>>>>
>>>>>> [ 0.520469] [<c03d85e8>] (of_dma_get_range) from [<c03d5ad8>] (of_dma_configure+0x48/0x234)
>>>>>> [ 0.520483] [<c03d5ad8>] (of_dma_configure) from [<c02fa154>] (pci_device_add+0xac/0x350)
>>>>>> [ 0.520493] [<c02fa154>] (pci_device_add) from [<c02fa488>] (pci_scan_single_device+0x90/0xb0)
>>>>>> [ 0.520501] [<c02fa488>] (pci_scan_single_device) from [<c02fa500>] (pci_scan_slot+0x58/0x100)
>>>>>> [ 0.520510] [<c02fa500>] (pci_scan_slot) from [<c02fb418>] (pci_scan_child_bus+0x20/0xf8)
>>>>>> [ 0.520519] [<c02fb418>] (pci_scan_child_bus) from [<c02fb6e8>] (pci_scan_root_bus_msi+0xcc/0xd8)
>>>>>> [ 0.520527] [<c02fb6e8>] (pci_scan_root_bus_msi) from> [<c02fb70c>] (pci_scan_root_bus+0x18/0x20)
>>>>>> [ 0.520537] [<c02fb70c>] (pci_scan_root_bus) from [<c0310544>] (pci_host_common_probe+0xc8/0x314)
>>>>>> [ 0.520546] [<c0310544>] (pci_host_common_probe) from [<c0310ce8>] (tango_pcie_probe+0x148/0x350)
>>>>>> [ 0.520557] [<c0310ce8>] (tango_pcie_probe) from [<c034d398>] (platform_drv_probe+0x34/0x6c)
>>>>>>
>>>>>> of_dma_get_range() is called on the pcie node (which is expected)
>>>>>> but after parsing n_addr_cells and n_size_cells in the while loop,
>>>>>> the code jumps to the parent node ("soc")... while my property is
>>>>>> attached to the pcie node...
>>>>>
>>>>> This is not your driver calling of_dma_get_range(), this is the PCI core
>>>>> doing so in the act of DMA master configuration for a discovered
>>>>> *endpoint*. The fact that the "pass the host controller's OF node
>>>>> because we don't have one for the endpoint" bodge only works properly
>>>>> for dma-coherent and not dma-ranges is a known, but irrelevant, problem.
>>>>>
>>>>> If your host controller driver needs to discover its windows from DT to
>>>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>>>> pcie-racar, pcie-xgene, etc. for examples.
>>>>
>>>> Yes, I'm aware that I need to do my own parsing of dma-ranges.
>>>> I can use that information to configure BAR0.base and the
>>>> region registers.
>>>>
>>>> But Linux needs to record my settings at some point, right?
>>>> Otherwise, how does the DMA framework know that devices can
>>>> only reach cpu addresses [0x80000000, 0xa0000000[ and when
>>>> to use bounce buffers?
>>>>
>>>> What's preventing the XHCI driver from allocating memory
>>>> outside of my "safe" range, and having the DMA framework
>>>> blindly map that?
>>>
>>> At the moment, nothing. Systems that have physical memory that is not
>>> visible in PCI mem space are having a bad time and will not go to space
>>> today.
>>>
>>> But that bears no relation to your MSI controller getting its doorbell
>>> address set appropriately.
>>
>> OK, so this is what I propose for v11 in order to not
>> hard code the MSI doorbell address (e.g. 0xa002e07c)
>>
>> I add the following property to the pcie node:
>>
>> 	dma-ranges = <0x0 0x80000000 0x80000000 0x20000000>;
>>
>> I.e. pci_addr = 0x80000000, cpu_addr = 0x80000000, len=0x20000000
>>
>> Then in the PCIe driver, I parse dma-ranges.
>>
>> Consequently
>>
>> 	MSI_doorbell_addr = cpu_addr + len + res.start + 0x7c
>>
>> Bjorn, Marc, Robin, is that an acceptable solution?
> 
> It seems to work, but I still have my doubts about this BAR0.base and
> the associated regions. Are these regions so hardcoded in HW that the RC
> cannot DMA outside of this 1GB region? Or can it be reconfigured by some
> SW agent to cover more RAM, should someone decide that 1GB is on the
> "too little" side?
> 
> If the former is true, the HW is remarkably busted and/or inflexible.

This HW block has already been deemed insane because
of the muxing of mem and config space... So you're
late to the party :-)

I wouldn't call the regions "hard-coded" since they are
software-configurable to point anywhere in the CPU bus.
(Although I'm not sure if that's any use, since the DMA
framework seems to expect a 1:1 mapping.)

But the other side (PCI bus) is quite inflexible:
accesses to addresses outside the window defined by BAR0
are silently ignored, no working around that :-(

> If the latter is true, then the dma-ranges property feels very fragile, as
> it must be kept in sync with the amount of memory that the system has.

I'm confused.

As I pointed out, the dma-ranges in the pcie node is
ignored by the DMA framework. And Robin confirmed that
"Systems that have physical memory that is not visible
in PCI mem space are having a bad time and will not go
to space today."

So there are several setups where something is bound
to break:

1) Linux manages more than 1 GB (contiguous)
=> because one region needs to point to the doorbell
area, so 128 MB are wasted.

2) Linux manages non-contiguous memory
=> e.g. 128MB at 0x80000000 + 128MB at 0xc0000000

That's why I've asked about bounce buffers.

The system I test on boots with mem=512MB

Regards.

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:25, Robin Murphy wrote:

> If your host controller driver needs to discover its windows from DT to
> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
> pcie-rcar, pcie-xgene, etc. for examples.

$ git grep '"dma-ranges"' drivers/pci/host
drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);

These 5 drivers are using the same function: pci_dma_range_parser_init

pci-ftpci100.c:		d3c68e0a7e34a (Linus Walleij     2017-03-12
pci-rcar-gen2.c:	8d598cabf50d8 (Phil Edworthy     2015-11-03
pci-xgene.c:		5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
pcie-iproc.c:		dd9d4e7498de3 (Ray Jui           2016-10-31
pcie-rcar.c:		c25da4778803b (Phil Edworthy     2014-05-12

which seems to be a copy of of_pci_range_parser_init
with "ranges" changed to "dma-ranges"

drivers/of/address.c	29b635c00f3eb (Andrew Murray     2013-05-16

Perhaps pci_dma_range_parser_init() can be factorized?

Rob, Frank, what do you think of the patch below?


diff --git a/drivers/of/address.c b/drivers/of/address.c
index 580bbf6ca2b1..4cfd29e4ee1b 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -232,8 +232,8 @@ int of_pci_address_to_resource(struct device_node *dev, int bar,
 }
 EXPORT_SYMBOL_GPL(of_pci_address_to_resource);
 
-int of_pci_range_parser_init(struct of_pci_range_parser *parser,
-				struct device_node *node)
+static int parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node, const char *name)
 {
 	const int na = 3, ns = 2;
 	int rlen;
@@ -242,7 +242,7 @@ int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 	parser->pna = of_n_addr_cells(node);
 	parser->np = parser->pna + na + ns;
 
-	parser->range = of_get_property(node, "ranges", &rlen);
+	parser->range = of_get_property(node, name, &rlen);
 	if (parser->range == NULL)
 		return -ENOENT;
 
@@ -250,8 +250,21 @@ int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 
 	return 0;
 }
+
+int of_pci_range_parser_init(struct of_pci_range_parser *parser,
+				struct device_node *node)
+{
+	return parser_init(parser, node, "ranges");
+}
 EXPORT_SYMBOL_GPL(of_pci_range_parser_init);
 
+int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+				struct device_node *node)
+{
+	return parser_init(parser, node, "dma-ranges");
+}
+EXPORT_SYMBOL_GPL(of_pci_dma_range_parser_init);
+
 struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
 						struct of_pci_range *range)
 {
diff --git a/include/linux/of_address.h b/include/linux/of_address.h
index 37864734ca50..8beed2de98e9 100644
--- a/include/linux/of_address.h
+++ b/include/linux/of_address.h
@@ -49,6 +49,8 @@ extern const __be32 *of_get_address(struct device_node *dev, int index,
 
 extern int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 			struct device_node *node);
+extern int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node);
 extern struct of_pci_range *of_pci_range_parser_one(
 					struct of_pci_range_parser *parser,
 					struct of_pci_range *range);
@@ -85,7 +87,13 @@ static inline const __be32 *of_get_address(struct device_node *dev, int index,
 static inline int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 			struct device_node *node)
 {
-	return -1;
+	return -ENOSYS;
+}
+
+static inline int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node)
+{
+	return -ENOSYS;
 }
 
 static inline struct of_pci_range *of_pci_range_parser_one(

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 25/08/2017 17:25, Robin Murphy wrote:

> If your host controller driver needs to discover its windows from DT to
> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
> pcie-rcar, pcie-xgene, etc. for examples.

$ git grep '"dma-ranges"' drivers/pci/host
drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);

These 5 drivers are using the same function: pci_dma_range_parser_init

pci-ftpci100.c:		d3c68e0a7e34a (Linus Walleij     2017-03-12
pci-rcar-gen2.c:	8d598cabf50d8 (Phil Edworthy     2015-11-03
pci-xgene.c:		5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
pcie-iproc.c:		dd9d4e7498de3 (Ray Jui           2016-10-31
pcie-rcar.c:		c25da4778803b (Phil Edworthy     2014-05-12

which seems to be a copy of of_pci_range_parser_init
with "ranges" changed to "dma-ranges"

drivers/of/address.c	29b635c00f3eb (Andrew Murray     2013-05-16

Perhaps pci_dma_range_parser_init() can be factorized?

Rob, Frank, what do you think of the patch below?


diff --git a/drivers/of/address.c b/drivers/of/address.c
index 580bbf6ca2b1..4cfd29e4ee1b 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -232,8 +232,8 @@ int of_pci_address_to_resource(struct device_node *dev, int bar,
 }
 EXPORT_SYMBOL_GPL(of_pci_address_to_resource);
 
-int of_pci_range_parser_init(struct of_pci_range_parser *parser,
-				struct device_node *node)
+static int parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node, const char *name)
 {
 	const int na = 3, ns = 2;
 	int rlen;
@@ -242,7 +242,7 @@ int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 	parser->pna = of_n_addr_cells(node);
 	parser->np = parser->pna + na + ns;
 
-	parser->range = of_get_property(node, "ranges", &rlen);
+	parser->range = of_get_property(node, name, &rlen);
 	if (parser->range == NULL)
 		return -ENOENT;
 
@@ -250,8 +250,21 @@ int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 
 	return 0;
 }
+
+int of_pci_range_parser_init(struct of_pci_range_parser *parser,
+				struct device_node *node)
+{
+	return parser_init(parser, node, "ranges");
+}
 EXPORT_SYMBOL_GPL(of_pci_range_parser_init);
 
+int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+				struct device_node *node)
+{
+	return parser_init(parser, node, "dma-ranges");
+}
+EXPORT_SYMBOL_GPL(of_pci_dma_range_parser_init);
+
 struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
 						struct of_pci_range *range)
 {
diff --git a/include/linux/of_address.h b/include/linux/of_address.h
index 37864734ca50..8beed2de98e9 100644
--- a/include/linux/of_address.h
+++ b/include/linux/of_address.h
@@ -49,6 +49,8 @@ extern const __be32 *of_get_address(struct device_node *dev, int index,
 
 extern int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 			struct device_node *node);
+extern int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node);
 extern struct of_pci_range *of_pci_range_parser_one(
 					struct of_pci_range_parser *parser,
 					struct of_pci_range *range);
@@ -85,7 +87,13 @@ static inline const __be32 *of_get_address(struct device_node *dev, int index,
 static inline int of_pci_range_parser_init(struct of_pci_range_parser *parser,
 			struct device_node *node)
 {
-	return -1;
+	return -ENOSYS;
+}
+
+static inline int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+			struct device_node *node)
+{
+	return -ENOSYS;
 }
 
 static inline struct of_pci_range *of_pci_range_parser_one(

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Rob Herring]

On Mon, Aug 28, 2017 at 11:13 AM, Mason <slash.tmp@free.fr> wrote:
> On 25/08/2017 17:25, Robin Murphy wrote:
>
>> If your host controller driver needs to discover its windows from DT to
>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>> pcie-rcar, pcie-xgene, etc. for examples.
>
> $ git grep '"dma-ranges"' drivers/pci/host
> drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>
> These 5 drivers are using the same function: pci_dma_range_parser_init
>
> pci-ftpci100.c:         d3c68e0a7e34a (Linus Walleij     2017-03-12
> pci-rcar-gen2.c:        8d598cabf50d8 (Phil Edworthy     2015-11-03
> pci-xgene.c:            5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
> pcie-iproc.c:           dd9d4e7498de3 (Ray Jui           2016-10-31
> pcie-rcar.c:            c25da4778803b (Phil Edworthy     2014-05-12
>
> which seems to be a copy of of_pci_range_parser_init
> with "ranges" changed to "dma-ranges"
>
> drivers/of/address.c    29b635c00f3eb (Andrew Murray     2013-05-16
>
> Perhaps pci_dma_range_parser_init() can be factorized?
>
> Rob, Frank, what do you think of the patch below?

I thought I had commented (maybe that was on IRC), but looks fine.

This series should also fix a build failure on rcar with !OF, so can
you send a proper patch?

Rob

[PATCH v10] PCI: tango: Add MSI controller support

[Rob Herring]

On Mon, Aug 28, 2017 at 11:13 AM, Mason <slash.tmp@free.fr> wrote:
> On 25/08/2017 17:25, Robin Murphy wrote:
>
>> If your host controller driver needs to discover its windows from DT to
>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>> pcie-rcar, pcie-xgene, etc. for examples.
>
> $ git grep '"dma-ranges"' drivers/pci/host
> drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
> drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>
> These 5 drivers are using the same function: pci_dma_range_parser_init
>
> pci-ftpci100.c:         d3c68e0a7e34a (Linus Walleij     2017-03-12
> pci-rcar-gen2.c:        8d598cabf50d8 (Phil Edworthy     2015-11-03
> pci-xgene.c:            5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
> pcie-iproc.c:           dd9d4e7498de3 (Ray Jui           2016-10-31
> pcie-rcar.c:            c25da4778803b (Phil Edworthy     2014-05-12
>
> which seems to be a copy of of_pci_range_parser_init
> with "ranges" changed to "dma-ranges"
>
> drivers/of/address.c    29b635c00f3eb (Andrew Murray     2013-05-16
>
> Perhaps pci_dma_range_parser_init() can be factorized?
>
> Rob, Frank, what do you think of the patch below?

I thought I had commented (maybe that was on IRC), but looks fine.

This series should also fix a build failure on rcar with !OF, so can
you send a proper patch?

Rob

Re: [PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 18/09/2017 23:30, Rob Herring wrote:
> On Aug 28, 2017 at 11:13 AM, Mason wrote:
>> On 25/08/2017 17:25, Robin Murphy wrote:
>>
>>> If your host controller driver needs to discover its windows from DT to
>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>> pcie-rcar, pcie-xgene, etc. for examples.
>>
>> $ git grep '"dma-ranges"' drivers/pci/host
>> drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>>
>> These 5 drivers are using the same function: pci_dma_range_parser_init
>>
>> pci-ftpci100.c:         d3c68e0a7e34a (Linus Walleij     2017-03-12
>> pci-rcar-gen2.c:        8d598cabf50d8 (Phil Edworthy     2015-11-03
>> pci-xgene.c:            5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
>> pcie-iproc.c:           dd9d4e7498de3 (Ray Jui           2016-10-31
>> pcie-rcar.c:            c25da4778803b (Phil Edworthy     2014-05-12
>>
>> which seems to be a copy of of_pci_range_parser_init
>> with "ranges" changed to "dma-ranges"
>>
>> drivers/of/address.c    29b635c00f3eb (Andrew Murray     2013-05-16
>>
>> Perhaps pci_dma_range_parser_init() can be factorized?
>>
>> Rob, Frank, what do you think of the patch below?
> 
> I thought I had commented (maybe that was on IRC), but looks fine.
> 
> This series should also fix a build failure on rcar with !OF, so can
> you send a proper patch?

You're not misremembering, you did review the formal patch ;-)

"[PATCH 0/3] Tango PCIe host bridge MSI support + required API"
https://patchwork.kernel.org/patch/9927539/
https://patchwork.kernel.org/patch/9927541/

Regards.

[PATCH v10] PCI: tango: Add MSI controller support

[Mason]

On 18/09/2017 23:30, Rob Herring wrote:
> On Aug 28, 2017 at 11:13 AM, Mason wrote:
>> On 25/08/2017 17:25, Robin Murphy wrote:
>>
>>> If your host controller driver needs to discover its windows from DT to
>>> configure *itself*, it needs to parse dma-ranges itself; see pcie-iproc,
>>> pcie-rcar, pcie-xgene, etc. for examples.
>>
>> $ git grep '"dma-ranges"' drivers/pci/host
>> drivers/pci/host/pci-ftpci100.c:        parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pci-rcar-gen2.c:       parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pci-xgene.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pcie-iproc.c:  parser->range = of_get_property(node, "dma-ranges", &rlen);
>> drivers/pci/host/pcie-rcar.c:   parser->range = of_get_property(node, "dma-ranges", &rlen);
>>
>> These 5 drivers are using the same function: pci_dma_range_parser_init
>>
>> pci-ftpci100.c:         d3c68e0a7e34a (Linus Walleij     2017-03-12
>> pci-rcar-gen2.c:        8d598cabf50d8 (Phil Edworthy     2015-11-03
>> pci-xgene.c:            5f6b6ccdbe1cd (Tanmay Inamdar    2014-10-01
>> pcie-iproc.c:           dd9d4e7498de3 (Ray Jui           2016-10-31
>> pcie-rcar.c:            c25da4778803b (Phil Edworthy     2014-05-12
>>
>> which seems to be a copy of of_pci_range_parser_init
>> with "ranges" changed to "dma-ranges"
>>
>> drivers/of/address.c    29b635c00f3eb (Andrew Murray     2013-05-16
>>
>> Perhaps pci_dma_range_parser_init() can be factorized?
>>
>> Rob, Frank, what do you think of the patch below?
> 
> I thought I had commented (maybe that was on IRC), but looks fine.
> 
> This series should also fix a build failure on rcar with !OF, so can
> you send a proper patch?

You're not misremembering, you did review the formal patch ;-)

"[PATCH 0/3] Tango PCIe host bridge MSI support + required API"
https://patchwork.kernel.org/patch/9927539/
https://patchwork.kernel.org/patch/9927541/

Regards.