[PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Thierry Reding]

From: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>

This patch makes the existing pci_host_bridge structure a proper device
that is usable by PCI host drivers in a more standard way. In addition
to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
and pci_create_root_bus interfaces, this unfortunately means having to
add yet another interface doing basically the same thing, and add some
extra code in the initial step.

However, this time it's more likely to be extensible enough that we
won't have to do another one again in the future, and we should be
able to reduce code much more as a result.

The main idea is to pull the allocation of 'struct pci_host_bridge' out
of the registration, and let individual host drivers and architecture
code fill the members before calling the registration function.

There are a number of things we can do based on this:

* Use a single memory allocation for the driver-specific structure
  and the generic PCI host bridge
* consolidate the contents of driver specific structures by moving
  them into pci_host_bridge
* Add a consistent interface for removing a PCI host bridge again
  when unloading a host driver module
* Replace the architecture specific __weak pcibios_* functions with
  callbacks in a pci_host_bridge device
* Move common boilerplate code from host drivers into the generic
  function, based on contents of the structure
* Extend pci_host_bridge with additional members when needed without
  having to add arguments to pci_scan_*.
* Move members of struct pci_bus into pci_host_bridge to avoid
  having lots of identical copies.

As mentioned in a previous email, one open question is whether we want
to export a function for allocating a pci_host_bridge device in
combination with the per-device structure or let the driver itself
call kzalloc.

Changes in v3 (Thierry Reding):
- swap out pci_host_bridge_init() for pci_alloc_host_bridge() with an
  extra parameter specifying the size of the driver's private data
- rename pci_host_bridge_register() to pci_register_host_bridge() for
  more consistency with existing functions
- split patches into smaller chunks to make diff more readable

Changes in v2 (Thierry Reding):
- add a pci_host_bridge_init() helper that drivers can use to perform
  all the necessary steps to initialize the bridge
- rename pci_register_host() to pci_host_bridge_register() to reflect
  the naming used by other functions
- plug memory leak on registration failure

Signed-off-by: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/probe.c | 238 +++++++++++++++++++++++++++++++---------------------
 include/linux/pci.h |   4 +
 2 files changed, 145 insertions(+), 97 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 104c46d53121..99f503c3ab81 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,7 +521,7 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(struct pci_bus *b)
+static struct pci_host_bridge *pci_alloc_host_bridge(void)
 {
 	struct pci_host_bridge *bridge;
 
@@ -530,7 +530,7 @@ static struct pci_host_bridge *pci_alloc_host_bridge(struct pci_bus *b)
 		return NULL;
 
 	INIT_LIST_HEAD(&bridge->windows);
-	bridge->bus = b;
+
 	return bridge;
 }
 
@@ -717,6 +717,122 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus)
 	dev_set_msi_domain(&bus->dev, d);
 }
 
+static int pci_register_host_bridge(struct pci_host_bridge *bridge)
+{
+	struct device *parent = bridge->dev.parent;
+	struct resource_entry *window, *n;
+	struct pci_bus *bus, *b;
+	resource_size_t offset;
+	LIST_HEAD(resources);
+	struct resource *res;
+	char addr[64], *fmt;
+	const char *name;
+	int err;
+
+	bus = pci_alloc_bus(NULL);
+	if (!bus)
+		return -ENOMEM;
+
+	bridge->bus = bus;
+
+	/* temporarily move resources off the list */
+	list_splice_init(&bridge->windows, &resources);
+	bus->sysdata = bridge->sysdata;
+	bus->msi = bridge->msi;
+	bus->ops = bridge->ops;
+	bus->number = bus->busn_res.start = bridge->busnr;
+#ifdef CONFIG_PCI_DOMAINS_GENERIC
+	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
+#endif
+
+	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
+	if (b) {
+		/* If we already got to this bus through a different bridge, ignore it */
+		dev_dbg(&b->dev, "bus already known\n");
+		err = -EEXIST;
+		goto free;
+	}
+
+	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
+		     bridge->busnr);
+
+	err = pcibios_root_bridge_prepare(bridge);
+	if (err)
+		goto free;
+
+	err = device_register(&bridge->dev);
+	if (err)
+		put_device(&bridge->dev);
+
+	bus->bridge = get_device(&bridge->dev);
+	device_enable_async_suspend(bus->bridge);
+	pci_set_bus_of_node(bus);
+	pci_set_bus_msi_domain(bus);
+
+	if (!parent)
+		set_dev_node(bus->bridge, pcibus_to_node(bus));
+
+	bus->dev.class = &pcibus_class;
+	bus->dev.parent = bus->bridge;
+
+	dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
+	name = dev_name(&bus->dev);
+
+	err = device_register(&bus->dev);
+	if (err)
+		goto unregister;
+
+	pcibios_add_bus(bus);
+
+	/* Create legacy_io and legacy_mem files for this bus */
+	pci_create_legacy_files(bus);
+
+	if (parent)
+		dev_info(parent, "PCI host bridge to bus %s\n", name);
+	else
+		pr_info("PCI host bridge to bus %s\n", name);
+
+	/* Add initial resources to the bus */
+	resource_list_for_each_entry_safe(window, n, &resources) {
+		list_move_tail(&window->node, &bridge->windows);
+		offset = window->offset;
+		res = window->res;
+
+		if (res->flags & IORESOURCE_BUS)
+			pci_bus_insert_busn_res(bus, bus->number, res->end);
+		else
+			pci_bus_add_resource(bus, res, 0);
+
+		if (offset) {
+			if (resource_type(res) == IORESOURCE_IO)
+				fmt = " (bus address [%#06llx-%#06llx])";
+			else
+				fmt = " (bus address [%#010llx-%#010llx])";
+
+			snprintf(addr, sizeof(addr), fmt,
+				 (unsigned long long)(res->start - offset),
+				 (unsigned long long)(res->end - offset));
+		} else
+			addr[0] = '\0';
+
+		dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
+	}
+
+	down_write(&pci_bus_sem);
+	list_add_tail(&bus->node, &pci_root_buses);
+	up_write(&pci_bus_sem);
+
+	return 0;
+
+unregister:
+	put_device(&bridge->dev);
+	device_unregister(&bridge->dev);
+
+free:
+	kfree(bus);
+	return err;
+}
+
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
 {
@@ -2156,113 +2272,43 @@ void __weak pcibios_remove_bus(struct pci_bus *bus)
 {
 }
 
-struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
-		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+static struct pci_bus *pci_create_root_bus_msi(struct device *parent,
+		int bus, struct pci_ops *ops, void *sysdata,
+		struct list_head *resources, struct msi_controller *msi)
 {
 	int error;
 	struct pci_host_bridge *bridge;
-	struct pci_bus *b, *b2;
-	struct resource_entry *window, *n;
-	struct resource *res;
-	resource_size_t offset;
-	char bus_addr[64];
-	char *fmt;
-
-	b = pci_alloc_bus(NULL);
-	if (!b)
-		return NULL;
 
-	b->sysdata = sysdata;
-	b->ops = ops;
-	b->number = b->busn_res.start = bus;
-#ifdef CONFIG_PCI_DOMAINS_GENERIC
-	b->domain_nr = pci_bus_find_domain_nr(b, parent);
-#endif
-	b2 = pci_find_bus(pci_domain_nr(b), bus);
-	if (b2) {
-		/* If we already got to this bus through a different bridge, ignore it */
-		dev_dbg(&b2->dev, "bus already known\n");
-		goto err_out;
-	}
-
-	bridge = pci_alloc_host_bridge(b);
+	bridge = pci_alloc_host_bridge();
 	if (!bridge)
-		goto err_out;
+		return NULL;
 
 	bridge->dev.parent = parent;
 	bridge->dev.release = pci_release_host_bridge_dev;
-	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(b), bus);
-	error = pcibios_root_bridge_prepare(bridge);
-	if (error) {
-		kfree(bridge);
-		goto err_out;
-	}
-
-	error = device_register(&bridge->dev);
-	if (error) {
-		put_device(&bridge->dev);
-		goto err_out;
-	}
-	b->bridge = get_device(&bridge->dev);
-	device_enable_async_suspend(b->bridge);
-	pci_set_bus_of_node(b);
-	pci_set_bus_msi_domain(b);
 
-	if (!parent)
-		set_dev_node(b->bridge, pcibus_to_node(b));
-
-	b->dev.class = &pcibus_class;
-	b->dev.parent = b->bridge;
-	dev_set_name(&b->dev, "%04x:%02x", pci_domain_nr(b), bus);
-	error = device_register(&b->dev);
-	if (error)
-		goto class_dev_reg_err;
+	list_splice_init(resources, &bridge->windows);
+	bridge->sysdata = sysdata;
+	bridge->busnr = bus;
+	bridge->ops = ops;
+	bridge->msi = msi;
 
-	pcibios_add_bus(b);
-
-	/* Create legacy_io and legacy_mem files for this bus */
-	pci_create_legacy_files(b);
-
-	if (parent)
-		dev_info(parent, "PCI host bridge to bus %s\n", dev_name(&b->dev));
-	else
-		printk(KERN_INFO "PCI host bridge to bus %s\n", dev_name(&b->dev));
-
-	/* Add initial resources to the bus */
-	resource_list_for_each_entry_safe(window, n, resources) {
-		list_move_tail(&window->node, &bridge->windows);
-		res = window->res;
-		offset = window->offset;
-		if (res->flags & IORESOURCE_BUS)
-			pci_bus_insert_busn_res(b, bus, res->end);
-		else
-			pci_bus_add_resource(b, res, 0);
-		if (offset) {
-			if (resource_type(res) == IORESOURCE_IO)
-				fmt = " (bus address [%#06llx-%#06llx])";
-			else
-				fmt = " (bus address [%#010llx-%#010llx])";
-			snprintf(bus_addr, sizeof(bus_addr), fmt,
-				 (unsigned long long) (res->start - offset),
-				 (unsigned long long) (res->end - offset));
-		} else
-			bus_addr[0] = '\0';
-		dev_info(&b->dev, "root bus resource %pR%s\n", res, bus_addr);
-	}
+	error = pci_register_host_bridge(bridge);
+	if (error < 0)
+		goto err_out;
 
-	down_write(&pci_bus_sem);
-	list_add_tail(&b->node, &pci_root_buses);
-	up_write(&pci_bus_sem);
+	return bridge->bus;
 
-	return b;
-
-class_dev_reg_err:
-	put_device(&bridge->dev);
-	device_unregister(&bridge->dev);
 err_out:
-	kfree(b);
+	kfree(bridge);
 	return NULL;
 }
+
+struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
+		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+{
+	return pci_create_root_bus_msi(parent, bus, ops, sysdata, resources,
+				       NULL);
+}
 EXPORT_SYMBOL_GPL(pci_create_root_bus);
 
 int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
@@ -2343,12 +2389,10 @@ struct pci_bus *pci_scan_root_bus_msi(struct device *parent, int bus,
 			break;
 		}
 
-	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
+	b = pci_create_root_bus_msi(parent, bus, ops, sysdata, resources, msi);
 	if (!b)
 		return NULL;
 
-	b->msi = msi;
-
 	if (!found) {
 		dev_info(&b->dev,
 		 "No busn resource found for root bus, will use [bus %02x-ff]\n",
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 30d6c162e053..ff16e44d8f84 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -420,9 +420,13 @@ static inline int pci_channel_offline(struct pci_dev *pdev)
 struct pci_host_bridge {
 	struct device dev;
 	struct pci_bus *bus;		/* root bus */
+	struct pci_ops *ops;
+	void *sysdata;
+	int busnr;
 	struct list_head windows;	/* resource_entry */
 	void (*release_fn)(struct pci_host_bridge *);
 	void *release_data;
+	struct msi_controller *msi;
 	unsigned int ignore_reset_delay:1;	/* for entire hierarchy */
 	/* Resource alignment requirements */
 	resource_size_t (*align_resource)(struct pci_dev *dev,
-- 
2.10.2

[PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Thierry Reding]

From: Arnd Bergmann <arnd@arndb.de>

This patch makes the existing pci_host_bridge structure a proper device
that is usable by PCI host drivers in a more standard way. In addition
to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
and pci_create_root_bus interfaces, this unfortunately means having to
add yet another interface doing basically the same thing, and add some
extra code in the initial step.

However, this time it's more likely to be extensible enough that we
won't have to do another one again in the future, and we should be
able to reduce code much more as a result.

The main idea is to pull the allocation of 'struct pci_host_bridge' out
of the registration, and let individual host drivers and architecture
code fill the members before calling the registration function.

There are a number of things we can do based on this:

* Use a single memory allocation for the driver-specific structure
  and the generic PCI host bridge
* consolidate the contents of driver specific structures by moving
  them into pci_host_bridge
* Add a consistent interface for removing a PCI host bridge again
  when unloading a host driver module
* Replace the architecture specific __weak pcibios_* functions with
  callbacks in a pci_host_bridge device
* Move common boilerplate code from host drivers into the generic
  function, based on contents of the structure
* Extend pci_host_bridge with additional members when needed without
  having to add arguments to pci_scan_*.
* Move members of struct pci_bus into pci_host_bridge to avoid
  having lots of identical copies.

As mentioned in a previous email, one open question is whether we want
to export a function for allocating a pci_host_bridge device in
combination with the per-device structure or let the driver itself
call kzalloc.

Changes in v3 (Thierry Reding):
- swap out pci_host_bridge_init() for pci_alloc_host_bridge() with an
  extra parameter specifying the size of the driver's private data
- rename pci_host_bridge_register() to pci_register_host_bridge() for
  more consistency with existing functions
- split patches into smaller chunks to make diff more readable

Changes in v2 (Thierry Reding):
- add a pci_host_bridge_init() helper that drivers can use to perform
  all the necessary steps to initialize the bridge
- rename pci_register_host() to pci_host_bridge_register() to reflect
  the naming used by other functions
- plug memory leak on registration failure

Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/probe.c | 238 +++++++++++++++++++++++++++++++---------------------
 include/linux/pci.h |   4 +
 2 files changed, 145 insertions(+), 97 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 104c46d53121..99f503c3ab81 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,7 +521,7 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(struct pci_bus *b)
+static struct pci_host_bridge *pci_alloc_host_bridge(void)
 {
 	struct pci_host_bridge *bridge;
 
@@ -530,7 +530,7 @@ static struct pci_host_bridge *pci_alloc_host_bridge(struct pci_bus *b)
 		return NULL;
 
 	INIT_LIST_HEAD(&bridge->windows);
-	bridge->bus = b;
+
 	return bridge;
 }
 
@@ -717,6 +717,122 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus)
 	dev_set_msi_domain(&bus->dev, d);
 }
 
+static int pci_register_host_bridge(struct pci_host_bridge *bridge)
+{
+	struct device *parent = bridge->dev.parent;
+	struct resource_entry *window, *n;
+	struct pci_bus *bus, *b;
+	resource_size_t offset;
+	LIST_HEAD(resources);
+	struct resource *res;
+	char addr[64], *fmt;
+	const char *name;
+	int err;
+
+	bus = pci_alloc_bus(NULL);
+	if (!bus)
+		return -ENOMEM;
+
+	bridge->bus = bus;
+
+	/* temporarily move resources off the list */
+	list_splice_init(&bridge->windows, &resources);
+	bus->sysdata = bridge->sysdata;
+	bus->msi = bridge->msi;
+	bus->ops = bridge->ops;
+	bus->number = bus->busn_res.start = bridge->busnr;
+#ifdef CONFIG_PCI_DOMAINS_GENERIC
+	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
+#endif
+
+	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
+	if (b) {
+		/* If we already got to this bus through a different bridge, ignore it */
+		dev_dbg(&b->dev, "bus already known\n");
+		err = -EEXIST;
+		goto free;
+	}
+
+	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
+		     bridge->busnr);
+
+	err = pcibios_root_bridge_prepare(bridge);
+	if (err)
+		goto free;
+
+	err = device_register(&bridge->dev);
+	if (err)
+		put_device(&bridge->dev);
+
+	bus->bridge = get_device(&bridge->dev);
+	device_enable_async_suspend(bus->bridge);
+	pci_set_bus_of_node(bus);
+	pci_set_bus_msi_domain(bus);
+
+	if (!parent)
+		set_dev_node(bus->bridge, pcibus_to_node(bus));
+
+	bus->dev.class = &pcibus_class;
+	bus->dev.parent = bus->bridge;
+
+	dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
+	name = dev_name(&bus->dev);
+
+	err = device_register(&bus->dev);
+	if (err)
+		goto unregister;
+
+	pcibios_add_bus(bus);
+
+	/* Create legacy_io and legacy_mem files for this bus */
+	pci_create_legacy_files(bus);
+
+	if (parent)
+		dev_info(parent, "PCI host bridge to bus %s\n", name);
+	else
+		pr_info("PCI host bridge to bus %s\n", name);
+
+	/* Add initial resources to the bus */
+	resource_list_for_each_entry_safe(window, n, &resources) {
+		list_move_tail(&window->node, &bridge->windows);
+		offset = window->offset;
+		res = window->res;
+
+		if (res->flags & IORESOURCE_BUS)
+			pci_bus_insert_busn_res(bus, bus->number, res->end);
+		else
+			pci_bus_add_resource(bus, res, 0);
+
+		if (offset) {
+			if (resource_type(res) == IORESOURCE_IO)
+				fmt = " (bus address [%#06llx-%#06llx])";
+			else
+				fmt = " (bus address [%#010llx-%#010llx])";
+
+			snprintf(addr, sizeof(addr), fmt,
+				 (unsigned long long)(res->start - offset),
+				 (unsigned long long)(res->end - offset));
+		} else
+			addr[0] = '\0';
+
+		dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
+	}
+
+	down_write(&pci_bus_sem);
+	list_add_tail(&bus->node, &pci_root_buses);
+	up_write(&pci_bus_sem);
+
+	return 0;
+
+unregister:
+	put_device(&bridge->dev);
+	device_unregister(&bridge->dev);
+
+free:
+	kfree(bus);
+	return err;
+}
+
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
 {
@@ -2156,113 +2272,43 @@ void __weak pcibios_remove_bus(struct pci_bus *bus)
 {
 }
 
-struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
-		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+static struct pci_bus *pci_create_root_bus_msi(struct device *parent,
+		int bus, struct pci_ops *ops, void *sysdata,
+		struct list_head *resources, struct msi_controller *msi)
 {
 	int error;
 	struct pci_host_bridge *bridge;
-	struct pci_bus *b, *b2;
-	struct resource_entry *window, *n;
-	struct resource *res;
-	resource_size_t offset;
-	char bus_addr[64];
-	char *fmt;
-
-	b = pci_alloc_bus(NULL);
-	if (!b)
-		return NULL;
 
-	b->sysdata = sysdata;
-	b->ops = ops;
-	b->number = b->busn_res.start = bus;
-#ifdef CONFIG_PCI_DOMAINS_GENERIC
-	b->domain_nr = pci_bus_find_domain_nr(b, parent);
-#endif
-	b2 = pci_find_bus(pci_domain_nr(b), bus);
-	if (b2) {
-		/* If we already got to this bus through a different bridge, ignore it */
-		dev_dbg(&b2->dev, "bus already known\n");
-		goto err_out;
-	}
-
-	bridge = pci_alloc_host_bridge(b);
+	bridge = pci_alloc_host_bridge();
 	if (!bridge)
-		goto err_out;
+		return NULL;
 
 	bridge->dev.parent = parent;
 	bridge->dev.release = pci_release_host_bridge_dev;
-	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(b), bus);
-	error = pcibios_root_bridge_prepare(bridge);
-	if (error) {
-		kfree(bridge);
-		goto err_out;
-	}
-
-	error = device_register(&bridge->dev);
-	if (error) {
-		put_device(&bridge->dev);
-		goto err_out;
-	}
-	b->bridge = get_device(&bridge->dev);
-	device_enable_async_suspend(b->bridge);
-	pci_set_bus_of_node(b);
-	pci_set_bus_msi_domain(b);
 
-	if (!parent)
-		set_dev_node(b->bridge, pcibus_to_node(b));
-
-	b->dev.class = &pcibus_class;
-	b->dev.parent = b->bridge;
-	dev_set_name(&b->dev, "%04x:%02x", pci_domain_nr(b), bus);
-	error = device_register(&b->dev);
-	if (error)
-		goto class_dev_reg_err;
+	list_splice_init(resources, &bridge->windows);
+	bridge->sysdata = sysdata;
+	bridge->busnr = bus;
+	bridge->ops = ops;
+	bridge->msi = msi;
 
-	pcibios_add_bus(b);
-
-	/* Create legacy_io and legacy_mem files for this bus */
-	pci_create_legacy_files(b);
-
-	if (parent)
-		dev_info(parent, "PCI host bridge to bus %s\n", dev_name(&b->dev));
-	else
-		printk(KERN_INFO "PCI host bridge to bus %s\n", dev_name(&b->dev));
-
-	/* Add initial resources to the bus */
-	resource_list_for_each_entry_safe(window, n, resources) {
-		list_move_tail(&window->node, &bridge->windows);
-		res = window->res;
-		offset = window->offset;
-		if (res->flags & IORESOURCE_BUS)
-			pci_bus_insert_busn_res(b, bus, res->end);
-		else
-			pci_bus_add_resource(b, res, 0);
-		if (offset) {
-			if (resource_type(res) == IORESOURCE_IO)
-				fmt = " (bus address [%#06llx-%#06llx])";
-			else
-				fmt = " (bus address [%#010llx-%#010llx])";
-			snprintf(bus_addr, sizeof(bus_addr), fmt,
-				 (unsigned long long) (res->start - offset),
-				 (unsigned long long) (res->end - offset));
-		} else
-			bus_addr[0] = '\0';
-		dev_info(&b->dev, "root bus resource %pR%s\n", res, bus_addr);
-	}
+	error = pci_register_host_bridge(bridge);
+	if (error < 0)
+		goto err_out;
 
-	down_write(&pci_bus_sem);
-	list_add_tail(&b->node, &pci_root_buses);
-	up_write(&pci_bus_sem);
+	return bridge->bus;
 
-	return b;
-
-class_dev_reg_err:
-	put_device(&bridge->dev);
-	device_unregister(&bridge->dev);
 err_out:
-	kfree(b);
+	kfree(bridge);
 	return NULL;
 }
+
+struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
+		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+{
+	return pci_create_root_bus_msi(parent, bus, ops, sysdata, resources,
+				       NULL);
+}
 EXPORT_SYMBOL_GPL(pci_create_root_bus);
 
 int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
@@ -2343,12 +2389,10 @@ struct pci_bus *pci_scan_root_bus_msi(struct device *parent, int bus,
 			break;
 		}
 
-	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
+	b = pci_create_root_bus_msi(parent, bus, ops, sysdata, resources, msi);
 	if (!b)
 		return NULL;
 
-	b->msi = msi;
-
 	if (!found) {
 		dev_info(&b->dev,
 		 "No busn resource found for root bus, will use [bus %02x-ff]\n",
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 30d6c162e053..ff16e44d8f84 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -420,9 +420,13 @@ static inline int pci_channel_offline(struct pci_dev *pdev)
 struct pci_host_bridge {
 	struct device dev;
 	struct pci_bus *bus;		/* root bus */
+	struct pci_ops *ops;
+	void *sysdata;
+	int busnr;
 	struct list_head windows;	/* resource_entry */
 	void (*release_fn)(struct pci_host_bridge *);
 	void *release_data;
+	struct msi_controller *msi;
 	unsigned int ignore_reset_delay:1;	/* for entire hierarchy */
 	/* Resource alignment requirements */
 	resource_size_t (*align_resource)(struct pci_dev *dev,
-- 
2.10.2

[PATCH v4 02/10] PCI: Allow driver-specific data in host bridge

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Provide a way to allocate driver-specific data along with a PCI host
bridge structure. The bridge's ->private field points to this data.

Changes in v4:
- use zero-sized, cache-aligned structure field to store private data
- provide static inline functions to obtain private data and upcast from
  private data to struct pci_host_bridge

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

---
 drivers/pci/probe.c |  6 +++---
 include/linux/pci.h | 11 +++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 99f503c3ab81..912b22ed96dd 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,11 +521,11 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(void)
+static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 {
 	struct pci_host_bridge *bridge;
 
-	bridge = kzalloc(sizeof(*bridge), GFP_KERNEL);
+	bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
 	if (!bridge)
 		return NULL;
 
@@ -2279,7 +2279,7 @@ static struct pci_bus *pci_create_root_bus_msi(struct device *parent,
 	int error;
 	struct pci_host_bridge *bridge;
 
-	bridge = pci_alloc_host_bridge();
+	bridge = pci_alloc_host_bridge(0);
 	if (!bridge)
 		return NULL;
 
diff --git a/include/linux/pci.h b/include/linux/pci.h
index ff16e44d8f84..e9ba95cbfd62 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -434,10 +434,21 @@ struct pci_host_bridge {
 			resource_size_t start,
 			resource_size_t size,
 			resource_size_t align);
+	unsigned long private[0] ____cacheline_aligned;
 };
 
 #define	to_pci_host_bridge(n) container_of(n, struct pci_host_bridge, dev)
 
+static inline void *pci_host_bridge_priv(struct pci_host_bridge *bridge)
+{
+	return (void *)bridge->private;
+}
+
+static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
+{
+	return container_of(priv, struct pci_host_bridge, private);
+}
+
 struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);
 
 void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
-- 
2.10.2

[PATCH v4 02/10] PCI: Allow driver-specific data in host bridge

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Provide a way to allocate driver-specific data along with a PCI host
bridge structure. The bridge's ->private field points to this data.

Changes in v4:
- use zero-sized, cache-aligned structure field to store private data
- provide static inline functions to obtain private data and upcast from
  private data to struct pci_host_bridge

Signed-off-by: Thierry Reding <treding@nvidia.com>

---
 drivers/pci/probe.c |  6 +++---
 include/linux/pci.h | 11 +++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 99f503c3ab81..912b22ed96dd 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,11 +521,11 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(void)
+static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 {
 	struct pci_host_bridge *bridge;
 
-	bridge = kzalloc(sizeof(*bridge), GFP_KERNEL);
+	bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
 	if (!bridge)
 		return NULL;
 
@@ -2279,7 +2279,7 @@ static struct pci_bus *pci_create_root_bus_msi(struct device *parent,
 	int error;
 	struct pci_host_bridge *bridge;
 
-	bridge = pci_alloc_host_bridge();
+	bridge = pci_alloc_host_bridge(0);
 	if (!bridge)
 		return NULL;
 
diff --git a/include/linux/pci.h b/include/linux/pci.h
index ff16e44d8f84..e9ba95cbfd62 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -434,10 +434,21 @@ struct pci_host_bridge {
 			resource_size_t start,
 			resource_size_t size,
 			resource_size_t align);
+	unsigned long private[0] ____cacheline_aligned;
 };
 
 #define	to_pci_host_bridge(n) container_of(n, struct pci_host_bridge, dev)
 
+static inline void *pci_host_bridge_priv(struct pci_host_bridge *bridge)
+{
+	return (void *)bridge->private;
+}
+
+static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
+{
+	return container_of(priv, struct pci_host_bridge, private);
+}
+
 struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);
 
 void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
-- 
2.10.2

[PATCH v4 03/10] PCI: Make host bridge interface publicly available

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Allow PCI host bridge drivers to use the new host bridge interfaces to
register their host bridge.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/probe.c | 6 ++++--
 include/linux/pci.h | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 912b22ed96dd..eeb48d3ce80c 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,7 +521,7 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 {
 	struct pci_host_bridge *bridge;
 
@@ -533,6 +533,7 @@ static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 
 	return bridge;
 }
+EXPORT_SYMBOL(pci_alloc_host_bridge);
 
 static const unsigned char pcix_bus_speed[] = {
 	PCI_SPEED_UNKNOWN,		/* 0 */
@@ -717,7 +718,7 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus)
 	dev_set_msi_domain(&bus->dev, d);
 }
 
-static int pci_register_host_bridge(struct pci_host_bridge *bridge)
+int pci_register_host_bridge(struct pci_host_bridge *bridge)
 {
 	struct device *parent = bridge->dev.parent;
 	struct resource_entry *window, *n;
@@ -832,6 +833,7 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 	kfree(bus);
 	return err;
 }
+EXPORT_SYMBOL(pci_register_host_bridge);
 
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
diff --git a/include/linux/pci.h b/include/linux/pci.h
index e9ba95cbfd62..e2d1a124216a 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -449,6 +449,8 @@ static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
 	return container_of(priv, struct pci_host_bridge, private);
 }
 
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv);
+int pci_register_host_bridge(struct pci_host_bridge *bridge);
 struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);
 
 void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
-- 
2.10.2

[PATCH v4 03/10] PCI: Make host bridge interface publicly available

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Allow PCI host bridge drivers to use the new host bridge interfaces to
register their host bridge.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/probe.c | 6 ++++--
 include/linux/pci.h | 2 ++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 912b22ed96dd..eeb48d3ce80c 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -521,7 +521,7 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 {
 	struct pci_host_bridge *bridge;
 
@@ -533,6 +533,7 @@ static struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 
 	return bridge;
 }
+EXPORT_SYMBOL(pci_alloc_host_bridge);
 
 static const unsigned char pcix_bus_speed[] = {
 	PCI_SPEED_UNKNOWN,		/* 0 */
@@ -717,7 +718,7 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus)
 	dev_set_msi_domain(&bus->dev, d);
 }
 
-static int pci_register_host_bridge(struct pci_host_bridge *bridge)
+int pci_register_host_bridge(struct pci_host_bridge *bridge)
 {
 	struct device *parent = bridge->dev.parent;
 	struct resource_entry *window, *n;
@@ -832,6 +833,7 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 	kfree(bus);
 	return err;
 }
+EXPORT_SYMBOL(pci_register_host_bridge);
 
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
diff --git a/include/linux/pci.h b/include/linux/pci.h
index e9ba95cbfd62..e2d1a124216a 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -449,6 +449,8 @@ static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
 	return container_of(priv, struct pci_host_bridge, private);
 }
 
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv);
+int pci_register_host_bridge(struct pci_host_bridge *bridge);
 struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);
 
 void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
-- 
2.10.2

[PATCH v4 04/10] PCI: tegra: Use new pci_register_host_bridge() interface

[Thierry Reding]

From: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>

Tegra is one of the remaining platforms that still use the traditional
pci_common_init_dev() interface for probing PCI host bridges.

This demonstrates how to convert it to the pci_register_host interface
I just added in a previous patch. This leads to a more linear probe
sequence that can handle errors better because we avoid callbacks into
the driver, and it makes the driver architecture independent.

Changes in v4 (Thierry Reding):
- update for changes in core to deal with driver-private data

Signed-off-by: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/host/pci-tegra.c | 105 ++++++++++++++++++++++---------------------
 1 file changed, 54 insertions(+), 51 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 8dfccf733241..d5206fa53353 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -322,11 +322,6 @@ struct tegra_pcie_bus {
 	unsigned int nr;
 };
 
-static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
-{
-	return sys->private_data;
-}
-
 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
 			      unsigned long offset)
 {
@@ -430,7 +425,8 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 
 static int tegra_pcie_add_bus(struct pci_bus *bus)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *b;
 
 	b = tegra_pcie_bus_alloc(pcie, bus->number);
@@ -444,7 +440,8 @@ static int tegra_pcie_add_bus(struct pci_bus *bus)
 
 static void tegra_pcie_remove_bus(struct pci_bus *child)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(child);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *bus, *tmp;
 
 	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
@@ -461,7 +458,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
 					unsigned int devfn,
 					int where)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct device *dev = pcie->dev;
 	void __iomem *addr = NULL;
 
@@ -610,39 +608,31 @@ static void tegra_pcie_relax_enable(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
 
-static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
+static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(sys);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct list_head *windows = &host->windows;
 	struct device *dev = pcie->dev;
 	int err;
 
-	sys->mem_offset = pcie->offset.mem;
-	sys->io_offset = pcie->offset.io;
+	pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
+	pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
+	pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
+	pci_add_resource(windows, &pcie->busn);
 
-	err = devm_request_resource(dev, &iomem_resource, &pcie->io);
+	err = devm_request_pci_bus_resources(dev, windows);
 	if (err < 0)
 		return err;
 
-	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
-	if (!err)
-		pci_add_resource_offset(&sys->resources, &pcie->pio,
-					sys->io_offset);
-
-	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
-	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
-				sys->mem_offset);
-	pci_add_resource(&sys->resources, &pcie->busn);
-
-	err = devm_request_pci_bus_resources(dev, &sys->resources);
-	if (err < 0)
-		return err;
+	pci_remap_iospace(&pcie->pio, pcie->io.start);
 
-	return 1;
+	return 0;
 }
 
 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	int irq;
 
 	tegra_cpuidle_pcie_irqs_in_use();
@@ -1499,10 +1489,11 @@ static const struct irq_domain_ops msi_domain_ops = {
 
 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 {
-	struct device *dev = pcie->dev;
-	struct platform_device *pdev = to_platform_device(dev);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct platform_device *pdev = to_platform_device(pcie->dev);
 	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_msi *msi = &pcie->msi;
+	struct device *dev = pcie->dev;
 	unsigned long base;
 	int err;
 	u32 reg;
@@ -1559,6 +1550,8 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 	reg |= AFI_INTR_MASK_MSI_MASK;
 	afi_writel(pcie, reg, AFI_INTR_MASK);
 
+	host->msi = &msi->chip;
+
 	return 0;
 
 err:
@@ -2021,11 +2014,10 @@ static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
 	return false;
 }
 
-static int tegra_pcie_enable(struct tegra_pcie *pcie)
+static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 	struct tegra_pcie_port *port, *tmp;
-	struct hw_pci hw;
 
 	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 		dev_info(dev, "probing port %u, using %u lanes\n",
@@ -2041,21 +2033,6 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
 		tegra_pcie_port_disable(port);
 		tegra_pcie_port_free(port);
 	}
-
-	memset(&hw, 0, sizeof(hw));
-
-#ifdef CONFIG_PCI_MSI
-	hw.msi_ctrl = &pcie->msi.chip;
-#endif
-
-	hw.nr_controllers = 1;
-	hw.private_data = (void **)&pcie;
-	hw.setup = tegra_pcie_setup;
-	hw.map_irq = tegra_pcie_map_irq;
-	hw.ops = &tegra_pcie_ops;
-
-	pci_common_init_dev(dev, &hw);
-	return 0;
 }
 
 static const struct tegra_pcie_soc tegra20_pcie = {
@@ -2217,13 +2194,17 @@ static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
 static int tegra_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct pci_host_bridge *host;
 	struct tegra_pcie *pcie;
+	struct pci_bus *child;
 	int err;
 
-	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
-	if (!pcie)
+	host = pci_alloc_host_bridge(sizeof(*pcie));
+	if (!host)
 		return -ENOMEM;
 
+	pcie = pci_host_bridge_priv(host);
+
 	pcie->soc = of_device_get_match_data(dev);
 	INIT_LIST_HEAD(&pcie->buses);
 	INIT_LIST_HEAD(&pcie->ports);
@@ -2243,6 +2224,10 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 	if (err)
 		goto put_resources;
 
+	err = tegra_pcie_request_resources(pcie);
+	if (err)
+		goto put_resources;
+
 	/* setup the AFI address translations */
 	tegra_pcie_setup_translations(pcie);
 
@@ -2254,12 +2239,30 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = tegra_pcie_enable(pcie);
+	tegra_pcie_enable_ports(pcie);
+
+	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+	host->busnr = pcie->busn.start;
+	host->dev.parent = &pdev->dev;
+	host->ops = &tegra_pcie_ops;
+
+	err = pci_register_host_bridge(host);
 	if (err < 0) {
-		dev_err(dev, "failed to enable PCIe ports: %d\n", err);
+		dev_err(dev, "failed to register host: %d\n", err);
 		goto disable_msi;
 	}
 
+	pci_scan_child_bus(host->bus);
+
+	pci_fixup_irqs(pci_common_swizzle, tegra_pcie_map_irq);
+	pci_bus_size_bridges(host->bus);
+	pci_bus_assign_resources(host->bus);
+
+	list_for_each_entry(child, &host->bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(host->bus);
+
 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
 		err = tegra_pcie_debugfs_init(pcie);
 		if (err < 0)
-- 
2.10.2

[PATCH v4 04/10] PCI: tegra: Use new pci_register_host_bridge() interface

[Thierry Reding]

From: Arnd Bergmann <arnd@arndb.de>

Tegra is one of the remaining platforms that still use the traditional
pci_common_init_dev() interface for probing PCI host bridges.

This demonstrates how to convert it to the pci_register_host interface
I just added in a previous patch. This leads to a more linear probe
sequence that can handle errors better because we avoid callbacks into
the driver, and it makes the driver architecture independent.

Changes in v4 (Thierry Reding):
- update for changes in core to deal with driver-private data

Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/host/pci-tegra.c | 105 ++++++++++++++++++++++---------------------
 1 file changed, 54 insertions(+), 51 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 8dfccf733241..d5206fa53353 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -322,11 +322,6 @@ struct tegra_pcie_bus {
 	unsigned int nr;
 };
 
-static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
-{
-	return sys->private_data;
-}
-
 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
 			      unsigned long offset)
 {
@@ -430,7 +425,8 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 
 static int tegra_pcie_add_bus(struct pci_bus *bus)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *b;
 
 	b = tegra_pcie_bus_alloc(pcie, bus->number);
@@ -444,7 +440,8 @@ static int tegra_pcie_add_bus(struct pci_bus *bus)
 
 static void tegra_pcie_remove_bus(struct pci_bus *child)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(child);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *bus, *tmp;
 
 	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
@@ -461,7 +458,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
 					unsigned int devfn,
 					int where)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct device *dev = pcie->dev;
 	void __iomem *addr = NULL;
 
@@ -610,39 +608,31 @@ static void tegra_pcie_relax_enable(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
 
-static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
+static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(sys);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct list_head *windows = &host->windows;
 	struct device *dev = pcie->dev;
 	int err;
 
-	sys->mem_offset = pcie->offset.mem;
-	sys->io_offset = pcie->offset.io;
+	pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
+	pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
+	pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
+	pci_add_resource(windows, &pcie->busn);
 
-	err = devm_request_resource(dev, &iomem_resource, &pcie->io);
+	err = devm_request_pci_bus_resources(dev, windows);
 	if (err < 0)
 		return err;
 
-	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
-	if (!err)
-		pci_add_resource_offset(&sys->resources, &pcie->pio,
-					sys->io_offset);
-
-	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
-	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
-				sys->mem_offset);
-	pci_add_resource(&sys->resources, &pcie->busn);
-
-	err = devm_request_pci_bus_resources(dev, &sys->resources);
-	if (err < 0)
-		return err;
+	pci_remap_iospace(&pcie->pio, pcie->io.start);
 
-	return 1;
+	return 0;
 }
 
 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	int irq;
 
 	tegra_cpuidle_pcie_irqs_in_use();
@@ -1499,10 +1489,11 @@ static const struct irq_domain_ops msi_domain_ops = {
 
 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 {
-	struct device *dev = pcie->dev;
-	struct platform_device *pdev = to_platform_device(dev);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct platform_device *pdev = to_platform_device(pcie->dev);
 	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_msi *msi = &pcie->msi;
+	struct device *dev = pcie->dev;
 	unsigned long base;
 	int err;
 	u32 reg;
@@ -1559,6 +1550,8 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 	reg |= AFI_INTR_MASK_MSI_MASK;
 	afi_writel(pcie, reg, AFI_INTR_MASK);
 
+	host->msi = &msi->chip;
+
 	return 0;
 
 err:
@@ -2021,11 +2014,10 @@ static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
 	return false;
 }
 
-static int tegra_pcie_enable(struct tegra_pcie *pcie)
+static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 	struct tegra_pcie_port *port, *tmp;
-	struct hw_pci hw;
 
 	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 		dev_info(dev, "probing port %u, using %u lanes\n",
@@ -2041,21 +2033,6 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
 		tegra_pcie_port_disable(port);
 		tegra_pcie_port_free(port);
 	}
-
-	memset(&hw, 0, sizeof(hw));
-
-#ifdef CONFIG_PCI_MSI
-	hw.msi_ctrl = &pcie->msi.chip;
-#endif
-
-	hw.nr_controllers = 1;
-	hw.private_data = (void **)&pcie;
-	hw.setup = tegra_pcie_setup;
-	hw.map_irq = tegra_pcie_map_irq;
-	hw.ops = &tegra_pcie_ops;
-
-	pci_common_init_dev(dev, &hw);
-	return 0;
 }
 
 static const struct tegra_pcie_soc tegra20_pcie = {
@@ -2217,13 +2194,17 @@ static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
 static int tegra_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct pci_host_bridge *host;
 	struct tegra_pcie *pcie;
+	struct pci_bus *child;
 	int err;
 
-	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
-	if (!pcie)
+	host = pci_alloc_host_bridge(sizeof(*pcie));
+	if (!host)
 		return -ENOMEM;
 
+	pcie = pci_host_bridge_priv(host);
+
 	pcie->soc = of_device_get_match_data(dev);
 	INIT_LIST_HEAD(&pcie->buses);
 	INIT_LIST_HEAD(&pcie->ports);
@@ -2243,6 +2224,10 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 	if (err)
 		goto put_resources;
 
+	err = tegra_pcie_request_resources(pcie);
+	if (err)
+		goto put_resources;
+
 	/* setup the AFI address translations */
 	tegra_pcie_setup_translations(pcie);
 
@@ -2254,12 +2239,30 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = tegra_pcie_enable(pcie);
+	tegra_pcie_enable_ports(pcie);
+
+	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+	host->busnr = pcie->busn.start;
+	host->dev.parent = &pdev->dev;
+	host->ops = &tegra_pcie_ops;
+
+	err = pci_register_host_bridge(host);
 	if (err < 0) {
-		dev_err(dev, "failed to enable PCIe ports: %d\n", err);
+		dev_err(dev, "failed to register host: %d\n", err);
 		goto disable_msi;
 	}
 
+	pci_scan_child_bus(host->bus);
+
+	pci_fixup_irqs(pci_common_swizzle, tegra_pcie_map_irq);
+	pci_bus_size_bridges(host->bus);
+	pci_bus_assign_resources(host->bus);
+
+	list_for_each_entry(child, &host->bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(host->bus);
+
 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
 		err = tegra_pcie_debugfs_init(pcie);
 		if (err < 0)
-- 
2.10.2

[PATCH v4 05/10] dt-bindings: pci: tegra: Add Tegra210 support

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Add support for the PCI host controller found on Tegra210 SoCs. It is
very similar to the variant found on Tegra124, with a couple of small
differences regarding the power supplies.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 .../bindings/pci/nvidia,tegra20-pcie.txt           | 110 +++++++++++++++++++++
 1 file changed, 110 insertions(+)

diff --git a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
index b8cc395fffea..982a74ea6df9 100644
--- a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
@@ -110,6 +110,20 @@ Power supplies for Tegra124:
   - avdd-pll-erefe-supply: Power supply for PLLE (shared with USB3). Must
     supply 1.05 V.
 
+Power supplies for Tegra210:
+- Required:
+  - avdd-pll-uerefe-supply: Power supply for PLLE (shared with USB3). Must
+    supply 1.05 V.
+  - hvddio-pex-supply: High-voltage supply for PCIe I/O and PCIe output
+    clocks. Must supply 1.8 V.
+  - dvddio-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
+  - dvdd-pex-pll-supply: Power supply for dedicated (internal) PCIe PLL. Must
+    supply 1.05 V.
+  - hvdd-pex-pll-e-supply: High-voltage supply for PLLE (shared with USB3).
+    Must supply 3.3 V.
+  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
+    supply 1.8 V.
+
 Root ports are defined as subnodes of the PCIe controller node.
 
 Required properties:
@@ -436,3 +450,99 @@ Board DTS:
 			status = "okay";
 		};
 	};
+
+Tegra210:
+---------
+
+SoC DTSI:
+
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
+Board DTS:
+
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
-- 
2.10.2

[PATCH v4 05/10] dt-bindings: pci: tegra: Add Tegra210 support

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Add support for the PCI host controller found on Tegra210 SoCs. It is
very similar to the variant found on Tegra124, with a couple of small
differences regarding the power supplies.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 .../bindings/pci/nvidia,tegra20-pcie.txt           | 110 +++++++++++++++++++++
 1 file changed, 110 insertions(+)

diff --git a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
index b8cc395fffea..982a74ea6df9 100644
--- a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
@@ -110,6 +110,20 @@ Power supplies for Tegra124:
   - avdd-pll-erefe-supply: Power supply for PLLE (shared with USB3). Must
     supply 1.05 V.
 
+Power supplies for Tegra210:
+- Required:
+  - avdd-pll-uerefe-supply: Power supply for PLLE (shared with USB3). Must
+    supply 1.05 V.
+  - hvddio-pex-supply: High-voltage supply for PCIe I/O and PCIe output
+    clocks. Must supply 1.8 V.
+  - dvddio-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
+  - dvdd-pex-pll-supply: Power supply for dedicated (internal) PCIe PLL. Must
+    supply 1.05 V.
+  - hvdd-pex-pll-e-supply: High-voltage supply for PLLE (shared with USB3).
+    Must supply 3.3 V.
+  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
+    supply 1.8 V.
+
 Root ports are defined as subnodes of the PCIe controller node.
 
 Required properties:
@@ -436,3 +450,99 @@ Board DTS:
 			status = "okay";
 		};
 	};
+
+Tegra210:
+---------
+
+SoC DTSI:
+
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
+Board DTS:
+
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
-- 
2.10.2

[PATCH v4 06/10] PCI: tegra: Implement PCA enable workaround

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Tegra210's PCIe controller has a bug that requires the PCA (performance
counter) feature to be enabled. If this isn't done, accesses to device
configuration space will hang the chip for tens of seconds. Implement
the workaround.

Based on commit 514e19138af2 ("pci: tegra: implement PCA enable
workaround") from U-Boot by Stephen Warren <swarren@nvidia.com>.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/host/pci-tegra.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index d5206fa53353..4bfaac6d3582 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -188,6 +188,9 @@
 #define RP_VEND_XP	0x00000f00
 #define  RP_VEND_XP_DL_UP	(1 << 30)
 
+#define RP_VEND_CTL2 0x00000fa8
+#define  RP_VEND_CTL2_PCA_ENABLE (1 << 7)
+
 #define RP_PRIV_MISC	0x00000fe0
 #define  RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
 #define  RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
@@ -252,6 +255,7 @@ struct tegra_pcie_soc {
 	bool has_intr_prsnt_sense;
 	bool has_cml_clk;
 	bool has_gen2;
+	bool force_pca_enable;
 };
 
 static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
@@ -556,6 +560,12 @@ static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
 	afi_writel(port->pcie, value, ctrl);
 
 	tegra_pcie_port_reset(port);
+
+	if (soc->force_pca_enable) {
+		value = readl(port->base + RP_VEND_CTL2);
+		value |= RP_VEND_CTL2_PCA_ENABLE;
+		writel(value, port->base + RP_VEND_CTL2);
+	}
 }
 
 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
@@ -2046,6 +2056,7 @@ static const struct tegra_pcie_soc tegra20_pcie = {
 	.has_intr_prsnt_sense = false,
 	.has_cml_clk = false,
 	.has_gen2 = false,
+	.force_pca_enable = false,
 };
 
 static const struct tegra_pcie_soc tegra30_pcie = {
@@ -2060,6 +2071,7 @@ static const struct tegra_pcie_soc tegra30_pcie = {
 	.has_intr_prsnt_sense = true,
 	.has_cml_clk = true,
 	.has_gen2 = false,
+	.force_pca_enable = false,
 };
 
 static const struct tegra_pcie_soc tegra124_pcie = {
@@ -2073,6 +2085,7 @@ static const struct tegra_pcie_soc tegra124_pcie = {
 	.has_intr_prsnt_sense = true,
 	.has_cml_clk = true,
 	.has_gen2 = true,
+	.force_pca_enable = false,
 };
 
 static const struct of_device_id tegra_pcie_of_match[] = {
-- 
2.10.2

[PATCH v4 07/10] PCI: tegra: Add Tegra210 support

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

The PCIe host controller found on Tegra X1 is very similar to its
predecessor on Tegra K1. A bug was introduced in the new revision that
is worked around by always enabling the performance counter, otherwise
accesses to configuration space will block for a number of seconds.

Changes in v4:
- use pgprot_device(PAGE_KERNEL) to increase portability

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/host/pci-tegra.c | 42 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 4bfaac6d3582..ed8a93f2bfb5 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -51,10 +51,6 @@
 #include <soc/tegra/cpuidle.h>
 #include <soc/tegra/pmc.h>
 
-#include <asm/mach/irq.h>
-#include <asm/mach/map.h>
-#include <asm/mach/pci.h>
-
 #define INT_PCI_MSI_NR (8 * 32)
 
 /* register definitions */
@@ -384,8 +380,7 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 						   unsigned int busnr)
 {
 	struct device *dev = pcie->dev;
-	pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-				 L_PTE_XN | L_PTE_MT_DEV_SHARED | L_PTE_SHARED);
+	pgprot_t prot = pgprot_device(PAGE_KERNEL);
 	phys_addr_t cs = pcie->cs->start;
 	struct tegra_pcie_bus *bus;
 	unsigned int i;
@@ -1612,7 +1607,8 @@ static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
 	struct device *dev = pcie->dev;
 	struct device_node *np = dev->of_node;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
+	    of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
 		switch (lanes) {
 		case 0x0000104:
 			dev_info(dev, "4x1, 1x1 configuration\n");
@@ -1733,7 +1729,22 @@ static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
 	struct device_node *np = dev->of_node;
 	unsigned int i = 0;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
+		pcie->num_supplies = 6;
+
+		pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
+					      sizeof(*pcie->supplies),
+					      GFP_KERNEL);
+		if (!pcie->supplies)
+			return -ENOMEM;
+
+		pcie->supplies[i++].supply = "avdd-pll-uerefe";
+		pcie->supplies[i++].supply = "hvddio-pex";
+		pcie->supplies[i++].supply = "dvddio-pex";
+		pcie->supplies[i++].supply = "dvdd-pex-pll";
+		pcie->supplies[i++].supply = "hvdd-pex-pll-e";
+		pcie->supplies[i++].supply = "vddio-pex-ctl";
+	} else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
 		pcie->num_supplies = 7;
 
 		pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
@@ -2088,7 +2099,22 @@ static const struct tegra_pcie_soc tegra124_pcie = {
 	.force_pca_enable = false,
 };
 
+static const struct tegra_pcie_soc tegra210_pcie = {
+	.num_ports = 2,
+	.msi_base_shift = 8,
+	.pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+	.tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+	.pads_refclk_cfg0 = 0x90b890b8,
+	.has_pex_clkreq_en = true,
+	.has_pex_bias_ctrl = true,
+	.has_intr_prsnt_sense = true,
+	.has_cml_clk = true,
+	.has_gen2 = true,
+	.force_pca_enable = true,
+};
+
 static const struct of_device_id tegra_pcie_of_match[] = {
+	{ .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
 	{ .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
 	{ .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
 	{ .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
-- 
2.10.2

[PATCH v4 07/10] PCI: tegra: Add Tegra210 support

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

The PCIe host controller found on Tegra X1 is very similar to its
predecessor on Tegra K1. A bug was introduced in the new revision that
is worked around by always enabling the performance counter, otherwise
accesses to configuration space will block for a number of seconds.

Changes in v4:
- use pgprot_device(PAGE_KERNEL) to increase portability

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/host/pci-tegra.c | 42 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 4bfaac6d3582..ed8a93f2bfb5 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -51,10 +51,6 @@
 #include <soc/tegra/cpuidle.h>
 #include <soc/tegra/pmc.h>
 
-#include <asm/mach/irq.h>
-#include <asm/mach/map.h>
-#include <asm/mach/pci.h>
-
 #define INT_PCI_MSI_NR (8 * 32)
 
 /* register definitions */
@@ -384,8 +380,7 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 						   unsigned int busnr)
 {
 	struct device *dev = pcie->dev;
-	pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-				 L_PTE_XN | L_PTE_MT_DEV_SHARED | L_PTE_SHARED);
+	pgprot_t prot = pgprot_device(PAGE_KERNEL);
 	phys_addr_t cs = pcie->cs->start;
 	struct tegra_pcie_bus *bus;
 	unsigned int i;
@@ -1612,7 +1607,8 @@ static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
 	struct device *dev = pcie->dev;
 	struct device_node *np = dev->of_node;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
+	    of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
 		switch (lanes) {
 		case 0x0000104:
 			dev_info(dev, "4x1, 1x1 configuration\n");
@@ -1733,7 +1729,22 @@ static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
 	struct device_node *np = dev->of_node;
 	unsigned int i = 0;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
+		pcie->num_supplies = 6;
+
+		pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
+					      sizeof(*pcie->supplies),
+					      GFP_KERNEL);
+		if (!pcie->supplies)
+			return -ENOMEM;
+
+		pcie->supplies[i++].supply = "avdd-pll-uerefe";
+		pcie->supplies[i++].supply = "hvddio-pex";
+		pcie->supplies[i++].supply = "dvddio-pex";
+		pcie->supplies[i++].supply = "dvdd-pex-pll";
+		pcie->supplies[i++].supply = "hvdd-pex-pll-e";
+		pcie->supplies[i++].supply = "vddio-pex-ctl";
+	} else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
 		pcie->num_supplies = 7;
 
 		pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
@@ -2088,7 +2099,22 @@ static const struct tegra_pcie_soc tegra124_pcie = {
 	.force_pca_enable = false,
 };
 
+static const struct tegra_pcie_soc tegra210_pcie = {
+	.num_ports = 2,
+	.msi_base_shift = 8,
+	.pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+	.tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+	.pads_refclk_cfg0 = 0x90b890b8,
+	.has_pex_clkreq_en = true,
+	.has_pex_bias_ctrl = true,
+	.has_intr_prsnt_sense = true,
+	.has_cml_clk = true,
+	.has_gen2 = true,
+	.force_pca_enable = true,
+};
+
 static const struct of_device_id tegra_pcie_of_match[] = {
+	{ .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
 	{ .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
 	{ .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
 	{ .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
-- 
2.10.2

[PATCH v4 08/10] PCI: tegra: Enable the driver on 64-bit ARM

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

The Tegra PCI host controller driver no longer relies on any of the 32-
bit ARM glue for PCI, so it can be enabled on 64-bit configurations.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/pci/host/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 14dc4053860e..9a109dad0d63 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -69,7 +69,7 @@ config PCI_IMX6
 
 config PCI_TEGRA
 	bool "NVIDIA Tegra PCIe controller"
-	depends on ARCH_TEGRA && !ARM64
+	depends on ARCH_TEGRA
 	help
 	  Say Y here if you want support for the PCIe host controller found
 	  on NVIDIA Tegra SoCs.
-- 
2.10.2

[PATCH v4 08/10] PCI: tegra: Enable the driver on 64-bit ARM

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

The Tegra PCI host controller driver no longer relies on any of the 32-
bit ARM glue for PCI, so it can be enabled on 64-bit configurations.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 drivers/pci/host/Kconfig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 14dc4053860e..9a109dad0d63 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -69,7 +69,7 @@ config PCI_IMX6
 
 config PCI_TEGRA
 	bool "NVIDIA Tegra PCIe controller"
-	depends on ARCH_TEGRA && !ARM64
+	depends on ARCH_TEGRA
 	help
 	  Say Y here if you want support for the PCIe host controller found
 	  on NVIDIA Tegra SoCs.
-- 
2.10.2

[PATCH v4 09/10] arm64: tegra: Add PCIe host bridge on Tegra210

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Add the PCIe host bridge found on Tegra X1. It implements two root ports
that support x4 and x1 configurations, respectively.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 arch/arm64/boot/dts/nvidia/tegra210.dtsi | 63 ++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 46045fe719da..2f832df29da8 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -11,6 +11,69 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
 	host1x@50000000 {
 		compatible = "nvidia,tegra210-host1x", "simple-bus";
 		reg = <0x0 0x50000000 0x0 0x00034000>;
-- 
2.10.2

[PATCH v4 09/10] arm64: tegra: Add PCIe host bridge on Tegra210

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Add the PCIe host bridge found on Tegra X1. It implements two root ports
that support x4 and x1 configurations, respectively.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 arch/arm64/boot/dts/nvidia/tegra210.dtsi | 63 ++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 46045fe719da..2f832df29da8 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -11,6 +11,69 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
 	host1x@50000000 {
 		compatible = "nvidia,tegra210-host1x", "simple-bus";
 		reg = <0x0 0x50000000 0x0 0x00034000>;
-- 
2.10.2

[PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
currently untested due to lack of hardware.

Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 ++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
index 983775e637a4..4c1ea7a08d43 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
@@ -7,6 +7,32 @@
 	model = "NVIDIA Jetson TX1 Developer Kit";
 	compatible = "nvidia,p2371-2180", "nvidia,tegra210";
 
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
+
 	host1x@50000000 {
 		dsi@54300000 {
 			status = "okay";
-- 
2.10.2

[PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

From: Thierry Reding <treding@nvidia.com>

Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
currently untested due to lack of hardware.

Signed-off-by: Thierry Reding <treding@nvidia.com>
---
 arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 ++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
index 983775e637a4..4c1ea7a08d43 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
@@ -7,6 +7,32 @@
 	model = "NVIDIA Jetson TX1 Developer Kit";
 	compatible = "nvidia,p2371-2180", "nvidia,tegra210";
 
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
+
 	host1x@50000000 {
 		dsi@54300000 {
 			status = "okay";
-- 
2.10.2

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/ 
works.. Relevant parts of bootlog:

[    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
[    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 
4 lanes
[    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000008
[    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 
1 lanes
[    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000000
[    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
[    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
bus 0000:00
[    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    3.217343] pci_bus 0000:00: root bus resource [mem 
0x13000000-0x1fffffff]
[    3.224218] pci_bus 0000:00: root bus resource [mem 
0x20000000-0x3fffffff pref]
[    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
[    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 
00-00]), reconfiguring
[    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem 
0x13000000-0x130fffff]
[    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 
0x20000000-0x200fffff 64bit pref]
[    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
[    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 
0x20000000-0x20003fff 64bit pref]
[    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 
0x13000000-0x13000fff 64bit]
[    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
[    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
[    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
[    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
[    3.314454] pci 0000:00:01.0:   bridge window [mem 
0x20000000-0x200fffff 64bit pref]
[    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host 
bridge
[    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
[    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
interrupt
[    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
[    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
[    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
[    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at 
0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
[    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 
bytes, tx checksumming: ko]

then

[    3.706240] tegra-mc 70019000.memory-controller: afiw: write 
@0x000000007a484000: EMEM address decode error (EMEM decode error)
[    3.717747] r8169 0000:01:00.0 eth0: link down

The card is alive though, the transfer LED is blinking according to 
incoming traffic, I assume; we can also see that the driver is able to 
access the hw to some extent. The kernel version is 4.9rc7 with just 
this series on top. The same board/card also works in U-boot.

- Mikko

On 11/25/2016 12:57 PM, Thierry Reding wrote:
> From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>
> Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
> currently untested due to lack of hardware.
>
> Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> ---
>  arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 ++++++++++++++++++++++
>  1 file changed, 26 insertions(+)
>
> diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> index 983775e637a4..4c1ea7a08d43 100644
> --- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> @@ -7,6 +7,32 @@
>  	model = "NVIDIA Jetson TX1 Developer Kit";
>  	compatible = "nvidia,p2371-2180", "nvidia,tegra210";
>
> +	pcie-controller@01003000 {
> +		status = "okay";
> +
> +		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
> +		hvddio-pex-supply = <&vdd_1v8>;
> +		dvddio-pex-supply = <&vdd_pex_1v05>;
> +		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
> +		hvdd-pex-pll-e-supply = <&vdd_1v8>;
> +		vddio-pex-ctl-supply = <&vdd_1v8>;
> +
> +		pci@1,0 {
> +			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
> +			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
> +			status = "okay";
> +		};
> +
> +		pci@2,0 {
> +			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
> +			phy-names = "pcie-0";
> +			status = "okay";
> +		};
> +	};
> +
>  	host1x@50000000 {
>  		dsi@54300000 {
>  			status = "okay";
>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/ 
works.. Relevant parts of bootlog:

[    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
[    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 
4 lanes
[    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000008
[    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 
1 lanes
[    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000000
[    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
[    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
bus 0000:00
[    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    3.217343] pci_bus 0000:00: root bus resource [mem 
0x13000000-0x1fffffff]
[    3.224218] pci_bus 0000:00: root bus resource [mem 
0x20000000-0x3fffffff pref]
[    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
[    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 
00-00]), reconfiguring
[    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem 
0x13000000-0x130fffff]
[    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 
0x20000000-0x200fffff 64bit pref]
[    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
[    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 
0x20000000-0x20003fff 64bit pref]
[    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 
0x13000000-0x13000fff 64bit]
[    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
[    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
[    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
[    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
[    3.314454] pci 0000:00:01.0:   bridge window [mem 
0x20000000-0x200fffff 64bit pref]
[    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host 
bridge
[    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
[    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
interrupt
[    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
[    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
[    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
[    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at 
0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
[    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 
bytes, tx checksumming: ko]

then

[    3.706240] tegra-mc 70019000.memory-controller: afiw: write 
@0x000000007a484000: EMEM address decode error (EMEM decode error)
[    3.717747] r8169 0000:01:00.0 eth0: link down

The card is alive though, the transfer LED is blinking according to 
incoming traffic, I assume; we can also see that the driver is able to 
access the hw to some extent. The kernel version is 4.9rc7 with just 
this series on top. The same board/card also works in U-boot.

- Mikko

On 11/25/2016 12:57 PM, Thierry Reding wrote:
> From: Thierry Reding <treding@nvidia.com>
>
> Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
> currently untested due to lack of hardware.
>
> Signed-off-by: Thierry Reding <treding@nvidia.com>
> ---
>  arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 ++++++++++++++++++++++
>  1 file changed, 26 insertions(+)
>
> diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> index 983775e637a4..4c1ea7a08d43 100644
> --- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
> @@ -7,6 +7,32 @@
>  	model = "NVIDIA Jetson TX1 Developer Kit";
>  	compatible = "nvidia,p2371-2180", "nvidia,tegra210";
>
> +	pcie-controller@01003000 {
> +		status = "okay";
> +
> +		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
> +		hvddio-pex-supply = <&vdd_1v8>;
> +		dvddio-pex-supply = <&vdd_pex_1v05>;
> +		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
> +		hvdd-pex-pll-e-supply = <&vdd_1v8>;
> +		vddio-pex-ctl-supply = <&vdd_1v8>;
> +
> +		pci@1,0 {
> +			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
> +			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
> +			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
> +			status = "okay";
> +		};
> +
> +		pci@2,0 {
> +			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
> +			phy-names = "pcie-0";
> +			status = "okay";
> +		};
> +	};
> +
>  	host1x@50000000 {
>  		dsi@54300000 {
>  			status = "okay";
>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Vidya Sagar]

I've also tried with OCZ SSD and things are working fine.

Boot log...

[    1.770836] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
[    1.796262] tegra-pcie 1003000.pcie-controller: probing port 0, using 
4 lanes
[    1.809456] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000008
[    1.828019] tegra-pcie 1003000.pcie-controller: probing port 1, using 
1 lanes
[    1.835179] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000000
[    2.254574] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    2.665182] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.075772] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.084231] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
[    3.090863] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
bus 0000:00
[    3.098138] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]
[    3.104340] pci_bus 0000:00: root bus resource [mem 
0x13000000-0x1fffffff]
[    3.111232] pci_bus 0000:00: root bus resource [mem 
0x20000000-0x3fffffff pref]
[    3.118566] pci_bus 0000:00: root bus resource [bus 00-ff]
[    3.124490] pci 0000:00:01.0: bridge configuration invalid ([bus 
00-00]), reconfiguring
[    3.144765] pci 0000:00:01.0: BAR 14: assigned [mem 
0x13000000-0x130fffff]
[    3.151670] pci 0000:01:00.0: BAR 2: assigned [mem 
0x13000000-0x1303ffff 64bit]
[    3.159013] pci 0000:01:00.0: BAR 0: assigned [mem 
0x13040000-0x1305ffff 64bit]
[    3.166338] pci 0000:01:00.0: BAR 6: assigned [mem 
0x13060000-0x1306ffff pref]
[    3.173559] pci 0000:00:01.0: PCI bridge to [bus 01]
[    3.178530] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
[    3.185338] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host 
bridge
[    3.192384] pcieport 0000:00:01.0: enabling device (0000 -> 0002)
[    3.198747] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
interrupt
[    3.205763] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
[    3.212491] mvsas 0000:01:00.0: mvsas: driver version 0.8.16
[    3.218182] mvsas 0000:01:00.0: enabling device (0000 -> 0002)
[    3.229007] mvsas 0000:01:00.0: mvsas: PCI-E x4, Bandwidth Usage: 2.5 
Gbps
[   10.184674] scsi host0: mvsas
[   10.368296] ata1.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.374400] ata1.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.388296] ata1.00: configured for UDMA/133
[   10.411571] scsi 0:0:0:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.441215] sd 0:0:0:0: [sda] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   10.620169] ata2.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.626283] ata2.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.640171] ata2.00: configured for UDMA/133
[   10.644714] sd 0:0:0:0: [sda] Write Protect is off
[   10.649597] sd 0:0:0:0: [sda] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   10.661798] sd 0:0:0:0: [sda] Attached SCSI disk
[   10.670963] scsi 0:0:1:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.701173] sd 0:0:1:0: [sdb] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   10.879961] ata3.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.886076] ata3.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.899961] ata3.00: configured for UDMA/133
[   10.904486] sd 0:0:1:0: [sdb] Write Protect is off
[   10.909367] sd 0:0:1:0: [sdb] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   10.920861]  sdb: sdb1
[   10.924375] sd 0:0:1:0: [sdb] Attached SCSI disk
[   10.933242] scsi 0:0:2:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.965166] sd 0:0:2:0: [sdc] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   11.143890] ata4.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   11.150002] ata4.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   11.163892] ata4.00: configured for UDMA/133
[   11.168424] sd 0:0:2:0: [sdc] Write Protect is off
[   11.173306] sd 0:0:2:0: [sdc] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   11.184751]  sdc: sdc1
[   11.188243] sd 0:0:2:0: [sdc] Attached SCSI disk
[   11.197107] scsi 0:0:3:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   11.229166] sd 0:0:3:0: [sdd] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   11.238220] 70090000.usb supply dvdd-pex-pll not found, using dummy 
regulator
[   11.245777] sd 0:0:3:0: [sdd] Write Protect is off
[   11.245848] sd 0:0:3:0: [sdd] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   11.251511]  sdd: sdd1
[   11.252354] sd 0:0:3:0: [sdd] Attached SCSI disk

Thanks,

Vidya Sagar


On Monday 28 November 2016 10:24 PM, Mikko Perttunen wrote:
> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/ 
> works.. Relevant parts of bootlog:
>
> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, 
> using 4 lanes
> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin 
> change, signature: 00000008
> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, 
> using 1 lanes
> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin 
> change, signature: 00000000
> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
> bus 0000:00
> [    3.211160] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]
> [    3.217343] pci_bus 0000:00: root bus resource [mem 
> 0x13000000-0x1fffffff]
> [    3.224218] pci_bus 0000:00: root bus resource [mem 
> 0x20000000-0x3fffffff pref]
> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 
> 00-00]), reconfiguring
> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem 
> 0x13000000-0x130fffff]
> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 
> 0x20000000-0x200fffff 64bit pref]
> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io 0x1000-0x1fff]
> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 
> 0x20000000-0x20003fff 64bit pref]
> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 
> 0x13000000-0x13000fff 64bit]
> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io 0x1000-0x10ff]
> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> [    3.301568] pci 0000:00:01.0:   bridge window [io 0x1000-0x1fff]
> [    3.307666] pci 0000:00:01.0:   bridge window [mem 
> 0x13000000-0x130fffff]
> [    3.314454] pci 0000:00:01.0:   bridge window [mem 
> 0x20000000-0x200fffff 64bit pref]
> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate 
> host bridge
> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
> interrupt
> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at 
> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 
> bytes, tx checksumming: ko]
>
> then
>
> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write 
> @0x000000007a484000: EMEM address decode error (EMEM decode error)
> [    3.717747] r8169 0000:01:00.0 eth0: link down
>
> The card is alive though, the transfer LED is blinking according to 
> incoming traffic, I assume; we can also see that the driver is able to 
> access the hw to some extent. The kernel version is 4.9rc7 with just 
> this series on top. The same board/card also works in U-boot.
>
> - Mikko
>
> On 11/25/2016 12:57 PM, Thierry Reding wrote:
>> From: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>>
>> Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
>> currently untested due to lack of hardware.
>>
>> Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
>> ---
>>  arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 
>> ++++++++++++++++++++++
>>  1 file changed, 26 insertions(+)
>>
>> diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts 
>> b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> index 983775e637a4..4c1ea7a08d43 100644
>> --- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> @@ -7,6 +7,32 @@
>>      model = "NVIDIA Jetson TX1 Developer Kit";
>>      compatible = "nvidia,p2371-2180", "nvidia,tegra210";
>>
>> +    pcie-controller@01003000 {
>> +        status = "okay";
>> +
>> +        avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
>> +        hvddio-pex-supply = <&vdd_1v8>;
>> +        dvddio-pex-supply = <&vdd_pex_1v05>;
>> +        dvdd-pex-pll-supply = <&vdd_pex_1v05>;
>> +        hvdd-pex-pll-e-supply = <&vdd_1v8>;
>> +        vddio-pex-ctl-supply = <&vdd_1v8>;
>> +
>> +        pci@1,0 {
>> +            phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
>> +            phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
>> +            status = "okay";
>> +        };
>> +
>> +        pci@2,0 {
>> +            phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
>> +            phy-names = "pcie-0";
>> +            status = "okay";
>> +        };
>> +    };
>> +
>>      host1x@50000000 {
>>          dsi@54300000 {
>>              status = "okay";
>>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Vidya Sagar]

I've also tried with OCZ SSD and things are working fine.

Boot log...

[    1.770836] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
[    1.796262] tegra-pcie 1003000.pcie-controller: probing port 0, using 
4 lanes
[    1.809456] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000008
[    1.828019] tegra-pcie 1003000.pcie-controller: probing port 1, using 
1 lanes
[    1.835179] tegra-pcie 1003000.pcie-controller: Slot present pin 
change, signature: 00000000
[    2.254574] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    2.665182] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.075772] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
[    3.084231] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
[    3.090863] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
bus 0000:00
[    3.098138] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]
[    3.104340] pci_bus 0000:00: root bus resource [mem 
0x13000000-0x1fffffff]
[    3.111232] pci_bus 0000:00: root bus resource [mem 
0x20000000-0x3fffffff pref]
[    3.118566] pci_bus 0000:00: root bus resource [bus 00-ff]
[    3.124490] pci 0000:00:01.0: bridge configuration invalid ([bus 
00-00]), reconfiguring
[    3.144765] pci 0000:00:01.0: BAR 14: assigned [mem 
0x13000000-0x130fffff]
[    3.151670] pci 0000:01:00.0: BAR 2: assigned [mem 
0x13000000-0x1303ffff 64bit]
[    3.159013] pci 0000:01:00.0: BAR 0: assigned [mem 
0x13040000-0x1305ffff 64bit]
[    3.166338] pci 0000:01:00.0: BAR 6: assigned [mem 
0x13060000-0x1306ffff pref]
[    3.173559] pci 0000:00:01.0: PCI bridge to [bus 01]
[    3.178530] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
[    3.185338] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host 
bridge
[    3.192384] pcieport 0000:00:01.0: enabling device (0000 -> 0002)
[    3.198747] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
interrupt
[    3.205763] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
[    3.212491] mvsas 0000:01:00.0: mvsas: driver version 0.8.16
[    3.218182] mvsas 0000:01:00.0: enabling device (0000 -> 0002)
[    3.229007] mvsas 0000:01:00.0: mvsas: PCI-E x4, Bandwidth Usage: 2.5 
Gbps
[   10.184674] scsi host0: mvsas
[   10.368296] ata1.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.374400] ata1.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.388296] ata1.00: configured for UDMA/133
[   10.411571] scsi 0:0:0:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.441215] sd 0:0:0:0: [sda] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   10.620169] ata2.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.626283] ata2.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.640171] ata2.00: configured for UDMA/133
[   10.644714] sd 0:0:0:0: [sda] Write Protect is off
[   10.649597] sd 0:0:0:0: [sda] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   10.661798] sd 0:0:0:0: [sda] Attached SCSI disk
[   10.670963] scsi 0:0:1:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.701173] sd 0:0:1:0: [sdb] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   10.879961] ata3.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   10.886076] ata3.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   10.899961] ata3.00: configured for UDMA/133
[   10.904486] sd 0:0:1:0: [sdb] Write Protect is off
[   10.909367] sd 0:0:1:0: [sdb] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   10.920861]  sdb: sdb1
[   10.924375] sd 0:0:1:0: [sdb] Attached SCSI disk
[   10.933242] scsi 0:0:2:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   10.965166] sd 0:0:2:0: [sdc] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   11.143890] ata4.00: ATA-8: OCZ-REVODRIVE3 X2, 2.25, max UDMA/133
[   11.150002] ata4.00: 117231408 sectors, multi 16: LBA48 NCQ (depth 31/32)
[   11.163892] ata4.00: configured for UDMA/133
[   11.168424] sd 0:0:2:0: [sdc] Write Protect is off
[   11.173306] sd 0:0:2:0: [sdc] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   11.184751]  sdc: sdc1
[   11.188243] sd 0:0:2:0: [sdc] Attached SCSI disk
[   11.197107] scsi 0:0:3:0: Direct-Access     ATA OCZ-REVODRIVE3 X 2.25 
PQ: 0 ANSI: 5
[   11.229166] sd 0:0:3:0: [sdd] 117231408 512-byte logical blocks: 
(60.0 GB/55.9 GiB)
[   11.238220] 70090000.usb supply dvdd-pex-pll not found, using dummy 
regulator
[   11.245777] sd 0:0:3:0: [sdd] Write Protect is off
[   11.245848] sd 0:0:3:0: [sdd] Write cache: enabled, read cache: 
enabled, doesn't support DPO or FUA
[   11.251511]  sdd: sdd1
[   11.252354] sd 0:0:3:0: [sdd] Attached SCSI disk

Thanks,

Vidya Sagar


On Monday 28 November 2016 10:24 PM, Mikko Perttunen wrote:
> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/ 
> works.. Relevant parts of bootlog:
>
> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, 
> using 4 lanes
> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin 
> change, signature: 00000008
> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, 
> using 1 lanes
> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin 
> change, signature: 00000000
> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to 
> bus 0000:00
> [    3.211160] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]
> [    3.217343] pci_bus 0000:00: root bus resource [mem 
> 0x13000000-0x1fffffff]
> [    3.224218] pci_bus 0000:00: root bus resource [mem 
> 0x20000000-0x3fffffff pref]
> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 
> 00-00]), reconfiguring
> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem 
> 0x13000000-0x130fffff]
> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 
> 0x20000000-0x200fffff 64bit pref]
> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io 0x1000-0x1fff]
> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 
> 0x20000000-0x20003fff 64bit pref]
> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 
> 0x13000000-0x13000fff 64bit]
> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io 0x1000-0x10ff]
> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> [    3.301568] pci 0000:00:01.0:   bridge window [io 0x1000-0x1fff]
> [    3.307666] pci 0000:00:01.0:   bridge window [mem 
> 0x13000000-0x130fffff]
> [    3.314454] pci 0000:00:01.0:   bridge window [mem 
> 0x20000000-0x200fffff 64bit pref]
> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate 
> host bridge
> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME 
> interrupt
> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at 
> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 
> bytes, tx checksumming: ko]
>
> then
>
> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write 
> @0x000000007a484000: EMEM address decode error (EMEM decode error)
> [    3.717747] r8169 0000:01:00.0 eth0: link down
>
> The card is alive though, the transfer LED is blinking according to 
> incoming traffic, I assume; we can also see that the driver is able to 
> access the hw to some extent. The kernel version is 4.9rc7 with just 
> this series on top. The same board/card also works in U-boot.
>
> - Mikko
>
> On 11/25/2016 12:57 PM, Thierry Reding wrote:
>> From: Thierry Reding <treding@nvidia.com>
>>
>> Enable the x4 PCIe and M.2 Key E slots on Jetson TX1. The Key E slot is
>> currently untested due to lack of hardware.
>>
>> Signed-off-by: Thierry Reding <treding@nvidia.com>
>> ---
>>  arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts | 26 
>> ++++++++++++++++++++++
>>  1 file changed, 26 insertions(+)
>>
>> diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts 
>> b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> index 983775e637a4..4c1ea7a08d43 100644
>> --- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> +++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
>> @@ -7,6 +7,32 @@
>>      model = "NVIDIA Jetson TX1 Developer Kit";
>>      compatible = "nvidia,p2371-2180", "nvidia,tegra210";
>>
>> +    pcie-controller@01003000 {
>> +        status = "okay";
>> +
>> +        avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
>> +        hvddio-pex-supply = <&vdd_1v8>;
>> +        dvddio-pex-supply = <&vdd_pex_1v05>;
>> +        dvdd-pex-pll-supply = <&vdd_pex_1v05>;
>> +        hvdd-pex-pll-e-supply = <&vdd_1v8>;
>> +        vddio-pex-ctl-supply = <&vdd_1v8>;
>> +
>> +        pci@1,0 {
>> +            phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
>> + <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
>> +            phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
>> +            status = "okay";
>> +        };
>> +
>> +        pci@2,0 {
>> +            phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
>> +            phy-names = "pcie-0";
>> +            status = "okay";
>> +        };
>> +    };
>> +
>>      host1x@50000000 {
>>          dsi@54300000 {
>>              status = "okay";
>>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 3903 bytes --]

On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
> works.. Relevant parts of bootlog:
> 
> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
> lanes
> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> signature: 00000008
> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
> lanes
> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> signature: 00000000
> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
> 0000:00
> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
> [    3.217343] pci_bus 0000:00: root bus resource [mem
> 0x13000000-0x1fffffff]
> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
> pref]
> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
> reconfiguring
> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
> 0x13000000-0x130fffff]
> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
> 64bit pref]
> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
> 64bit pref]
> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
> 64bit]
> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
> 64bit pref]
> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
> bridge
> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
> interrupt
> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
> tx checksumming: ko]
> 
> then
> 
> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
> @0x000000007a484000: EMEM address decode error (EMEM decode error)
> [    3.717747] r8169 0000:01:00.0 eth0: link down

Hmm... that's very odd. It seems like for some reason the PCIe
controller wants to access memory that's below the DRAM. Do you happen
to have the SMMU enabled for PCIe? Can you try adding some debug prints
to the networking driver to find out where this address is coming from?

> The card is alive though, the transfer LED is blinking according to incoming
> traffic, I assume; we can also see that the driver is able to access the hw
> to some extent. The kernel version is 4.9rc7 with just this series on top.
> The same board/card also works in U-boot.

I've tested this on a recent linux-next, so perhaps that's something
else to try out. I wouldn't expect v4.9-rc7 to have any issues with a
PCI network driver, but who knows.

Thanks,
Thierry

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 801 bytes --]

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 3903 bytes --]

On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
> works.. Relevant parts of bootlog:
> 
> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
> lanes
> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> signature: 00000008
> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
> lanes
> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> signature: 00000000
> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
> 0000:00
> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
> [    3.217343] pci_bus 0000:00: root bus resource [mem
> 0x13000000-0x1fffffff]
> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
> pref]
> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
> reconfiguring
> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
> 0x13000000-0x130fffff]
> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
> 64bit pref]
> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
> 64bit pref]
> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
> 64bit]
> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
> 64bit pref]
> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
> bridge
> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
> interrupt
> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
> tx checksumming: ko]
> 
> then
> 
> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
> @0x000000007a484000: EMEM address decode error (EMEM decode error)
> [    3.717747] r8169 0000:01:00.0 eth0: link down

Hmm... that's very odd. It seems like for some reason the PCIe
controller wants to access memory that's below the DRAM. Do you happen
to have the SMMU enabled for PCIe? Can you try adding some debug prints
to the networking driver to find out where this address is coming from?

> The card is alive though, the transfer LED is blinking according to incoming
> traffic, I assume; we can also see that the driver is able to access the hw
> to some extent. The kernel version is 4.9rc7 with just this series on top.
> The same board/card also works in U-boot.

I've tested this on a recent linux-next, so perhaps that's something
else to try out. I wouldn't expect v4.9-rc7 to have any issues with a
PCI network driver, but who knows.

Thanks,
Thierry

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 801 bytes --]

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

On 11/30/2016 07:48 PM, Thierry Reding wrote:
> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>> works.. Relevant parts of bootlog:
>>
>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
>> lanes
>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>> signature: 00000008
>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
>> lanes
>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>> signature: 00000000
>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
>> 0000:00
>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>> 0x13000000-0x1fffffff]
>> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
>> pref]
>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
>> reconfiguring
>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>> 0x13000000-0x130fffff]
>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
>> 64bit pref]
>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
>> 64bit pref]
>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
>> 64bit]
>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
>> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
>> 64bit pref]
>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
>> bridge
>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>> interrupt
>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
>> tx checksumming: ko]
>>
>> then
>>
>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>
> Hmm... that's very odd. It seems like for some reason the PCIe
> controller wants to access memory that's below the DRAM. Do you happen
> to have the SMMU enabled for PCIe? Can you try adding some debug prints
> to the networking driver to find out where this address is coming from?

SMMU is disabled; I'll try adding debug prints. The behavior certainly 
looks pretty strange.

>
>> The card is alive though, the transfer LED is blinking according to incoming
>> traffic, I assume; we can also see that the driver is able to access the hw
>> to some extent. The kernel version is 4.9rc7 with just this series on top.
>> The same board/card also works in U-boot.
>
> I've tested this on a recent linux-next, so perhaps that's something
> else to try out. I wouldn't expect v4.9-rc7 to have any issues with a
> PCI network driver, but who knows.

I did test earlier with -next too but with some random set of patches 
and config options, but I agree that it's not likely to be the reason. I 
had some IP stack (UDPv6 warnings IIRC?) and lockdep issues or somesuch 
with recent -next's so I switched to rc7.

>
> Thanks,
> Thierry
>

Thanks,
Mikko.

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

On 11/30/2016 07:48 PM, Thierry Reding wrote:
> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>> works.. Relevant parts of bootlog:
>>
>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
>> lanes
>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>> signature: 00000008
>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
>> lanes
>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>> signature: 00000000
>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
>> 0000:00
>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>> 0x13000000-0x1fffffff]
>> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
>> pref]
>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
>> reconfiguring
>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>> 0x13000000-0x130fffff]
>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
>> 64bit pref]
>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
>> 64bit pref]
>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
>> 64bit]
>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
>> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
>> 64bit pref]
>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
>> bridge
>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>> interrupt
>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
>> tx checksumming: ko]
>>
>> then
>>
>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>
> Hmm... that's very odd. It seems like for some reason the PCIe
> controller wants to access memory that's below the DRAM. Do you happen
> to have the SMMU enabled for PCIe? Can you try adding some debug prints
> to the networking driver to find out where this address is coming from?

SMMU is disabled; I'll try adding debug prints. The behavior certainly 
looks pretty strange.

>
>> The card is alive though, the transfer LED is blinking according to incoming
>> traffic, I assume; we can also see that the driver is able to access the hw
>> to some extent. The kernel version is 4.9rc7 with just this series on top.
>> The same board/card also works in U-boot.
>
> I've tested this on a recent linux-next, so perhaps that's something
> else to try out. I wouldn't expect v4.9-rc7 to have any issues with a
> PCI network driver, but who knows.

I did test earlier with -next too but with some random set of patches 
and config options, but I agree that it's not likely to be the reason. I 
had some IP stack (UDPv6 warnings IIRC?) and lockdep issues or somesuch 
with recent -next's so I switched to rc7.

>
> Thanks,
> Thierry
>

Thanks,
Mikko.

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 4206 bytes --]

On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
> On 11/30/2016 07:48 PM, Thierry Reding wrote:
> > On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
> > > Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
> > > works.. Relevant parts of bootlog:
> > > 
> > > [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> > > [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
> > > lanes
> > > [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> > > signature: 00000008
> > > [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
> > > lanes
> > > [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> > > signature: 00000000
> > > [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> > > [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
> > > 0000:00
> > > [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
> > > [    3.217343] pci_bus 0000:00: root bus resource [mem
> > > 0x13000000-0x1fffffff]
> > > [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
> > > pref]
> > > [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> > > [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
> > > reconfiguring
> > > [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
> > > 0x13000000-0x130fffff]
> > > [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
> > > 64bit pref]
> > > [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
> > > [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
> > > 64bit pref]
> > > [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
> > > 64bit]
> > > [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
> > > [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> > > [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
> > > [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
> > > [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
> > > 64bit pref]
> > > [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
> > > bridge
> > > [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> > > [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
> > > interrupt
> > > [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> > > [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> > > [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> > > [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
> > > 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> > > [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
> > > tx checksumming: ko]
> > > 
> > > then
> > > 
> > > [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
> > > @0x000000007a484000: EMEM address decode error (EMEM decode error)
> > > [    3.717747] r8169 0000:01:00.0 eth0: link down
> > 
> > Hmm... that's very odd. It seems like for some reason the PCIe
> > controller wants to access memory that's below the DRAM. Do you happen
> > to have the SMMU enabled for PCIe? Can you try adding some debug prints
> > to the networking driver to find out where this address is coming from?
> 
> SMMU is disabled; I'll try adding debug prints. The behavior certainly looks
> pretty strange.

Maybe you can also find out if at any point in the above the driver is
actually accessing the I/O ports. I don't think we've ever tested that
particular part very much.

I seem to be using a very similar card to yours, which makes it all the
more surprising that it isn't working for you.

Thierry

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 801 bytes --]

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 4206 bytes --]

On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
> On 11/30/2016 07:48 PM, Thierry Reding wrote:
> > On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
> > > Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
> > > works.. Relevant parts of bootlog:
> > > 
> > > [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
> > > [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
> > > lanes
> > > [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> > > signature: 00000008
> > > [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
> > > lanes
> > > [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
> > > signature: 00000000
> > > [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
> > > [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
> > > [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
> > > 0000:00
> > > [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
> > > [    3.217343] pci_bus 0000:00: root bus resource [mem
> > > 0x13000000-0x1fffffff]
> > > [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
> > > pref]
> > > [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
> > > [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
> > > reconfiguring
> > > [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
> > > 0x13000000-0x130fffff]
> > > [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
> > > 64bit pref]
> > > [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
> > > [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
> > > 64bit pref]
> > > [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
> > > 64bit]
> > > [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
> > > [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
> > > [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
> > > [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
> > > [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
> > > 64bit pref]
> > > [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
> > > bridge
> > > [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
> > > [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
> > > interrupt
> > > [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
> > > [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
> > > [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
> > > [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
> > > 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
> > > [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
> > > tx checksumming: ko]
> > > 
> > > then
> > > 
> > > [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
> > > @0x000000007a484000: EMEM address decode error (EMEM decode error)
> > > [    3.717747] r8169 0000:01:00.0 eth0: link down
> > 
> > Hmm... that's very odd. It seems like for some reason the PCIe
> > controller wants to access memory that's below the DRAM. Do you happen
> > to have the SMMU enabled for PCIe? Can you try adding some debug prints
> > to the networking driver to find out where this address is coming from?
> 
> SMMU is disabled; I'll try adding debug prints. The behavior certainly looks
> pretty strange.

Maybe you can also find out if at any point in the above the driver is
actually accessing the I/O ports. I don't think we've ever tested that
particular part very much.

I seem to be using a very similar card to yours, which makes it all the
more surprising that it isn't working for you.

Thierry

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 801 bytes --]

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Vidya Sagar]

Is it possible that, this NIC card's DMA is only 32-bit capable (can be 
confirmed from lspci -vv output) and since SMMU is disabled, allocated 
memory's physical address happen to fall beyond 0xFFFF_FFFF region, so, 
 >32-bits are stripped off by DMA controller of NIC, resulting in 
accesses to random addresses?


On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
> * PGP Signed by an unknown key
>
> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>>>> works.. Relevant parts of bootlog:
>>>>
>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
>>>> lanes
>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>>>> signature: 00000008
>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
>>>> lanes
>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>>>> signature: 00000000
>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
>>>> 0000:00
>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>> 0x13000000-0x1fffffff]
>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
>>>> pref]
>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
>>>> reconfiguring
>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>> 0x13000000-0x130fffff]
>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
>>>> 64bit pref]
>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
>>>> 64bit pref]
>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
>>>> 64bit]
>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
>>>> 64bit pref]
>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
>>>> bridge
>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>> interrupt
>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
>>>> tx checksumming: ko]
>>>>
>>>> then
>>>>
>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>> controller wants to access memory that's below the DRAM. Do you happen
>>> to have the SMMU enabled for PCIe? Can you try adding some debug prints
>>> to the networking driver to find out where this address is coming from?
>> SMMU is disabled; I'll try adding debug prints. The behavior certainly looks
>> pretty strange.
> Maybe you can also find out if at any point in the above the driver is
> actually accessing the I/O ports. I don't think we've ever tested that
> particular part very much.
>
> I seem to be using a very similar card to yours, which makes it all the
> more surprising that it isn't working for you.
>
> Thierry
>
> * Unknown Key
> * 0x7F3EB3A1

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Vidya Sagar]

Is it possible that, this NIC card's DMA is only 32-bit capable (can be 
confirmed from lspci -vv output) and since SMMU is disabled, allocated 
memory's physical address happen to fall beyond 0xFFFF_FFFF region, so, 
 >32-bits are stripped off by DMA controller of NIC, resulting in 
accesses to random addresses?


On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
> * PGP Signed by an unknown key
>
> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>>>> works.. Relevant parts of bootlog:
>>>>
>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1 configuration
>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0, using 4
>>>> lanes
>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>>>> signature: 00000008
>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1, using 1
>>>> lanes
>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin change,
>>>> signature: 00000000
>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down, retrying
>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down, ignoring
>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge to bus
>>>> 0000:00
>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>> 0x13000000-0x1fffffff]
>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem 0x20000000-0x3fffffff
>>>> pref]
>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus 00-00]),
>>>> reconfiguring
>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>> 0x13000000-0x130fffff]
>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem 0x20000000-0x200fffff
>>>> 64bit pref]
>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem 0x20000000-0x20003fff
>>>> 64bit pref]
>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem 0x13000000-0x13000fff
>>>> 64bit]
>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem 0x13000000-0x130fffff]
>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem 0x20000000-0x200fffff
>>>> 64bit pref]
>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate host
>>>> bridge
>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>> interrupt
>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME interrupt
>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes,
>>>> tx checksumming: ko]
>>>>
>>>> then
>>>>
>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>> controller wants to access memory that's below the DRAM. Do you happen
>>> to have the SMMU enabled for PCIe? Can you try adding some debug prints
>>> to the networking driver to find out where this address is coming from?
>> SMMU is disabled; I'll try adding debug prints. The behavior certainly looks
>> pretty strange.
> Maybe you can also find out if at any point in the above the driver is
> actually accessing the I/O ports. I don't think we've ever tested that
> particular part very much.
>
> I seem to be using a very similar card to yours, which makes it all the
> more surprising that it isn't working for you.
>
> Thierry
>
> * Unknown Key
> * 0x7F3EB3A1

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

It would seem that it is 64-capable based on lspci and code in the 
driver. Looks like the memory that is allocated is coming from below 4G 
as well.

On 11/30/2016 08:39 PM, Vidya Sagar wrote:
> Is it possible that, this NIC card's DMA is only 32-bit capable (can be
> confirmed from lspci -vv output) and since SMMU is disabled, allocated
> memory's physical address happen to fall beyond 0xFFFF_FFFF region, so,
>>32-bits are stripped off by DMA controller of NIC, resulting in
> accesses to random addresses?
>
>
> On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
>> * PGP Signed by an unknown key
>>
>> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>>>>> works.. Relevant parts of bootlog:
>>>>>
>>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1
>>>>> configuration
>>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0,
>>>>> using 4
>>>>> lanes
>>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>> change,
>>>>> signature: 00000008
>>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1,
>>>>> using 1
>>>>> lanes
>>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>> change,
>>>>> signature: 00000000
>>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> ignoring
>>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge
>>>>> to bus
>>>>> 0000:00
>>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>>> 0x13000000-0x1fffffff]
>>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem
>>>>> 0x20000000-0x3fffffff
>>>>> pref]
>>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus
>>>>> 00-00]),
>>>>> reconfiguring
>>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>>> 0x13000000-0x130fffff]
>>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem
>>>>> 0x20000000-0x200fffff
>>>>> 64bit pref]
>>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem
>>>>> 0x20000000-0x20003fff
>>>>> 64bit pref]
>>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem
>>>>> 0x13000000-0x13000fff
>>>>> 64bit]
>>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem
>>>>> 0x13000000-0x130fffff]
>>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem
>>>>> 0x20000000-0x200fffff
>>>>> 64bit pref]
>>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate
>>>>> host
>>>>> bridge
>>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>>> interrupt
>>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME
>>>>> interrupt
>>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames:
>>>>> 9200 bytes,
>>>>> tx checksumming: ko]
>>>>>
>>>>> then
>>>>>
>>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>>> controller wants to access memory that's below the DRAM. Do you happen
>>>> to have the SMMU enabled for PCIe? Can you try adding some debug prints
>>>> to the networking driver to find out where this address is coming from?
>>> SMMU is disabled; I'll try adding debug prints. The behavior
>>> certainly looks
>>> pretty strange.
>> Maybe you can also find out if at any point in the above the driver is
>> actually accessing the I/O ports. I don't think we've ever tested that
>> particular part very much.
>>
>> I seem to be using a very similar card to yours, which makes it all the
>> more surprising that it isn't working for you.
>>
>> Thierry
>>
>> * Unknown Key
>> * 0x7F3EB3A1
>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

It would seem that it is 64-capable based on lspci and code in the 
driver. Looks like the memory that is allocated is coming from below 4G 
as well.

On 11/30/2016 08:39 PM, Vidya Sagar wrote:
> Is it possible that, this NIC card's DMA is only 32-bit capable (can be
> confirmed from lspci -vv output) and since SMMU is disabled, allocated
> memory's physical address happen to fall beyond 0xFFFF_FFFF region, so,
>>32-bits are stripped off by DMA controller of NIC, resulting in
> accesses to random addresses?
>
>
> On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
>> * PGP Signed by an unknown key
>>
>> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it /almost/
>>>>> works.. Relevant parts of bootlog:
>>>>>
>>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1
>>>>> configuration
>>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0,
>>>>> using 4
>>>>> lanes
>>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>> change,
>>>>> signature: 00000008
>>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1,
>>>>> using 1
>>>>> lanes
>>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>> change,
>>>>> signature: 00000000
>>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> retrying
>>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>> ignoring
>>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge
>>>>> to bus
>>>>> 0000:00
>>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>>> 0x13000000-0x1fffffff]
>>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem
>>>>> 0x20000000-0x3fffffff
>>>>> pref]
>>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus
>>>>> 00-00]),
>>>>> reconfiguring
>>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>>> 0x13000000-0x130fffff]
>>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem
>>>>> 0x20000000-0x200fffff
>>>>> 64bit pref]
>>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem
>>>>> 0x20000000-0x20003fff
>>>>> 64bit pref]
>>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem
>>>>> 0x13000000-0x13000fff
>>>>> 64bit]
>>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem
>>>>> 0x13000000-0x130fffff]
>>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem
>>>>> 0x20000000-0x200fffff
>>>>> 64bit pref]
>>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate
>>>>> host
>>>>> bridge
>>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>>> interrupt
>>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME
>>>>> interrupt
>>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames:
>>>>> 9200 bytes,
>>>>> tx checksumming: ko]
>>>>>
>>>>> then
>>>>>
>>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>>> controller wants to access memory that's below the DRAM. Do you happen
>>>> to have the SMMU enabled for PCIe? Can you try adding some debug prints
>>>> to the networking driver to find out where this address is coming from?
>>> SMMU is disabled; I'll try adding debug prints. The behavior
>>> certainly looks
>>> pretty strange.
>> Maybe you can also find out if at any point in the above the driver is
>> actually accessing the I/O ports. I don't think we've ever tested that
>> particular part very much.
>>
>> I seem to be using a very similar card to yours, which makes it all the
>> more surprising that it isn't working for you.
>>
>> Thierry
>>
>> * Unknown Key
>> * 0x7F3EB3A1
>

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

I added debug prints but couldn't find any values matching the faulting 
address. I noticed that the error only happens if a cable is plugged in 
when I run "ifconfig enp1s0 up", so perhaps it is related to receiving a 
frame?

In any case, since other people's cards are working, I think my issue 
shouldn't block merging this series. If we find an issue in it then we 
can fix it later.

Cheers,
Mikko.

On 30.11.2016 21:38, Mikko Perttunen wrote:
> It would seem that it is 64-capable based on lspci and code in the
> driver. Looks like the memory that is allocated is coming from below 4G
> as well.
>
> On 11/30/2016 08:39 PM, Vidya Sagar wrote:
>> Is it possible that, this NIC card's DMA is only 32-bit capable (can be
>> confirmed from lspci -vv output) and since SMMU is disabled, allocated
>> memory's physical address happen to fall beyond 0xFFFF_FFFF region, so,
>>> 32-bits are stripped off by DMA controller of NIC, resulting in
>> accesses to random addresses?
>>
>>
>> On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
>>> * PGP Signed by an unknown key
>>>
>>> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>>>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it
>>>>>> /almost/
>>>>>> works.. Relevant parts of bootlog:
>>>>>>
>>>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1
>>>>>> configuration
>>>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0,
>>>>>> using 4
>>>>>> lanes
>>>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>>> change,
>>>>>> signature: 00000008
>>>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1,
>>>>>> using 1
>>>>>> lanes
>>>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>>> change,
>>>>>> signature: 00000000
>>>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> ignoring
>>>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge
>>>>>> to bus
>>>>>> 0000:00
>>>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>>>> 0x13000000-0x1fffffff]
>>>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem
>>>>>> 0x20000000-0x3fffffff
>>>>>> pref]
>>>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus
>>>>>> 00-00]),
>>>>>> reconfiguring
>>>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>>>> 0x13000000-0x130fffff]
>>>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem
>>>>>> 0x20000000-0x200fffff
>>>>>> 64bit pref]
>>>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem
>>>>>> 0x20000000-0x20003fff
>>>>>> 64bit pref]
>>>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem
>>>>>> 0x13000000-0x13000fff
>>>>>> 64bit]
>>>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem
>>>>>> 0x13000000-0x130fffff]
>>>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem
>>>>>> 0x20000000-0x200fffff
>>>>>> 64bit pref]
>>>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate
>>>>>> host
>>>>>> bridge
>>>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>>>> interrupt
>>>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME
>>>>>> interrupt
>>>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames:
>>>>>> 9200 bytes,
>>>>>> tx checksumming: ko]
>>>>>>
>>>>>> then
>>>>>>
>>>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>>>> controller wants to access memory that's below the DRAM. Do you happen
>>>>> to have the SMMU enabled for PCIe? Can you try adding some debug
>>>>> prints
>>>>> to the networking driver to find out where this address is coming
>>>>> from?
>>>> SMMU is disabled; I'll try adding debug prints. The behavior
>>>> certainly looks
>>>> pretty strange.
>>> Maybe you can also find out if at any point in the above the driver is
>>> actually accessing the I/O ports. I don't think we've ever tested that
>>> particular part very much.
>>>
>>> I seem to be using a very similar card to yours, which makes it all the
>>> more surprising that it isn't working for you.
>>>
>>> Thierry
>>>
>>> * Unknown Key
>>> * 0x7F3EB3A1
>>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-tegra" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 10/10] arm64: tegra: Enable PCIe on Jetson TX1

[Mikko Perttunen]

I added debug prints but couldn't find any values matching the faulting 
address. I noticed that the error only happens if a cable is plugged in 
when I run "ifconfig enp1s0 up", so perhaps it is related to receiving a 
frame?

In any case, since other people's cards are working, I think my issue 
shouldn't block merging this series. If we find an issue in it then we 
can fix it later.

Cheers,
Mikko.

On 30.11.2016 21:38, Mikko Perttunen wrote:
> It would seem that it is 64-capable based on lspci and code in the
> driver. Looks like the memory that is allocated is coming from below 4G
> as well.
>
> On 11/30/2016 08:39 PM, Vidya Sagar wrote:
>> Is it possible that, this NIC card's DMA is only 32-bit capable (can be
>> confirmed from lspci -vv output) and since SMMU is disabled, allocated
>> memory's physical address happen to fall beyond 0xFFFF_FFFF region, so,
>>> 32-bits are stripped off by DMA controller of NIC, resulting in
>> accesses to random addresses?
>>
>>
>> On Wednesday 30 November 2016 11:44 PM, Thierry Reding wrote:
>>> * PGP Signed by an unknown key
>>>
>>> On Wed, Nov 30, 2016 at 08:06:45PM +0200, Mikko Perttunen wrote:
>>>> On 11/30/2016 07:48 PM, Thierry Reding wrote:
>>>>> On Mon, Nov 28, 2016 at 06:54:44PM +0200, Mikko Perttunen wrote:
>>>>>> Testing this series with a Jetson TX1 + r8168e PCI-E card, it
>>>>>> /almost/
>>>>>> works.. Relevant parts of bootlog:
>>>>>>
>>>>>> [    1.876191] tegra-pcie 1003000.pcie-controller: 4x1, 1x1
>>>>>> configuration
>>>>>> [    1.884200] tegra-pcie 1003000.pcie-controller: probing port 0,
>>>>>> using 4
>>>>>> lanes
>>>>>> [    1.893368] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>>> change,
>>>>>> signature: 00000008
>>>>>> [    1.948049] tegra-pcie 1003000.pcie-controller: probing port 1,
>>>>>> using 1
>>>>>> lanes
>>>>>> [    1.957209] tegra-pcie 1003000.pcie-controller: Slot present pin
>>>>>> change,
>>>>>> signature: 00000000
>>>>>> [    2.367748] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    2.778307] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    3.188888] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> retrying
>>>>>> [    3.197344] tegra-pcie 1003000.pcie-controller: link 1 down,
>>>>>> ignoring
>>>>>> [    3.203931] tegra-pcie 1003000.pcie-controller: PCI host bridge
>>>>>> to bus
>>>>>> 0000:00
>>>>>> [    3.211160] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
>>>>>> [    3.217343] pci_bus 0000:00: root bus resource [mem
>>>>>> 0x13000000-0x1fffffff]
>>>>>> [    3.224218] pci_bus 0000:00: root bus resource [mem
>>>>>> 0x20000000-0x3fffffff
>>>>>> pref]
>>>>>> [    3.231525] pci_bus 0000:00: root bus resource [bus 00-ff]
>>>>>> [    3.237380] pci 0000:00:01.0: bridge configuration invalid ([bus
>>>>>> 00-00]),
>>>>>> reconfiguring
>>>>>> [    3.254499] pci 0000:00:01.0: BAR 14: assigned [mem
>>>>>> 0x13000000-0x130fffff]
>>>>>> [    3.261389] pci 0000:00:01.0: BAR 15: assigned [mem
>>>>>> 0x20000000-0x200fffff
>>>>>> 64bit pref]
>>>>>> [    3.269220] pci 0000:00:01.0: BAR 13: assigned [io  0x1000-0x1fff]
>>>>>> [    3.275412] pci 0000:01:00.0: BAR 4: assigned [mem
>>>>>> 0x20000000-0x20003fff
>>>>>> 64bit pref]
>>>>>> [    3.283172] pci 0000:01:00.0: BAR 2: assigned [mem
>>>>>> 0x13000000-0x13000fff
>>>>>> 64bit]
>>>>>> [    3.290498] pci 0000:01:00.0: BAR 0: assigned [io  0x1000-0x10ff]
>>>>>> [    3.296596] pci 0000:00:01.0: PCI bridge to [bus 01]
>>>>>> [    3.301568] pci 0000:00:01.0:   bridge window [io  0x1000-0x1fff]
>>>>>> [    3.307666] pci 0000:00:01.0:   bridge window [mem
>>>>>> 0x13000000-0x130fffff]
>>>>>> [    3.314454] pci 0000:00:01.0:   bridge window [mem
>>>>>> 0x20000000-0x200fffff
>>>>>> 64bit pref]
>>>>>> [    3.322213] pci 0000:00:01.0: nv_msi_ht_cap_quirk didn't locate
>>>>>> host
>>>>>> bridge
>>>>>> [    3.329257] pcieport 0000:00:01.0: enabling device (0000 -> 0003)
>>>>>> [    3.335572] pcieport 0000:00:01.0: Signaling PME through PCIe PME
>>>>>> interrupt
>>>>>> [    3.342537] pci 0000:01:00.0: Signaling PME through PCIe PME
>>>>>> interrupt
>>>>>> [    3.349256] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded
>>>>>> [    3.354858] r8169 0000:01:00.0: enabling device (0000 -> 0003)
>>>>>> [    3.361460] r8169 0000:01:00.0 eth0: RTL8168e/8111e at
>>>>>> 0xffff000008eae000, 98:de:d0:04:25:14, XID 0c200000 IRQ 348
>>>>>> [    3.371812] r8169 0000:01:00.0 eth0: jumbo features [frames:
>>>>>> 9200 bytes,
>>>>>> tx checksumming: ko]
>>>>>>
>>>>>> then
>>>>>>
>>>>>> [    3.706240] tegra-mc 70019000.memory-controller: afiw: write
>>>>>> @0x000000007a484000: EMEM address decode error (EMEM decode error)
>>>>>> [    3.717747] r8169 0000:01:00.0 eth0: link down
>>>>> Hmm... that's very odd. It seems like for some reason the PCIe
>>>>> controller wants to access memory that's below the DRAM. Do you happen
>>>>> to have the SMMU enabled for PCIe? Can you try adding some debug
>>>>> prints
>>>>> to the networking driver to find out where this address is coming
>>>>> from?
>>>> SMMU is disabled; I'll try adding debug prints. The behavior
>>>> certainly looks
>>>> pretty strange.
>>> Maybe you can also find out if at any point in the above the driver is
>>> actually accessing the I/O ports. I don't think we've ever tested that
>>> particular part very much.
>>>
>>> I seem to be using a very similar card to yours, which makes it all the
>>> more surprising that it isn't working for you.
>>>
>>> Thierry
>>>
>>> * Unknown Key
>>> * 0x7F3EB3A1
>>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-tegra" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Bjorn Helgaas]

On Fri, Nov 25, 2016 at 11:57:09AM +0100, Thierry Reding wrote:
> From: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
> 
> This patch makes the existing pci_host_bridge structure a proper device
> that is usable by PCI host drivers in a more standard way. In addition
> to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
> and pci_create_root_bus interfaces, this unfortunately means having to
> add yet another interface doing basically the same thing, and add some
> extra code in the initial step.
> 
> However, this time it's more likely to be extensible enough that we
> won't have to do another one again in the future, and we should be
> able to reduce code much more as a result.
> 
> The main idea is to pull the allocation of 'struct pci_host_bridge' out
> of the registration, and let individual host drivers and architecture
> code fill the members before calling the registration function.

I really like this idea.  Can you include pointers to similar
interfaces in other subsystems?  I'm hoping to reuse existing designs
as much as possible, including using similar names.

> There are a number of things we can do based on this:
> 
> * Use a single memory allocation for the driver-specific structure
>   and the generic PCI host bridge
> * consolidate the contents of driver specific structures by moving
>   them into pci_host_bridge
> * Add a consistent interface for removing a PCI host bridge again
>   when unloading a host driver module
> * Replace the architecture specific __weak pcibios_* functions with
>   callbacks in a pci_host_bridge device
> * Move common boilerplate code from host drivers into the generic
>   function, based on contents of the structure
> * Extend pci_host_bridge with additional members when needed without
>   having to add arguments to pci_scan_*.
> * Move members of struct pci_bus into pci_host_bridge to avoid
>   having lots of identical copies.
> 
> As mentioned in a previous email, one open question is whether we want
> to export a function for allocating a pci_host_bridge device in
> combination with the per-device structure or let the driver itself
> call kzalloc.

The next patch implements the former, doesn't it?  If this is no
longer an open question, let's update this changelog.

> Changes in v3 (Thierry Reding):
> - swap out pci_host_bridge_init() for pci_alloc_host_bridge() with an
>   extra parameter specifying the size of the driver's private data
> - rename pci_host_bridge_register() to pci_register_host_bridge() for
>   more consistency with existing functions
> - split patches into smaller chunks to make diff more readable
> 
> Changes in v2 (Thierry Reding):
> - add a pci_host_bridge_init() helper that drivers can use to perform
>   all the necessary steps to initialize the bridge
> - rename pci_register_host() to pci_host_bridge_register() to reflect
>   the naming used by other functions
> - plug memory leak on registration failure

You can also omit these v2/v3 change comments from the changelog; it's
useful to have them in a 0/n cover letter or after the "---" line in
an individual patch.  I omit them from the git changelogs because
they're not very useful after the patches get merged.

Bjorn

Re: [PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Bjorn Helgaas]

On Fri, Nov 25, 2016 at 11:57:09AM +0100, Thierry Reding wrote:
> From: Arnd Bergmann <arnd@arndb.de>
> 
> This patch makes the existing pci_host_bridge structure a proper device
> that is usable by PCI host drivers in a more standard way. In addition
> to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
> and pci_create_root_bus interfaces, this unfortunately means having to
> add yet another interface doing basically the same thing, and add some
> extra code in the initial step.
> 
> However, this time it's more likely to be extensible enough that we
> won't have to do another one again in the future, and we should be
> able to reduce code much more as a result.
> 
> The main idea is to pull the allocation of 'struct pci_host_bridge' out
> of the registration, and let individual host drivers and architecture
> code fill the members before calling the registration function.

I really like this idea.  Can you include pointers to similar
interfaces in other subsystems?  I'm hoping to reuse existing designs
as much as possible, including using similar names.

> There are a number of things we can do based on this:
> 
> * Use a single memory allocation for the driver-specific structure
>   and the generic PCI host bridge
> * consolidate the contents of driver specific structures by moving
>   them into pci_host_bridge
> * Add a consistent interface for removing a PCI host bridge again
>   when unloading a host driver module
> * Replace the architecture specific __weak pcibios_* functions with
>   callbacks in a pci_host_bridge device
> * Move common boilerplate code from host drivers into the generic
>   function, based on contents of the structure
> * Extend pci_host_bridge with additional members when needed without
>   having to add arguments to pci_scan_*.
> * Move members of struct pci_bus into pci_host_bridge to avoid
>   having lots of identical copies.
> 
> As mentioned in a previous email, one open question is whether we want
> to export a function for allocating a pci_host_bridge device in
> combination with the per-device structure or let the driver itself
> call kzalloc.

The next patch implements the former, doesn't it?  If this is no
longer an open question, let's update this changelog.

> Changes in v3 (Thierry Reding):
> - swap out pci_host_bridge_init() for pci_alloc_host_bridge() with an
>   extra parameter specifying the size of the driver's private data
> - rename pci_host_bridge_register() to pci_register_host_bridge() for
>   more consistency with existing functions
> - split patches into smaller chunks to make diff more readable
> 
> Changes in v2 (Thierry Reding):
> - add a pci_host_bridge_init() helper that drivers can use to perform
>   all the necessary steps to initialize the bridge
> - rename pci_register_host() to pci_host_bridge_register() to reflect
>   the naming used by other functions
> - plug memory leak on registration failure

You can also omit these v2/v3 change comments from the changelog; it's
useful to have them in a 0/n cover letter or after the "---" line in
an individual patch.  I omit them from the git changelogs because
they're not very useful after the patches get merged.

Bjorn

Re: [PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Bjorn Helgaas]

On Fri, Nov 25, 2016 at 11:57:09AM +0100, Thierry Reding wrote:
> From: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
> 
> This patch makes the existing pci_host_bridge structure a proper device
> that is usable by PCI host drivers in a more standard way. In addition
> to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
> and pci_create_root_bus interfaces, this unfortunately means having to
> add yet another interface doing basically the same thing, and add some
> extra code in the initial step.

I applied this whole series to pci/host-tegra for v4.10, thanks!

Bjorn

Re: [PATCH v4 01/10] PCI: Add new method for registering PCI hosts

[Bjorn Helgaas]

On Fri, Nov 25, 2016 at 11:57:09AM +0100, Thierry Reding wrote:
> From: Arnd Bergmann <arnd@arndb.de>
> 
> This patch makes the existing pci_host_bridge structure a proper device
> that is usable by PCI host drivers in a more standard way. In addition
> to the existing pci_scan_bus, pci_scan_root_bus, pci_scan_root_bus_msi,
> and pci_create_root_bus interfaces, this unfortunately means having to
> add yet another interface doing basically the same thing, and add some
> extra code in the initial step.

I applied this whole series to pci/host-tegra for v4.10, thanks!

Bjorn

Re: [PATCH v4 04/10] PCI: tegra: Use new pci_register_host_bridge() interface

[Tomasz Nowicki]

On 25.11.2016 11:57, Thierry Reding wrote:
> From: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
>
> Tegra is one of the remaining platforms that still use the traditional
> pci_common_init_dev() interface for probing PCI host bridges.
>
> This demonstrates how to convert it to the pci_register_host interface
> I just added in a previous patch. This leads to a more linear probe
> sequence that can handle errors better because we avoid callbacks into
> the driver, and it makes the driver architecture independent.
>
> Changes in v4 (Thierry Reding):
> - update for changes in core to deal with driver-private data
>
> Signed-off-by: Arnd Bergmann <arnd-r2nGTMty4D4@public.gmane.org>
> Signed-off-by: Thierry Reding <treding-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/pci/host/pci-tegra.c | 105 ++++++++++++++++++++++---------------------
>  1 file changed, 54 insertions(+), 51 deletions(-)
>
> diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
> index 8dfccf733241..d5206fa53353 100644
> --- a/drivers/pci/host/pci-tegra.c
> +++ b/drivers/pci/host/pci-tegra.c
> @@ -322,11 +322,6 @@ struct tegra_pcie_bus {
>  	unsigned int nr;
>  };
>
> -static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
> -{
> -	return sys->private_data;
> -}
> -
>  static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
>  			      unsigned long offset)
>  {
> @@ -430,7 +425,8 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>
>  static int tegra_pcie_add_bus(struct pci_bus *bus)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct tegra_pcie_bus *b;
>
>  	b = tegra_pcie_bus_alloc(pcie, bus->number);
> @@ -444,7 +440,8 @@ static int tegra_pcie_add_bus(struct pci_bus *bus)
>
>  static void tegra_pcie_remove_bus(struct pci_bus *child)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(child);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct tegra_pcie_bus *bus, *tmp;
>
>  	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
> @@ -461,7 +458,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>  					unsigned int devfn,
>  					int where)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct device *dev = pcie->dev;
>  	void __iomem *addr = NULL;
>
> @@ -610,39 +608,31 @@ static void tegra_pcie_relax_enable(struct pci_dev *dev)
>  }
>  DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
>
> -static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
> +static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(sys);
> +	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
> +	struct list_head *windows = &host->windows;
>  	struct device *dev = pcie->dev;
>  	int err;
>
> -	sys->mem_offset = pcie->offset.mem;
> -	sys->io_offset = pcie->offset.io;
> +	pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
> +	pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
> +	pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
> +	pci_add_resource(windows, &pcie->busn);
>
> -	err = devm_request_resource(dev, &iomem_resource, &pcie->io);
> +	err = devm_request_pci_bus_resources(dev, windows);
>  	if (err < 0)
>  		return err;
>
> -	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
> -	if (!err)
> -		pci_add_resource_offset(&sys->resources, &pcie->pio,
> -					sys->io_offset);
> -
> -	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
> -	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
> -				sys->mem_offset);
> -	pci_add_resource(&sys->resources, &pcie->busn);
> -
> -	err = devm_request_pci_bus_resources(dev, &sys->resources);
> -	if (err < 0)
> -		return err;
> +	pci_remap_iospace(&pcie->pio, pcie->io.start);
>
> -	return 1;
> +	return 0;
>  }
>
>  static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	int irq;
>
>  	tegra_cpuidle_pcie_irqs_in_use();
> @@ -1499,10 +1489,11 @@ static const struct irq_domain_ops msi_domain_ops = {
>
>  static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  {
> -	struct device *dev = pcie->dev;
> -	struct platform_device *pdev = to_platform_device(dev);
> +	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
> +	struct platform_device *pdev = to_platform_device(pcie->dev);
>  	const struct tegra_pcie_soc *soc = pcie->soc;
>  	struct tegra_msi *msi = &pcie->msi;
> +	struct device *dev = pcie->dev;
>  	unsigned long base;
>  	int err;
>  	u32 reg;
> @@ -1559,6 +1550,8 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  	reg |= AFI_INTR_MASK_MSI_MASK;
>  	afi_writel(pcie, reg, AFI_INTR_MASK);
>
> +	host->msi = &msi->chip;
> +
>  	return 0;
>
>  err:
> @@ -2021,11 +2014,10 @@ static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
>  	return false;
>  }
>
> -static int tegra_pcie_enable(struct tegra_pcie *pcie)
> +static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
>  {
>  	struct device *dev = pcie->dev;
>  	struct tegra_pcie_port *port, *tmp;
> -	struct hw_pci hw;
>
>  	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
>  		dev_info(dev, "probing port %u, using %u lanes\n",
> @@ -2041,21 +2033,6 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
>  		tegra_pcie_port_disable(port);
>  		tegra_pcie_port_free(port);
>  	}
> -
> -	memset(&hw, 0, sizeof(hw));
> -
> -#ifdef CONFIG_PCI_MSI
> -	hw.msi_ctrl = &pcie->msi.chip;
> -#endif
> -
> -	hw.nr_controllers = 1;
> -	hw.private_data = (void **)&pcie;
> -	hw.setup = tegra_pcie_setup;
> -	hw.map_irq = tegra_pcie_map_irq;
> -	hw.ops = &tegra_pcie_ops;
> -
> -	pci_common_init_dev(dev, &hw);
> -	return 0;
>  }
>
>  static const struct tegra_pcie_soc tegra20_pcie = {
> @@ -2217,13 +2194,17 @@ static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
>  static int tegra_pcie_probe(struct platform_device *pdev)
>  {
>  	struct device *dev = &pdev->dev;
> +	struct pci_host_bridge *host;
>  	struct tegra_pcie *pcie;
> +	struct pci_bus *child;
>  	int err;
>
> -	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
> -	if (!pcie)
> +	host = pci_alloc_host_bridge(sizeof(*pcie));
> +	if (!host)
>  		return -ENOMEM;
>
> +	pcie = pci_host_bridge_priv(host);
> +
>  	pcie->soc = of_device_get_match_data(dev);
>  	INIT_LIST_HEAD(&pcie->buses);
>  	INIT_LIST_HEAD(&pcie->ports);
> @@ -2243,6 +2224,10 @@ static int tegra_pcie_probe(struct platform_device *pdev)
>  	if (err)
>  		goto put_resources;
>
> +	err = tegra_pcie_request_resources(pcie);
> +	if (err)
> +		goto put_resources;
> +
>  	/* setup the AFI address translations */
>  	tegra_pcie_setup_translations(pcie);
>
> @@ -2254,12 +2239,30 @@ static int tegra_pcie_probe(struct platform_device *pdev)
>  		}
>  	}
>
> -	err = tegra_pcie_enable(pcie);
> +	tegra_pcie_enable_ports(pcie);
> +
> +	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
> +	host->busnr = pcie->busn.start;
> +	host->dev.parent = &pdev->dev;
> +	host->ops = &tegra_pcie_ops;

What about:
host->dev.release = pci_release_host_bridge_dev;
which was used in previous approach (pci_common_init_dev())

Tomasz

Re: [PATCH v4 04/10] PCI: tegra: Use new pci_register_host_bridge() interface

[Tomasz Nowicki]

On 25.11.2016 11:57, Thierry Reding wrote:
> From: Arnd Bergmann <arnd@arndb.de>
>
> Tegra is one of the remaining platforms that still use the traditional
> pci_common_init_dev() interface for probing PCI host bridges.
>
> This demonstrates how to convert it to the pci_register_host interface
> I just added in a previous patch. This leads to a more linear probe
> sequence that can handle errors better because we avoid callbacks into
> the driver, and it makes the driver architecture independent.
>
> Changes in v4 (Thierry Reding):
> - update for changes in core to deal with driver-private data
>
> Signed-off-by: Arnd Bergmann <arnd@arndb.de>
> Signed-off-by: Thierry Reding <treding@nvidia.com>
> ---
>  drivers/pci/host/pci-tegra.c | 105 ++++++++++++++++++++++---------------------
>  1 file changed, 54 insertions(+), 51 deletions(-)
>
> diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
> index 8dfccf733241..d5206fa53353 100644
> --- a/drivers/pci/host/pci-tegra.c
> +++ b/drivers/pci/host/pci-tegra.c
> @@ -322,11 +322,6 @@ struct tegra_pcie_bus {
>  	unsigned int nr;
>  };
>
> -static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
> -{
> -	return sys->private_data;
> -}
> -
>  static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
>  			      unsigned long offset)
>  {
> @@ -430,7 +425,8 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
>
>  static int tegra_pcie_add_bus(struct pci_bus *bus)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct tegra_pcie_bus *b;
>
>  	b = tegra_pcie_bus_alloc(pcie, bus->number);
> @@ -444,7 +440,8 @@ static int tegra_pcie_add_bus(struct pci_bus *bus)
>
>  static void tegra_pcie_remove_bus(struct pci_bus *child)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(child);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct tegra_pcie_bus *bus, *tmp;
>
>  	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
> @@ -461,7 +458,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
>  					unsigned int devfn,
>  					int where)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	struct device *dev = pcie->dev;
>  	void __iomem *addr = NULL;
>
> @@ -610,39 +608,31 @@ static void tegra_pcie_relax_enable(struct pci_dev *dev)
>  }
>  DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
>
> -static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
> +static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(sys);
> +	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
> +	struct list_head *windows = &host->windows;
>  	struct device *dev = pcie->dev;
>  	int err;
>
> -	sys->mem_offset = pcie->offset.mem;
> -	sys->io_offset = pcie->offset.io;
> +	pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
> +	pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
> +	pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
> +	pci_add_resource(windows, &pcie->busn);
>
> -	err = devm_request_resource(dev, &iomem_resource, &pcie->io);
> +	err = devm_request_pci_bus_resources(dev, windows);
>  	if (err < 0)
>  		return err;
>
> -	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
> -	if (!err)
> -		pci_add_resource_offset(&sys->resources, &pcie->pio,
> -					sys->io_offset);
> -
> -	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
> -	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
> -				sys->mem_offset);
> -	pci_add_resource(&sys->resources, &pcie->busn);
> -
> -	err = devm_request_pci_bus_resources(dev, &sys->resources);
> -	if (err < 0)
> -		return err;
> +	pci_remap_iospace(&pcie->pio, pcie->io.start);
>
> -	return 1;
> +	return 0;
>  }
>
>  static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
>  {
> -	struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
> +	struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
> +	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
>  	int irq;
>
>  	tegra_cpuidle_pcie_irqs_in_use();
> @@ -1499,10 +1489,11 @@ static const struct irq_domain_ops msi_domain_ops = {
>
>  static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  {
> -	struct device *dev = pcie->dev;
> -	struct platform_device *pdev = to_platform_device(dev);
> +	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
> +	struct platform_device *pdev = to_platform_device(pcie->dev);
>  	const struct tegra_pcie_soc *soc = pcie->soc;
>  	struct tegra_msi *msi = &pcie->msi;
> +	struct device *dev = pcie->dev;
>  	unsigned long base;
>  	int err;
>  	u32 reg;
> @@ -1559,6 +1550,8 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
>  	reg |= AFI_INTR_MASK_MSI_MASK;
>  	afi_writel(pcie, reg, AFI_INTR_MASK);
>
> +	host->msi = &msi->chip;
> +
>  	return 0;
>
>  err:
> @@ -2021,11 +2014,10 @@ static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
>  	return false;
>  }
>
> -static int tegra_pcie_enable(struct tegra_pcie *pcie)
> +static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
>  {
>  	struct device *dev = pcie->dev;
>  	struct tegra_pcie_port *port, *tmp;
> -	struct hw_pci hw;
>
>  	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
>  		dev_info(dev, "probing port %u, using %u lanes\n",
> @@ -2041,21 +2033,6 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
>  		tegra_pcie_port_disable(port);
>  		tegra_pcie_port_free(port);
>  	}
> -
> -	memset(&hw, 0, sizeof(hw));
> -
> -#ifdef CONFIG_PCI_MSI
> -	hw.msi_ctrl = &pcie->msi.chip;
> -#endif
> -
> -	hw.nr_controllers = 1;
> -	hw.private_data = (void **)&pcie;
> -	hw.setup = tegra_pcie_setup;
> -	hw.map_irq = tegra_pcie_map_irq;
> -	hw.ops = &tegra_pcie_ops;
> -
> -	pci_common_init_dev(dev, &hw);
> -	return 0;
>  }
>
>  static const struct tegra_pcie_soc tegra20_pcie = {
> @@ -2217,13 +2194,17 @@ static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
>  static int tegra_pcie_probe(struct platform_device *pdev)
>  {
>  	struct device *dev = &pdev->dev;
> +	struct pci_host_bridge *host;
>  	struct tegra_pcie *pcie;
> +	struct pci_bus *child;
>  	int err;
>
> -	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
> -	if (!pcie)
> +	host = pci_alloc_host_bridge(sizeof(*pcie));
> +	if (!host)
>  		return -ENOMEM;
>
> +	pcie = pci_host_bridge_priv(host);
> +
>  	pcie->soc = of_device_get_match_data(dev);
>  	INIT_LIST_HEAD(&pcie->buses);
>  	INIT_LIST_HEAD(&pcie->ports);
> @@ -2243,6 +2224,10 @@ static int tegra_pcie_probe(struct platform_device *pdev)
>  	if (err)
>  		goto put_resources;
>
> +	err = tegra_pcie_request_resources(pcie);
> +	if (err)
> +		goto put_resources;
> +
>  	/* setup the AFI address translations */
>  	tegra_pcie_setup_translations(pcie);
>
> @@ -2254,12 +2239,30 @@ static int tegra_pcie_probe(struct platform_device *pdev)
>  		}
>  	}
>
> -	err = tegra_pcie_enable(pcie);
> +	tegra_pcie_enable_ports(pcie);
> +
> +	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
> +	host->busnr = pcie->busn.start;
> +	host->dev.parent = &pdev->dev;
> +	host->ops = &tegra_pcie_ops;

What about:
host->dev.release = pci_release_host_bridge_dev;
which was used in previous approach (pci_common_init_dev())

Tomasz