Re: [PATCH 3/5] misc: Add Apple DART driver

[Mark Kettenis <mark.kettenis@xs4all.nl>]

> From: Simon Glass <sjg@chromium.org>
> Date: Mon, 20 Sep 2021 19:11:25 -0600

Hi Simon,

> [..]
> 
> > > > +static int apple_dart_bind(struct udevice *dev)
> > > > +{
> > > > +       void *base;
> > > > +       int sid, i;
> > > > +
> > > > +       base = dev_read_addr_ptr(dev);
> > > > +       if (!base)
> > > > +               return -EINVAL;
> > > > +
> > > > +       u32 params2 = readl(base + DART_PARAMS2);
> > > > +       if (params2 & DART_PARAMS2_BYPASS_SUPPORT) {
> > > > +               for (sid = 0; sid < 16; sid++) {
> > > > +                       writel(DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF,
> > > > +                              base + DART_TCR(sid));
> > > > +                       for (i = 0; i < 4; i++)
> > > > +                               writel(0, base + DART_TTBR(sid, i));
> > > > +               }
> > > > +       }
> > >
> > > Not allowed hardware access in bind(). Can this more to probe() ?
> >
> > Well, I need to make sure that this happens before other drivers get
> > probed (in particular the xhci-dwc3 driver).  Is there a better
> > mechanism to achieve that?
> 
> If those drivers have something in the DT indicating that they need
> this, then you can add a uclass_get_device_by_phandle() in those
> drivers.

Yes, there is an "iommus" property that points at the IOMMU.  This is
a standard property and I think it would make sense to have the core
device probing code handle it in a way similar to how "power-domains"
is handled.  That way we don't have to add IOMMU hooks into each and
every driver.

> If not, then you can probe all the DART devices with uclass_probe_all().
> 
> Having said that, I see you are using UCLASS_MISC. I suspect this
> should have its own UCLASS_IOMMU.

So here is a proof-of-concept diff to add UCLASS_IOMMU.  This
deliberately does not yet introduce driver ops yet; coming up with
something sensible there needs a bit more thought.  For USB support
all I need is for the probe function to put the IOMMU in bypass mode,
so that is what's implemented here.

Does this look reasonable?  I'll split this up eventually into a
commit that adds UCLASS_IOMMU and a commit that adds the apple_dart
driver.



commit 606979a32d8e0d19ee45df97ad6855c24e27e5ad
Author: Mark Kettenis <kettenis@openbsd.org>
Date:   Sun Sep 26 22:35:30 2021 +0200

    iommu: Proof of concept

diff --git a/drivers/Kconfig b/drivers/Kconfig
index 417d6f88c2..b26ca8cf70 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -50,6 +50,8 @@ source "drivers/i2c/Kconfig"
 
 source "drivers/input/Kconfig"
 
+source "drivers/iommu/Kconfig"
+
 source "drivers/led/Kconfig"
 
 source "drivers/mailbox/Kconfig"
diff --git a/drivers/Makefile b/drivers/Makefile
index fd218c9056..166aeb9817 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -109,6 +109,7 @@ obj-y += mtd/
 obj-y += pwm/
 obj-y += reset/
 obj-y += input/
+obj-y += iommu/
 # SOC specific infrastructure drivers.
 obj-y += smem/
 obj-y += thermal/
diff --git a/drivers/core/device.c b/drivers/core/device.c
index 29668f6fb3..5f480ad443 100644
--- a/drivers/core/device.c
+++ b/drivers/core/device.c
@@ -28,6 +28,7 @@
 #include <dm/uclass.h>
 #include <dm/uclass-internal.h>
 #include <dm/util.h>
+#include <iommu.h>
 #include <linux/err.h>
 #include <linux/list.h>
 #include <power-domain.h>
@@ -543,6 +544,13 @@ int device_probe(struct udevice *dev)
 			goto fail;
 	}
 
+	if (CONFIG_IS_ENABLED(IOMMU) && dev->parent &&
+	    (device_get_uclass_id(dev) != UCLASS_IOMMU)) {
+		ret = dev_iommu_probe(dev);
+		if (ret)
+			goto fail;
+	}
+
 	ret = device_get_dma_constraints(dev);
 	if (ret)
 		goto fail;
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
new file mode 100644
index 0000000000..d0638140af
--- /dev/null
+++ b/drivers/iommu/Kconfig
@@ -0,0 +1,20 @@
+#
+# IOMMU devices
+#
+
+menu "IOMMU device drivers"
+
+config IOMMU
+	bool "Enable Driver Model for IOMMU drivers"
+	depends on DM
+	help
+	  Enable driver model for IOMMU devices.
+
+config APPLE_DART
+	bool "Apple DART support"
+	depends on IOMMU && ARCH_APPLE
+	default y
+	help
+	  Enable support for the DART on Apple SoCs.
+
+endmenu
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
new file mode 100644
index 0000000000..8a50e3aba8
--- /dev/null
+++ b/drivers/iommu/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+obj-$(CONFIG_IOMMU) += iommu-uclass.o
+
+obj-$(CONFIG_APPLE_DART) += apple_dart.o
diff --git a/drivers/iommu/apple_dart.c b/drivers/iommu/apple_dart.c
new file mode 100644
index 0000000000..ff8c5fa62c
--- /dev/null
+++ b/drivers/iommu/apple_dart.c
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org>
+ */
+
+#include <common.h>
+#include <cpu_func.h>
+#include <dm.h>
+#include <asm/io.h>
+
+#define DART_PARAMS2		0x0004
+#define  DART_PARAMS2_BYPASS_SUPPORT	BIT(0)
+#define DART_TLB_OP		0x0020
+#define  DART_TLB_OP_OPMASK	(0xfff << 20)
+#define  DART_TLB_OP_FLUSH	(0x001 << 20)
+#define  DART_TLB_OP_BUSY	BIT(2)
+#define DART_TLB_OP_SIDMASK	0x0034
+#define DART_ERROR_STATUS	0x0040
+#define DART_TCR(sid)		(0x0100 + 4 * (sid))
+#define  DART_TCR_TRANSLATE_ENABLE	BIT(7)
+#define  DART_TCR_BYPASS_DART		BIT(8)
+#define  DART_TCR_BYPASS_DAPF		BIT(12)
+#define DART_TTBR(sid, idx)	(0x0200 + 16 * (sid) + 4 * (idx))
+#define  DART_TTBR_VALID	BIT(31)
+#define  DART_TTBR_SHIFT	12
+
+static int apple_dart_probe(struct udevice *dev)
+{
+	void *base;
+	int sid, i;
+
+	base = dev_read_addr_ptr(dev);
+	if (!base)
+		return -EINVAL;
+
+	u32 params2 = readl(base + DART_PARAMS2);
+	if (params2 & DART_PARAMS2_BYPASS_SUPPORT) {
+		for (sid = 0; sid < 16; sid++) {
+			writel(DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF,
+			       base + DART_TCR(sid));
+			for (i = 0; i < 4; i++)
+				writel(0, base + DART_TTBR(sid, i));
+		}
+	}
+
+	return 0;
+}
+
+static const struct udevice_id apple_dart_ids[] = {
+	{ .compatible = "apple,t8103-dart" },
+	{ /* sentinel */ }
+};
+
+U_BOOT_DRIVER(apple_dart) = {
+	.name = "apple_dart",
+	.id = UCLASS_IOMMU,
+	.of_match = apple_dart_ids,
+	.probe = apple_dart_probe
+};
diff --git a/drivers/iommu/iommu-uclass.c b/drivers/iommu/iommu-uclass.c
new file mode 100644
index 0000000000..5c55df3066
--- /dev/null
+++ b/drivers/iommu/iommu-uclass.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org>
+ */
+
+#define LOG_CATEGORY UCLASS_IOMMU
+
+#include <common.h>
+#include <dm.h>
+
+#if (CONFIG_IS_ENABLED(OF_CONTROL) && !CONFIG_IS_ENABLED(OF_PLATDATA))
+int dev_iommu_probe(struct udevice *dev)
+{
+	struct ofnode_phandle_args args;
+	struct udevice *dev_iommu;
+	int i, count, ret = 0;
+
+	count = dev_count_phandle_with_args(dev, "iommus",
+					    "#iommu-cells", 0);
+	for (i = 0; i < count; i++) {
+		ret = dev_read_phandle_with_args(dev, "iommus",
+						 "#iommu-cells", 0, i, &args);
+		if (ret) {
+			debug("%s: dev_read_phandle_with_args failed: %d\n",
+			      __func__, ret);
+			return ret;
+		}
+
+		ret = uclass_get_device_by_ofnode(UCLASS_IOMMU, args.node,
+						  &dev_iommu);
+		if (ret) {
+			debug("%s: uclass_get_device_by_ofnode failed: %d\n",
+			      __func__, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+#endif
+
+UCLASS_DRIVER(iommu) = {
+	.id		= UCLASS_IOMMU,
+	.name		= "iommu",
+};
diff --git a/include/dm/uclass-id.h b/include/dm/uclass-id.h
index e7edd409f3..56aa981613 100644
--- a/include/dm/uclass-id.h
+++ b/include/dm/uclass-id.h
@@ -61,6 +61,7 @@ enum uclass_id {
 	UCLASS_I2C_MUX,		/* I2C multiplexer */
 	UCLASS_I2S,		/* I2S bus */
 	UCLASS_IDE,		/* IDE device */
+	UCLASS_IOMMU,		/* IOMMU */
 	UCLASS_IRQ,		/* Interrupt controller */
 	UCLASS_KEYBOARD,	/* Keyboard input device */
 	UCLASS_LED,		/* Light-emitting diode (LED) */