[PATCH v21 0/5] Provide basic driver to control Arm R5 co-processor found on Xilinx ZynqMP

[PATCH v21 0/5] Provide basic driver to control Arm R5 co-processor found on Xilinx ZynqMP

[Ben Levinsky]

Update Xilinx R5 Remoteproc Driver as follows:
- update documentation for zynqmp_r5_probe
- restructure so that cluster initialization code is all in one place
- add memory allocation check for cluster
- add error handling in case of second core fails at probe but first core succeeded.
- remove unneeded lines in zynqmp_r5_remoteproc_remove

Previous version https://patchwork.kernel.org/project/linux-remoteproc/list/?series=374399

Ben Levinsky (5):
  firmware: xilinx: Add ZynqMP firmware ioctl enums for RPU
    configuration.
  firmware: xilinx: Add shutdown/wakeup APIs
  firmware: xilinx: Add RPU configuration APIs
  dt-bindings: remoteproc: Add documentation for ZynqMP R5 rproc
    bindings
  remoteproc: Add initial zynqmp R5 remoteproc driver

 .../xilinx,zynqmp-r5-remoteproc.yaml          | 143 ++++
 drivers/firmware/xilinx/zynqmp.c              |  96 +++
 drivers/remoteproc/Kconfig                    |   8 +
 drivers/remoteproc/Makefile                   |   1 +
 drivers/remoteproc/zynqmp_r5_remoteproc.c     | 785 ++++++++++++++++++
 include/linux/firmware/xlnx-zynqmp.h          |  60 ++
 6 files changed, 1093 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml
 create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c

-- 
2.17.1

[PATCH v21 1/5] firmware: xilinx: Add ZynqMP firmware ioctl enums for RPU configuration.

[Ben Levinsky]

Add ZynqMP firmware ioctl enums for RPU configuration.

Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
---
 include/linux/firmware/xlnx-zynqmp.h | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index 5968df82b991..bb347dfe4ba4 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -104,6 +104,10 @@ enum pm_ret_status {
 };
 
 enum pm_ioctl_id {
+	IOCTL_GET_RPU_OPER_MODE = 0,
+	IOCTL_SET_RPU_OPER_MODE = 1,
+	IOCTL_RPU_BOOT_ADDR_CONFIG = 2,
+	IOCTL_TCM_COMB_CONFIG = 3,
 	IOCTL_SD_DLL_RESET = 6,
 	IOCTL_SET_SD_TAPDELAY,
 	IOCTL_SET_PLL_FRAC_MODE,
@@ -129,6 +133,21 @@ enum pm_query_id {
 	PM_QID_CLOCK_GET_MAX_DIVISOR,
 };
 
+enum rpu_oper_mode {
+	PM_RPU_MODE_LOCKSTEP = 0,
+	PM_RPU_MODE_SPLIT = 1,
+};
+
+enum rpu_boot_mem {
+	PM_RPU_BOOTMEM_LOVEC = 0,
+	PM_RPU_BOOTMEM_HIVEC = 1,
+};
+
+enum rpu_tcm_comb {
+	PM_RPU_TCM_SPLIT = 0,
+	PM_RPU_TCM_COMB = 1,
+};
+
 enum zynqmp_pm_reset_action {
 	PM_RESET_ACTION_RELEASE,
 	PM_RESET_ACTION_ASSERT,
-- 
2.17.1

[PATCH v21 2/5] firmware: xilinx: Add shutdown/wakeup APIs

[Ben Levinsky]

Add shutdown/wakeup a resource eemi operations to shutdown
or bringup a resource.

Note alignment of args matches convention of other fn's in this file.
The reason being that the long fn name results in aligned args that
otherwise go over 80 chars so shift right to avoid this

Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
---
 drivers/firmware/xilinx/zynqmp.c     | 35 ++++++++++++++++++++++++++++
 include/linux/firmware/xlnx-zynqmp.h | 23 ++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index 8d1ff2454e2e..a966ee956573 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -846,6 +846,41 @@ int zynqmp_pm_release_node(const u32 node)
 }
 EXPORT_SYMBOL_GPL(zynqmp_pm_release_node);
 
+/**
+ * zynqmp_pm_force_pwrdwn - PM call to request for another PU or subsystem to
+ *             be powered down forcefully
+ * @node:  Node ID of the targeted PU or subsystem
+ * @ack:   Flag to specify whether acknowledge is requested
+ *
+ * Return: status, either success or error+reason
+ */
+int zynqmp_pm_force_pwrdwn(const u32 node,
+			   const enum zynqmp_pm_request_ack ack)
+{
+	return zynqmp_pm_invoke_fn(PM_FORCE_POWERDOWN, node, ack, 0, 0, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_force_pwrdwn);
+
+/**
+ * zynqmp_pm_request_wake - PM call to wake up selected master or subsystem
+ * @node:  Node ID of the master or subsystem
+ * @set_addr:  Specifies whether the address argument is relevant
+ * @address:   Address from which to resume when woken up
+ * @ack:   Flag to specify whether acknowledge requested
+ *
+ * Return: status, either success or error+reason
+ */
+int zynqmp_pm_request_wake(const u32 node,
+			   const bool set_addr,
+			   const u64 address,
+			   const enum zynqmp_pm_request_ack ack)
+{
+	/* set_addr flag is encoded into 1st bit of address */
+	return zynqmp_pm_invoke_fn(PM_REQUEST_WAKEUP, node, address | set_addr,
+				   address >> 32, ack, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_request_wake);
+
 /**
  * zynqmp_pm_set_requirement() - PM call to set requirement for PM slaves
  * @node:		Node ID of the slave
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index bb347dfe4ba4..6241c5ac51b3 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -12,6 +12,7 @@
 
 #ifndef __FIRMWARE_ZYNQMP_H__
 #define __FIRMWARE_ZYNQMP_H__
+#include <linux/types.h>
 
 #define ZYNQMP_PM_VERSION_MAJOR	1
 #define ZYNQMP_PM_VERSION_MINOR	0
@@ -64,6 +65,8 @@
 
 enum pm_api_id {
 	PM_GET_API_VERSION = 1,
+	PM_FORCE_POWERDOWN = 8,
+	PM_REQUEST_WAKEUP = 10,
 	PM_SYSTEM_SHUTDOWN = 12,
 	PM_REQUEST_NODE = 13,
 	PM_RELEASE_NODE,
@@ -376,6 +379,12 @@ int zynqmp_pm_write_pggs(u32 index, u32 value);
 int zynqmp_pm_read_pggs(u32 index, u32 *value);
 int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype);
 int zynqmp_pm_set_boot_health_status(u32 value);
+int zynqmp_pm_force_pwrdwn(const u32 target,
+			   const enum zynqmp_pm_request_ack ack);
+int zynqmp_pm_request_wake(const u32 node,
+			   const bool set_addr,
+			   const u64 address,
+			   const enum zynqmp_pm_request_ack ack);
 #else
 static inline struct zynqmp_eemi_ops *zynqmp_pm_get_eemi_ops(void)
 {
@@ -526,6 +535,20 @@ static inline int zynqmp_pm_set_boot_health_status(u32 value)
 {
 	return -ENODEV;
 }
+
+static inline int zynqmp_pm_force_pwrdwn(const u32 target,
+					 const enum zynqmp_pm_request_ack ack)
+{
+	return -ENODEV;
+}
+
+static inline int zynqmp_pm_request_wake(const u32 node,
+					 const bool set_addr,
+					 const u64 address,
+					 const enum zynqmp_pm_request_ack ack)
+{
+	return -ENODEV;
+}
 #endif
 
 #endif /* __FIRMWARE_ZYNQMP_H__ */
-- 
2.17.1

[PATCH v21 3/5] firmware: xilinx: Add RPU configuration APIs

[Ben Levinsky]

This patch adds APIs to access to configure RPU and its
processor-specific memory.

That is query the run-time mode of RPU as either split or lockstep as well
as API to set this mode. In addition add APIs to access configuration of
the RPUs' tightly coupled memory (TCM).

Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
---
 drivers/firmware/xilinx/zynqmp.c     | 61 ++++++++++++++++++++++++++++
 include/linux/firmware/xlnx-zynqmp.h | 18 ++++++++
 2 files changed, 79 insertions(+)

diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index a966ee956573..b390a00338d0 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -846,6 +846,67 @@ int zynqmp_pm_release_node(const u32 node)
 }
 EXPORT_SYMBOL_GPL(zynqmp_pm_release_node);
 
+/**
+ * zynqmp_pm_get_rpu_mode() - Get RPU mode
+ * @node_id:	Node ID of the device
+ * @rpu_mode:	return by reference value
+ *		either split or lockstep
+ *
+ * Return:	return 0 on success or error+reason.
+ *		if success, then  rpu_mode will be set
+ *		to current rpu mode.
+ */
+int zynqmp_pm_get_rpu_mode(u32 node_id, enum rpu_oper_mode *rpu_mode)
+{
+	u32 ret_payload[PAYLOAD_ARG_CNT];
+	int ret;
+
+	ret = zynqmp_pm_invoke_fn(PM_IOCTL, node_id,
+				  IOCTL_GET_RPU_OPER_MODE, 0, 0, ret_payload);
+
+	/* only set rpu_mode if no error */
+	if (ret == XST_PM_SUCCESS)
+		*rpu_mode = ret_payload[0];
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_rpu_mode);
+
+/**
+ * zynqmp_pm_set_rpu_mode() - Set RPU mode
+ * @node_id:	Node ID of the device
+ * @rpu_mode:	Argument 1 to requested IOCTL call. either split or lockstep
+ *
+ *		This function is used to set RPU mode to split or
+ *		lockstep
+ *
+ * Return:	Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node_id,
+				   IOCTL_SET_RPU_OPER_MODE, (u32)rpu_mode,
+				   0, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_rpu_mode);
+
+/**
+ * zynqmp_pm_set_tcm_config - configure TCM
+ * @tcm_mode:	Argument 1 to requested IOCTL call
+ *              either PM_RPU_TCM_COMB or PM_RPU_TCM_SPLIT
+ *
+ * This function is used to set RPU mode to split or combined
+ *
+ * Return: status: 0 for success, else failure
+ */
+int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node_id,
+				   IOCTL_TCM_COMB_CONFIG, (u32)tcm_mode, 0,
+				   NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_tcm_config);
+
 /**
  * zynqmp_pm_force_pwrdwn - PM call to request for another PU or subsystem to
  *             be powered down forcefully
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index 6241c5ac51b3..79aa2fcbcd54 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -385,6 +385,9 @@ int zynqmp_pm_request_wake(const u32 node,
 			   const bool set_addr,
 			   const u64 address,
 			   const enum zynqmp_pm_request_ack ack);
+int zynqmp_pm_get_rpu_mode(u32 node_id, enum rpu_oper_mode *rpu_mode);
+int zynqmp_pm_set_rpu_mode(u32 node_id, u32 arg1);
+int zynqmp_pm_set_tcm_config(u32 node_id, u32 arg1);
 #else
 static inline struct zynqmp_eemi_ops *zynqmp_pm_get_eemi_ops(void)
 {
@@ -549,6 +552,21 @@ static inline int zynqmp_pm_request_wake(const u32 node,
 {
 	return -ENODEV;
 }
+
+static inline int zynqmp_pm_get_rpu_mode(u32 node_id, enum rpu_oper_mode *rpu_mode)
+{
+	return -ENODEV;
+}
+
+static inline int zynqmp_pm_set_rpu_mode(u32 node_id, u32 arg1)
+{
+	return -ENODEV;
+}
+
+static inline int zynqmp_pm_set_tcm_config(u32 node_id, u32 arg1)
+{
+	return -ENODEV;
+}
 #endif
 
 #endif /* __FIRMWARE_ZYNQMP_H__ */
-- 
2.17.1

[PATCH v21 4/5] dt-bindings: remoteproc: Add documentation for ZynqMP R5 rproc bindings

[Ben Levinsky]

Add binding for ZynqMP R5 OpenAMP.

Represent the RPU domain resources in one device node. Each RPU
processor is a subnode of the top RPU domain node.

Signed-off-by: Jason Wu <j.wu@xilinx.com>
Signed-off-by: Wendy Liang <jliang@xilinx.com>
Signed-off-by: Michal Simek <michal.simek@xilinx.com>
Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
---
 .../xilinx,zynqmp-r5-remoteproc.yaml          | 143 ++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml

diff --git a/Documentation/devicetree/bindings/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml b/Documentation/devicetree/bindings/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml
new file mode 100644
index 000000000000..9cb358389cd7
--- /dev/null
+++ b/Documentation/devicetree/bindings/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml
@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: "http://devicetree.org/schemas/remoteproc/xilinx,zynqmp-r5-remoteproc.yaml#"
+$schema: "http://devicetree.org/meta-schemas/core.yaml#"
+
+title: Xilinx R5 remote processor controller bindings
+
+description:
+  This document defines the binding for the remoteproc component that loads and
+  boots firmwares on the Xilinx Zynqmp and Versal family chipset.
+
+  Note that the Linux has global addressing view of the R5-related memory (TCM)
+  so the absolute address ranges are provided in TCM reg's.
+
+maintainers:
+  - Ed Mooring <ed.mooring@xilinx.com>
+  - Ben Levinsky <ben.levinsky@xilinx.com>
+
+properties:
+  compatible:
+    const: xlnx,zynqmp-r5-remoteproc
+
+  lockstep-mode:
+    description:
+      If this property is present, then the configuration is lock-step.
+      Otherwise RPU is split.
+    type: boolean
+    maxItems: 1
+
+  interrupts:
+    description:
+      Interrupt mapping for remoteproc IPI. It is required if the
+      user uses the remoteproc driver with the RPMsg kernel driver.
+    maxItems: 6
+
+  memory-regions:
+    description:
+      Collection of memory carveouts used for elf-loading and inter-processor
+      communication. each carveout in this case should be in DDR, not
+      chip-specific memory. In Xilinx case, this is TCM, OCM, BRAM, etc.
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+
+  meta-memory-regions:
+    description:
+      Collection of memories that are not present in the top level memory
+      nodes' mapping. For example, R5s' TCM banks. These banks are needed
+      for R5 firmware meta data such as the R5 firmware's heap and stack.
+      To be more precise, this is on-chip reserved SRAM regions, e.g. TCM,
+      BRAM, OCM, etc.
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+
+  pnode-id:
+    maxItems: 1
+    description:
+      Power node id that is used to uniquely identify the node for Xilinx
+      Power Management. The value is then passed to Xilinx platform
+      manager for power on/off and access.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  mboxes:
+    description:
+      Array of phandles that describe the rx and tx for xilinx zynqmp
+      mailbox driver. order of rx and tx is described by the mbox-names
+      property. This will be used for communication with remote
+      processor.
+    maxItems: 2
+
+  mbox-names:
+    description:
+      Array of strings that denote which item in the mboxes property array
+      are the rx and tx for xilinx zynqmp mailbox driver
+    maxItems: 2
+    $ref: /schemas/types.yaml#/definitions/string-array
+
+
+examples:
+  - |
+     reserved-memory {
+          #address-cells = <1>;
+          #size-cells = <1>;
+          ranges;
+          elf_load: rproc@3ed000000 {
+               no-map;
+               reg = <0x3ed00000 0x40000>;
+          };
+
+          rpu0vdev0vring0: rpu0vdev0vring0@3ed40000 {
+               no-map;
+               reg = <0x3ed40000 0x4000>;
+          };
+          rpu0vdev0vring1: rpu0vdev0vring1@3ed44000 {
+               no-map;
+               reg = <0x3ed44000 0x4000>;
+          };
+          rpu0vdev0buffer: rpu0vdev0buffer@3ed48000 {
+               no-map;
+               reg = <0x3ed48000 0x100000>;
+          };
+
+     };
+
+     /*
+      * Below nodes are required if using TCM to load R5 firmware
+      * if not, then either do not provide nodes or label as disabled in
+      * status property
+      */
+     tcm0a: tcm_0a@ffe00000 {
+         reg = <0xffe00000 0x10000>;
+         pnode-id = <0xf>;
+         no-map;
+         status = "okay";
+         phandle = <0x40>;
+     };
+     tcm0b: tcm_1a@ffe20000 {
+         reg = <0xffe20000 0x10000>;
+         pnode-id = <0x10>;
+         no-map;
+         status = "okay";
+         phandle = <0x41>;
+     };
+
+     rpu {
+          compatible = "xlnx,zynqmp-r5-remoteproc";
+          #address-cells = <1>;
+          #size-cells = <1>;
+          ranges;
+          lockstep-mode;
+          r5_0 {
+               compatible = "xlnx,zynqmp-r5-single-remoteproc";
+               ranges;
+               #address-cells = <1>;
+               #size-cells = <1>;
+               memory-regions = <&elf_load>,
+                               <&rpu0vdev0vring0>,
+                               <&rpu0vdev0vring1>,
+                               <&rpu0vdev0buffer>;
+               meta-memory-regions = <&tcm_0a>, <&tcm_0b>;
+               pnode-id = <0x7>;
+          };
+     };
+
+...
-- 
2.17.1

[PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc driver

[Ben Levinsky]

R5 is included in Xilinx Zynq UltraScale MPSoC so by adding this
remotproc driver, we can boot the R5 sub-system in different 2
configurations -
	* split
	* lock-step

The Xilinx R5 Remoteproc Driver boots the R5's via calls to the Xilinx
Platform Management Unit that handles the R5 configuration, memory access
and R5 lifecycle management. The interface to this manager is done in this
driver via zynqmp_pm_* function calls.

Signed-off-by: Wendy Liang <wendy.liang@xilinx.com>
Signed-off-by: Michal Simek <michal.simek@xilinx.com>
Signed-off-by: Ed Mooring <ed.mooring@xilinx.com>
Signed-off-by: Jason Wu <j.wu@xilinx.com>
Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
---
Update Xilinx R5 Remoteproc Driver as follows:
- update documentation for zynqmp_r5_probe
- restructure so that cluster initialization code is all in one place
- add memory allocation check for cluster
- add error handling in case of second core fails at probe but first core succeeded.
  to clean up the first core
- remove unneeded lines in zynqmp_r5_remoteproc_remove
---

 drivers/remoteproc/Kconfig                |   8 +
 drivers/remoteproc/Makefile               |   1 +
 drivers/remoteproc/zynqmp_r5_remoteproc.c | 784 ++++++++++++++++++++++
 3 files changed, 793 insertions(+)
 create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c

diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index c6659dfea7c7..c2fe54b1d94f 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -275,6 +275,14 @@ config TI_K3_DSP_REMOTEPROC
 	  It's safe to say N here if you're not interested in utilizing
 	  the DSP slave processors.
 
+config ZYNQMP_R5_REMOTEPROC
+	tristate "ZynqMP R5 remoteproc support"
+	depends on PM && ARCH_ZYNQMP
+	select RPMSG_VIRTIO
+	select ZYNQMP_IPI_MBOX
+	help
+	  Say y or m here to support ZynqMP R5 remote processors via the remote
+	  processor framework.
 endif # REMOTEPROC
 
 endmenu
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index 3dfa28e6c701..ef1abff654c2 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
 obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
 obj-$(CONFIG_STM32_RPROC)		+= stm32_rproc.o
 obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
+obj-$(CONFIG_ZYNQMP_R5_REMOTEPROC)	+= zynqmp_r5_remoteproc.o
diff --git a/drivers/remoteproc/zynqmp_r5_remoteproc.c b/drivers/remoteproc/zynqmp_r5_remoteproc.c
new file mode 100644
index 000000000000..993bd72e5664
--- /dev/null
+++ b/drivers/remoteproc/zynqmp_r5_remoteproc.c
@@ -0,0 +1,784 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Zynq R5 Remote Processor driver
+ *
+ * Based on origin OMAP and Zynq Remote Processor driver
+ *
+ */
+
+#include <linux/firmware/xlnx-zynqmp.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mailbox_client.h>
+#include <linux/mailbox/zynqmp-ipi-message.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/remoteproc.h>
+#include <linux/skbuff.h>
+#include <linux/sysfs.h>
+
+#include "remoteproc_internal.h"
+
+#define MAX_RPROCS	2 /* Support up to 2 RPU */
+#define MAX_MEM_PNODES	4 /* Max power nodes for one RPU memory instance */
+
+#define BANK_LIST_PROP	"meta-memory-regions"
+#define DDR_LIST_PROP	"memory-regions"
+
+/* IPI buffer MAX length */
+#define IPI_BUF_LEN_MAX	32U
+/* RX mailbox client buffer max length */
+#define RX_MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
+				 sizeof(struct zynqmp_ipi_message))
+
+/**
+ * struct zynqmp_r5_mem - zynqmp rpu memory data
+ * @pnode_id: TCM power domain ids
+ * @res: memory resource
+ * @node: list node
+ */
+struct zynqmp_r5_mem {
+	u32 pnode_id[MAX_MEM_PNODES];
+	struct resource res;
+	struct list_head node;
+};
+
+/**
+ * struct zynqmp_r5_rproc - zynqmp rpu remote processor state
+ *			    this is for each individual R5 core's state
+ *
+ * @rx_mc_buf: rx mailbox client buffer to save the rx message
+ * @tx_mc: tx mailbox client
+ * @rx_mc: rx mailbox client * @dev: device of RPU instance
+ * @mbox_work: mbox_work for the RPU remoteproc
+ * @tx_mc_skbs: socket buffers for tx mailbox client
+ * @dev: device of RPU instance
+ * @rproc: rproc handle
+ * @tx_chan: tx mailbox channel
+ * @rx_chan: rx mailbox channel
+ * @pnode_id: RPU CPU power domain id
+ * @elem: linked list item
+ * @dt_node: device tree node that holds information for 1 R5 core.
+ */
+struct zynqmp_r5_rproc {
+	unsigned char rx_mc_buf[RX_MBOX_CLIENT_BUF_MAX];
+	struct mbox_client tx_mc;
+	struct mbox_client rx_mc;
+	struct work_struct mbox_work;
+	struct sk_buff_head tx_mc_skbs;
+	struct device *dev;
+	struct rproc *rproc;
+	struct mbox_chan *tx_chan;
+	struct mbox_chan *rx_chan;
+	u32 pnode_id;
+	struct list_head elem;
+};
+
+/*
+ * r5_set_mode - set RPU operation mode
+ * @z_rproc: Remote processor private data
+ * @rpu_mode: mode specified by device tree to configure the RPU to
+ *
+ * set RPU operation mode
+ *
+ * Return: 0 for success, negative value for failure
+ */
+static int r5_set_mode(struct zynqmp_r5_rproc *z_rproc,
+		       enum rpu_oper_mode rpu_mode)
+{
+	enum rpu_tcm_comb tcm_mode;
+	enum rpu_oper_mode cur_rpu_mode;
+	int ret;
+
+	ret = zynqmp_pm_get_rpu_mode(z_rproc->pnode_id, &cur_rpu_mode);
+	if (ret < 0)
+		return ret;
+
+	if (rpu_mode != cur_rpu_mode) {
+		ret = zynqmp_pm_set_rpu_mode(z_rproc->pnode_id,
+					     rpu_mode);
+		if (ret < 0)
+			return ret;
+	}
+
+	tcm_mode = (rpu_mode == PM_RPU_MODE_LOCKSTEP) ?
+		    PM_RPU_TCM_COMB : PM_RPU_TCM_SPLIT;
+	return zynqmp_pm_set_tcm_config(z_rproc->pnode_id, tcm_mode);
+}
+
+/*
+ * release TCM banks when powering down R5 core
+ */
+static int tcm_mem_release(struct rproc *rproc, struct rproc_mem_entry *mem)
+{
+	u32 pnode_id = (u64)mem->priv;
+
+	iounmap(mem->va);
+	return zynqmp_pm_release_node(pnode_id);
+}
+
+/*
+ * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
+ * try to use xpm wake call to wake R5 core
+ */
+static int zynqmp_r5_rproc_start(struct rproc *rproc)
+{
+	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
+	enum rpu_boot_mem bootmem;
+
+	bootmem = (rproc->bootaddr & 0xF0000000) == 0xF0000000 ?
+		  PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
+
+	dev_dbg(rproc->dev.parent, "RPU boot from %s.",
+		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
+
+	return zynqmp_pm_request_wake(z_rproc->pnode_id, 1,
+				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
+}
+
+/*
+ * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
+ * try to use xpm power down call to power off R5 core
+ */
+static int zynqmp_r5_rproc_stop(struct rproc *rproc)
+{
+	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
+
+	return zynqmp_pm_force_pwrdwn(z_rproc->pnode_id,
+				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
+}
+
+/*
+ * map in physical addr for  DDR mem carveout in rproc
+ */
+static int zynqmp_r5_rproc_mem_alloc(struct rproc *rproc,
+				     struct rproc_mem_entry *mem)
+{
+	void *va;
+
+	va = ioremap_wc(mem->dma, mem->len);
+	if (IS_ERR_OR_NULL(va))
+		return -ENOMEM;
+
+	/* Update memory entry va */
+	mem->va = va;
+
+	return 0;
+}
+
+/* unmap rproc_mem_entry virtual addr */
+static int zynqmp_r5_rproc_mem_release(struct rproc *rproc,
+				       struct rproc_mem_entry *mem)
+{
+	iounmap(mem->va);
+	return 0;
+}
+
+/* construct rproc mem carveouts for DDR regions specified in device tree */
+static int parse_mem_regions(struct rproc *rproc)
+{
+	int num_mems, i;
+	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
+	struct device *dev = &rproc->dev;
+	struct device_node *np = z_rproc->dev->of_node;
+	struct rproc_mem_entry *mem;
+
+	num_mems = of_count_phandle_with_args(np, DDR_LIST_PROP, NULL);
+	if (num_mems <= 0)
+		return 0;
+
+	for (i = 0; i < num_mems; i++) {
+		struct device_node *node;
+		struct reserved_mem *rmem;
+
+		node = of_parse_phandle(np, DDR_LIST_PROP, i);
+		if (!node)
+			return -EINVAL;
+
+		rmem = of_reserved_mem_lookup(node);
+		if (!rmem)
+			return -EINVAL;
+
+		if (strstr(node->name, "vdev0vring")) {
+			int vring_id;
+			char name[16];
+
+			/*
+			 * expecting form of "rpuXvdev0vringX as documented
+			 * in xilinx remoteproc device tree binding
+			 */
+			if (strlen(node->name) < 15) {
+				dev_err(dev, "%pOF is less than 14 chars",
+					node);
+				return -EINVAL;
+			}
+
+			/*
+			 * can be 1 of multiple vring IDs per IPC channel
+			 * e.g. 'vdev0vring0' and 'vdev0vring1'
+			 */
+			vring_id = node->name[14] - '0';
+			snprintf(name, sizeof(name), "vdev0vring%d", vring_id);
+			/* Register vring */
+			mem = rproc_mem_entry_init(dev, NULL,
+						   (dma_addr_t)rmem->base,
+						   rmem->size, rmem->base,
+						   zynqmp_r5_rproc_mem_alloc,
+						   zynqmp_r5_rproc_mem_release,
+						   name);
+		} else {
+			/* Register DMA region */
+			int (*alloc)(struct rproc *r,
+				     struct rproc_mem_entry *rme);
+			int (*release)(struct rproc *r,
+				       struct rproc_mem_entry *rme);
+			char name[20];
+
+			if (strstr(node->name, "vdev0buffer")) {
+				alloc = NULL;
+				release = NULL;
+				strcpy(name, "vdev0buffer");
+			} else {
+				alloc = zynqmp_r5_rproc_mem_alloc;
+				release = zynqmp_r5_rproc_mem_release;
+				strcpy(name, node->name);
+			}
+
+			mem = rproc_mem_entry_init(dev, NULL,
+						   (dma_addr_t)rmem->base,
+						   rmem->size, rmem->base,
+						   alloc, release, name);
+		}
+		if (!mem)
+			return -ENOMEM;
+
+		rproc_add_carveout(rproc, mem);
+	}
+
+	return 0;
+}
+
+/* call Xilinx Platform manager to request access to TCM bank */
+static int zynqmp_r5_pm_request_tcm(struct device_node *tcm_node,
+				    struct device *dev, u32 *pnode_id)
+{
+	int ret;
+
+	ret = of_property_read_u32(tcm_node, "pnode-id", pnode_id);
+	if (ret)
+		return ret;
+
+	return zynqmp_pm_request_node(*pnode_id, ZYNQMP_PM_CAPABILITY_ACCESS, 0,
+				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
+}
+
+/*
+ * Given TCM bank entry,
+ * this callback will set device address for R5 running on TCM
+ * and also setup virtual address for TCM bank remoteproc carveout
+ */
+static int tcm_mem_alloc(struct rproc *rproc,
+			 struct rproc_mem_entry *mem)
+{
+	void *va;
+	struct device *dev = rproc->dev.parent;
+
+	va = ioremap_wc(mem->dma, mem->len);
+	if (IS_ERR_OR_NULL(va))
+		return -ENOMEM;
+
+	/* Update memory entry va */
+	mem->va = va;
+
+	va = devm_ioremap_wc(dev, mem->da, mem->len);
+	if (!va)
+		return -ENOMEM;
+	/* As R5 is 32 bit, wipe out extra high bits */
+	mem->da &= 0x000fffff;
+	/*
+	 * TCM Banks 0A and 0B (0xffe00000 and 0xffe20000)
+	 * are handled with the above line of code so do nothing
+	 * for this 2 banks
+	 */
+
+	/*
+	 * TCM Banks 1A and 1B (0xffe90000 and 0xffeb0000) still
+	 * need to be translated to 0x0 and 0x20000
+	 */
+	if (mem->da == 0x90000 || mem->da == 0xB0000)
+		mem->da -= 0x90000;
+
+	/* if translated TCM bank address is not valid report error */
+	if (mem->da != 0x0 && mem->da != 0x20000) {
+		dev_err(dev, "invalid TCM bank address: %x\n", mem->da);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * Given R5 node in remoteproc instance
+ * allocate remoteproc carveout for TCM memory
+ * needed for firmware to be loaded
+ */
+static int parse_tcm_banks(struct rproc *rproc)
+{
+	int i, num_banks;
+	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
+	struct device *dev = &rproc->dev;
+	struct device_node *r5_node = z_rproc->dev->of_node;
+
+	/* go through TCM banks for r5 node */
+	num_banks = of_count_phandle_with_args(r5_node, BANK_LIST_PROP, NULL);
+	if (num_banks <= 0) {
+		dev_err(dev, "need to specify TCM banks\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < num_banks; i++) {
+		struct resource rsc;
+		resource_size_t size;
+		struct device_node *dt_node;
+		struct rproc_mem_entry *mem;
+		int ret;
+		u32 pnode_id; /* zynqmp_pm* fn's expect u32 */
+
+		dt_node = of_parse_phandle(r5_node, BANK_LIST_PROP, i);
+		if (!dt_node)
+			return -EINVAL;
+
+		if (of_device_is_available(dt_node)) {
+			ret = of_address_to_resource(dt_node, 0, &rsc);
+			if (ret < 0)
+				return ret;
+
+			ret = zynqmp_r5_pm_request_tcm(dt_node, dev, &pnode_id);
+			if (ret < 0)
+				return ret;
+
+			/* add carveout */
+			size = resource_size(&rsc);
+			mem = rproc_mem_entry_init(dev, NULL, rsc.start,
+						   (int)size, rsc.start,
+						   tcm_mem_alloc,
+						   tcm_mem_release,
+						   rsc.name);
+			if (!mem)
+				return -ENOMEM;
+
+			mem->priv = (void *)(u64)pnode_id;
+			rproc_add_carveout(rproc, mem);
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * when loading firmware, load in needed DDR, TCM memory regions and wire
+ * these into remoteproc core's carveouts
+ */
+static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware *fw)
+{
+	int ret;
+
+	ret = parse_tcm_banks(rproc);
+	if (ret)
+		return ret;
+
+	ret = parse_mem_regions(rproc);
+	if (ret)
+		return ret;
+
+	ret = rproc_elf_load_rsc_table(rproc, fw);
+	if (ret == -EINVAL) {
+		/*
+		 * resource table only required for IPC.
+		 * if not present, this is not necessarily an error;
+		 * for example, loading r5 hello world application
+		 * so simply inform user and keep going.
+		 */
+		dev_info(&rproc->dev, "no resource table found.\n");
+		ret = 0;
+	}
+	return ret;
+}
+
+/* kick a firmware */
+static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
+{
+	struct sk_buff *skb;
+	unsigned int skb_len;
+	struct zynqmp_ipi_message *mb_msg;
+	int ret;
+
+	struct device *dev = rproc->dev.parent;
+	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
+
+	skb_len = (unsigned int)(sizeof(vqid) + sizeof(mb_msg));
+	skb = alloc_skb(skb_len, GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	mb_msg = (struct zynqmp_ipi_message *)skb_put(skb, skb_len);
+	mb_msg->len = sizeof(vqid);
+	memcpy(mb_msg->data, &vqid, sizeof(vqid));
+	skb_queue_tail(&z_rproc->tx_mc_skbs, skb);
+	ret = mbox_send_message(z_rproc->tx_chan, mb_msg);
+	if (ret < 0) {
+		dev_warn(dev, "Failed to kick remote.\n");
+		skb_dequeue_tail(&z_rproc->tx_mc_skbs);
+		kfree_skb(skb);
+	}
+}
+
+static struct rproc_ops zynqmp_r5_rproc_ops = {
+	.start		= zynqmp_r5_rproc_start,
+	.stop		= zynqmp_r5_rproc_stop,
+	.load		= rproc_elf_load_segments,
+	.parse_fw	= zynqmp_r5_parse_fw,
+	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
+	.sanity_check	= rproc_elf_sanity_check,
+	.get_boot_addr	= rproc_elf_get_boot_addr,
+	.kick		= zynqmp_r5_rproc_kick,
+};
+
+/**
+ * event_notified_idr_cb() - event notified idr callback
+ * @id: idr id
+ * @ptr: pointer to idr private data
+ * @data: data passed to idr_for_each callback
+ *
+ * Pass notification to remoteproc virtio
+ *
+ * Return: 0. having return is to satisfy the idr_for_each() function
+ *          pointer input argument requirement.
+ **/
+static int event_notified_idr_cb(int id, void *ptr, void *data)
+{
+	struct rproc *rproc = data;
+
+	(void)rproc_vq_interrupt(rproc, id);
+	return 0;
+}
+
+/**
+ * handle_event_notified() - remoteproc notification work funciton
+ * @work: pointer to the work structure
+ *
+ * It checks each registered remoteproc notify IDs.
+ */
+static void handle_event_notified(struct work_struct *work)
+{
+	struct rproc *rproc;
+	struct zynqmp_r5_rproc *z_rproc;
+
+	z_rproc = container_of(work, struct zynqmp_r5_rproc, mbox_work);
+
+	(void)mbox_send_message(z_rproc->rx_chan, NULL);
+	rproc = z_rproc->rproc;
+	/*
+	 * We only use IPI for interrupt. The firmware side may or may
+	 * not write the notifyid when it trigger IPI.
+	 * And thus, we scan through all the registered notifyids.
+	 */
+	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
+}
+
+/**
+ * zynqmp_r5_mb_rx_cb() - Receive channel mailbox callback
+ * @cl: mailbox client
+ * @mssg: message pointer
+ *
+ * It will schedule the R5 notification work.
+ */
+static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *mssg)
+{
+	struct zynqmp_r5_rproc *z_rproc;
+
+	z_rproc = container_of(cl, struct zynqmp_r5_rproc, rx_mc);
+	if (mssg) {
+		struct zynqmp_ipi_message *ipi_msg, *buf_msg;
+		size_t len;
+
+		ipi_msg = (struct zynqmp_ipi_message *)mssg;
+		buf_msg = (struct zynqmp_ipi_message *)z_rproc->rx_mc_buf;
+		len = (ipi_msg->len >= IPI_BUF_LEN_MAX) ?
+		      IPI_BUF_LEN_MAX : ipi_msg->len;
+		buf_msg->len = len;
+		memcpy(buf_msg->data, ipi_msg->data, len);
+	}
+	schedule_work(&z_rproc->mbox_work);
+}
+
+/**
+ * zynqmp_r5_mb_tx_done() - Request has been sent to the remote
+ * @cl: mailbox client
+ * @mssg: pointer to the message which has been sent
+ * @r: status of last TX - OK or error
+ *
+ * It will be called by the mailbox framework when the last TX has done.
+ */
+static void zynqmp_r5_mb_tx_done(struct mbox_client *cl, void *mssg, int r)
+{
+	struct zynqmp_r5_rproc *z_rproc;
+	struct sk_buff *skb;
+
+	if (!mssg)
+		return;
+	z_rproc = container_of(cl, struct zynqmp_r5_rproc, tx_mc);
+	skb = skb_dequeue(&z_rproc->tx_mc_skbs);
+	kfree_skb(skb);
+}
+
+/**
+ * zynqmp_r5_setup_mbox() - Setup mailboxes
+ *			    this is used for each individual R5 core
+ *
+ * @z_rproc: pointer to the ZynqMP R5 processor platform data
+ * @node: pointer of the device node
+ *
+ * Function to setup mailboxes to talk to RPU.
+ *
+ * Return: 0 for success, negative value for failure.
+ */
+static int zynqmp_r5_setup_mbox(struct zynqmp_r5_rproc *z_rproc,
+				struct device_node *node)
+{
+	struct mbox_client *mclient;
+
+	/* Setup TX mailbox channel client */
+	mclient = &z_rproc->tx_mc;
+	mclient->rx_callback = NULL;
+	mclient->tx_block = false;
+	mclient->knows_txdone = false;
+	mclient->tx_done = zynqmp_r5_mb_tx_done;
+	mclient->dev = z_rproc->dev;
+
+	/* Setup TX mailbox channel client */
+	mclient = &z_rproc->rx_mc;
+	mclient->dev = z_rproc->dev;
+	mclient->rx_callback = zynqmp_r5_mb_rx_cb;
+	mclient->tx_block = false;
+	mclient->knows_txdone = false;
+
+	INIT_WORK(&z_rproc->mbox_work, handle_event_notified);
+
+	/* Request TX and RX channels */
+	z_rproc->tx_chan = mbox_request_channel_byname(&z_rproc->tx_mc, "tx");
+	if (IS_ERR(z_rproc->tx_chan)) {
+		dev_err(z_rproc->dev, "failed to request mbox tx channel.\n");
+		z_rproc->tx_chan = NULL;
+		return -EINVAL;
+	}
+
+	z_rproc->rx_chan = mbox_request_channel_byname(&z_rproc->rx_mc, "rx");
+	if (IS_ERR(z_rproc->rx_chan)) {
+		dev_err(z_rproc->dev, "failed to request mbox rx channel.\n");
+		z_rproc->rx_chan = NULL;
+		return -EINVAL;
+	}
+	skb_queue_head_init(&z_rproc->tx_mc_skbs);
+
+	return 0;
+}
+
+/**
+ * zynqmp_r5_probe() - Probes ZynqMP R5 processor device node
+ *		       this is called for each individual R5 core to
+ *		       set up mailbox, Xilinx platform manager unique ID,
+ *		       add to rproc core
+ *
+ * @z_rproc: pointer to the ZynqMP R5 processor platform data
+ * @pdev: domain platform device for current R5 core
+ * @node: pointer of the device node for current R5 core
+ * @rpu_mode: mode to configure RPU, split or lockstep
+ * @core: Xilinx specific remoteproc structure used later to link
+ *           in to cluster of cores
+ *
+ * Function to retrieve the information of the ZynqMP R5 device node.
+ *
+ * Return: 0 for success, negative value for failure.
+ */
+static int zynqmp_r5_probe(struct platform_device *pdev,
+			   struct device_node *node,
+			   enum rpu_oper_mode rpu_mode,
+			   struct zynqmp_r5_rproc **core)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+	struct rproc *rproc_ptr;
+	struct zynqmp_r5_rproc *z_rproc;
+
+	/* Allocate remoteproc instance */
+	/* dev here is parent device of the allocated rproc's dev field */
+	rproc_ptr = rproc_alloc(dev, dev_name(dev), &zynqmp_r5_rproc_ops,
+				NULL, sizeof(struct zynqmp_r5_rproc));
+	if (!rproc_ptr)
+		return -ENOMEM;
+	z_rproc = rproc_ptr->priv;
+	z_rproc->rproc = rproc_ptr;
+	z_rproc->dev = dev;
+	/* Set up DMA mask */
+	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
+	if (ret)
+		goto error;
+	/* Get R5 power domain node */
+	ret = of_property_read_u32(node, "pnode-id", &z_rproc->pnode_id);
+	if (ret)
+		goto error;
+
+	ret = r5_set_mode(z_rproc, rpu_mode);
+	if (ret)
+		return ret;
+
+	if (of_property_read_bool(node, "mboxes")) {
+		ret = zynqmp_r5_setup_mbox(z_rproc, node);
+		if (ret)
+			goto error;
+	}
+	/* Add R5 remoteproc */
+	ret = rproc_add(rproc_ptr);
+	if (ret)
+		goto error;
+	*core = z_rproc;
+
+	return 0;
+error:
+	if (z_rproc->rproc)
+		rproc_free(z_rproc->rproc);
+	z_rproc->rproc = NULL;
+	return ret;
+}
+
+/*
+ * called when driver is probed, for each R5 core specified in DT,
+ * setup as needed to do remoteproc-related operations
+ */
+static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev)
+{
+	int ret, i;
+	struct device *dev = &pdev->dev;
+	struct device_node *nc;
+	enum rpu_oper_mode rpu_mode;
+	struct list_head *cluster; /* list to track each core's rproc */
+	struct zynqmp_r5_rproc *z_rproc;
+	struct platform_device *child_pdev;
+
+	rpu_mode = of_property_read_bool(dev->of_node, "lockstep-mode") ?
+		   PM_RPU_MODE_LOCKSTEP : PM_RPU_MODE_SPLIT;
+	dev_dbg(dev, "RPU configuration: %s\n",
+		rpu_mode == PM_RPU_MODE_LOCKSTEP ? "lockstep" : "split");
+
+	/*
+	 * if 2 RPUs provided but one is lockstep, then we have an
+	 * invalid configuration.
+	 */
+	i = of_get_available_child_count(dev->of_node);
+	if ((rpu_mode == PM_RPU_MODE_LOCKSTEP && i != 1) || i > MAX_RPROCS)
+		return -EINVAL;
+
+	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
+	if (!cluster)
+		return -ENOMEM;
+	INIT_LIST_HEAD(cluster);
+
+	ret = devm_of_platform_populate(dev);
+	if (ret) {
+		dev_err(dev, "devm_of_platform_populate failed, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	/* probe each individual r5 core's remoteproc-related info */
+	i = 0;
+	for_each_available_child_of_node(dev->of_node, nc) {
+		child_pdev = of_find_device_by_node(nc);
+		if (!child_pdev) {
+			dev_err(dev, "could not get R5 core platform device\n");
+			ret = -ENODEV;
+			goto out;
+		}
+		ret = zynqmp_r5_probe(child_pdev, nc, rpu_mode, &z_rproc);
+		dev_dbg(dev, "%s to probe rpu %pOF\n",
+			ret ? "Failed" : "Able",
+			nc);
+		if (!z_rproc)
+			ret = -EINVAL;
+		if (ret)
+			goto out;
+
+		list_add_tail(&z_rproc->elem, cluster);
+		i++;
+	}
+	/* wire in so each core can be cleaned up at drive remove */
+	platform_set_drvdata(pdev, cluster);
+	ret = 0;
+out:
+	/* undo core0 upon any failures on core1 in split-mode */
+	if (rpu_mode == PM_RPU_MODE_SPLIT && i == 1 && ret != 0) {
+		z_rproc = container_of(cluster, struct zynqmp_r5_rproc, elem);
+		if (z_rproc->rproc) {
+			rproc_del(z_rproc->rproc);
+			rproc_free(z_rproc->rproc);
+		}
+
+		if (z_rproc->tx_chan)
+			mbox_free_channel(z_rproc->tx_chan);
+		if (z_rproc->rx_chan)
+			mbox_free_channel(z_rproc->rx_chan);
+	}
+	return ret;
+}
+
+/*
+ * for each core, clean up the following:
+ *	single rproc entry
+ *	mailbox tx, rx
+ */
+static int zynqmp_r5_remoteproc_remove(struct platform_device *pdev)
+{
+	struct list_head *pos, *cluster = (struct list_head *)
+					  platform_get_drvdata(pdev);
+	struct zynqmp_r5_rproc *z_rproc = NULL;
+	struct rproc *rproc = NULL;
+
+	list_for_each(pos, cluster) {
+		z_rproc = list_entry(pos, struct zynqmp_r5_rproc, elem);
+		rproc = z_rproc->rproc;
+		if (rproc) {
+			rproc_del(rproc);
+			rproc_free(rproc);
+		}
+
+		if (z_rproc->tx_chan)
+			mbox_free_channel(z_rproc->tx_chan);
+		if (z_rproc->rx_chan)
+			mbox_free_channel(z_rproc->rx_chan);
+	}
+	return 0;
+}
+
+/* Match table for OF platform binding */
+static const struct of_device_id zynqmp_r5_remoteproc_match[] = {
+	{ .compatible = "xlnx,zynqmp-r5-remoteproc", },
+	{ /* end of list */ },
+};
+MODULE_DEVICE_TABLE(of, zynqmp_r5_remoteproc_match);
+
+static struct platform_driver zynqmp_r5_remoteproc_driver = {
+	.probe = zynqmp_r5_remoteproc_probe,
+	.remove = zynqmp_r5_remoteproc_remove,
+	.driver = {
+		.name = "zynqmp_r5_remoteproc",
+		.of_match_table = zynqmp_r5_remoteproc_match,
+	},
+};
+module_platform_driver(zynqmp_r5_remoteproc_driver);
+
+MODULE_AUTHOR("Ben Levinsky <ben.levinsky@xilinx.com>");
+MODULE_LICENSE("GPL v2");
-- 
2.17.1

Re: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc driver

[Michael Auchter]

On Mon, Nov 02, 2020 at 11:38:59AM -0800, Ben Levinsky wrote:
> R5 is included in Xilinx Zynq UltraScale MPSoC so by adding this
> remotproc driver, we can boot the R5 sub-system in different 2
> configurations -
> 	* split
> 	* lock-step
> 
> The Xilinx R5 Remoteproc Driver boots the R5's via calls to the Xilinx
> Platform Management Unit that handles the R5 configuration, memory access
> and R5 lifecycle management. The interface to this manager is done in this
> driver via zynqmp_pm_* function calls.
> 
> Signed-off-by: Wendy Liang <wendy.liang@xilinx.com>
> Signed-off-by: Michal Simek <michal.simek@xilinx.com>
> Signed-off-by: Ed Mooring <ed.mooring@xilinx.com>
> Signed-off-by: Jason Wu <j.wu@xilinx.com>
> Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
> ---
> Update Xilinx R5 Remoteproc Driver as follows:
> - update documentation for zynqmp_r5_probe
> - restructure so that cluster initialization code is all in one place
> - add memory allocation check for cluster
> - add error handling in case of second core fails at probe but first core succeeded.
>   to clean up the first core
> - remove unneeded lines in zynqmp_r5_remoteproc_remove
> ---
> 
>  drivers/remoteproc/Kconfig                |   8 +
>  drivers/remoteproc/Makefile               |   1 +
>  drivers/remoteproc/zynqmp_r5_remoteproc.c | 784 ++++++++++++++++++++++
>  3 files changed, 793 insertions(+)
>  create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c
> 
> diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
> index c6659dfea7c7..c2fe54b1d94f 100644
> --- a/drivers/remoteproc/Kconfig
> +++ b/drivers/remoteproc/Kconfig
> @@ -275,6 +275,14 @@ config TI_K3_DSP_REMOTEPROC
>  	  It's safe to say N here if you're not interested in utilizing
>  	  the DSP slave processors.
>  
> +config ZYNQMP_R5_REMOTEPROC
> +	tristate "ZynqMP R5 remoteproc support"
> +	depends on PM && ARCH_ZYNQMP
> +	select RPMSG_VIRTIO
> +	select ZYNQMP_IPI_MBOX
> +	help
> +	  Say y or m here to support ZynqMP R5 remote processors via the remote
> +	  processor framework.
>  endif # REMOTEPROC
>  
>  endmenu
> diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
> index 3dfa28e6c701..ef1abff654c2 100644
> --- a/drivers/remoteproc/Makefile
> +++ b/drivers/remoteproc/Makefile
> @@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
>  obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
>  obj-$(CONFIG_STM32_RPROC)		+= stm32_rproc.o
>  obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
> +obj-$(CONFIG_ZYNQMP_R5_REMOTEPROC)	+= zynqmp_r5_remoteproc.o
> diff --git a/drivers/remoteproc/zynqmp_r5_remoteproc.c b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> new file mode 100644
> index 000000000000..993bd72e5664
> --- /dev/null
> +++ b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> @@ -0,0 +1,784 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Zynq R5 Remote Processor driver
> + *
> + * Based on origin OMAP and Zynq Remote Processor driver
> + *
> + */
> +
> +#include <linux/firmware/xlnx-zynqmp.h>
> +#include <linux/interrupt.h>
> +#include <linux/kernel.h>
> +#include <linux/list.h>
> +#include <linux/mailbox_client.h>
> +#include <linux/mailbox/zynqmp-ipi-message.h>
> +#include <linux/module.h>
> +#include <linux/of_address.h>
> +#include <linux/of_platform.h>
> +#include <linux/of_reserved_mem.h>
> +#include <linux/platform_device.h>
> +#include <linux/remoteproc.h>
> +#include <linux/skbuff.h>
> +#include <linux/sysfs.h>
> +
> +#include "remoteproc_internal.h"
> +
> +#define MAX_RPROCS	2 /* Support up to 2 RPU */
> +#define MAX_MEM_PNODES	4 /* Max power nodes for one RPU memory instance */
> +
> +#define BANK_LIST_PROP	"meta-memory-regions"
> +#define DDR_LIST_PROP	"memory-regions"
> +
> +/* IPI buffer MAX length */
> +#define IPI_BUF_LEN_MAX	32U
> +/* RX mailbox client buffer max length */
> +#define RX_MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
> +				 sizeof(struct zynqmp_ipi_message))
> +
> +/**
> + * struct zynqmp_r5_mem - zynqmp rpu memory data
> + * @pnode_id: TCM power domain ids
> + * @res: memory resource
> + * @node: list node
> + */
> +struct zynqmp_r5_mem {
> +	u32 pnode_id[MAX_MEM_PNODES];
> +	struct resource res;
> +	struct list_head node;
> +};
> +
> +/**
> + * struct zynqmp_r5_rproc - zynqmp rpu remote processor state
> + *			    this is for each individual R5 core's state
> + *
> + * @rx_mc_buf: rx mailbox client buffer to save the rx message
> + * @tx_mc: tx mailbox client
> + * @rx_mc: rx mailbox client * @dev: device of RPU instance
> + * @mbox_work: mbox_work for the RPU remoteproc
> + * @tx_mc_skbs: socket buffers for tx mailbox client
> + * @dev: device of RPU instance
> + * @rproc: rproc handle
> + * @tx_chan: tx mailbox channel
> + * @rx_chan: rx mailbox channel
> + * @pnode_id: RPU CPU power domain id
> + * @elem: linked list item
> + * @dt_node: device tree node that holds information for 1 R5 core.
> + */
> +struct zynqmp_r5_rproc {
> +	unsigned char rx_mc_buf[RX_MBOX_CLIENT_BUF_MAX];
> +	struct mbox_client tx_mc;
> +	struct mbox_client rx_mc;
> +	struct work_struct mbox_work;
> +	struct sk_buff_head tx_mc_skbs;
> +	struct device *dev;
> +	struct rproc *rproc;
> +	struct mbox_chan *tx_chan;
> +	struct mbox_chan *rx_chan;
> +	u32 pnode_id;
> +	struct list_head elem;
> +};
> +
> +/*
> + * r5_set_mode - set RPU operation mode
> + * @z_rproc: Remote processor private data
> + * @rpu_mode: mode specified by device tree to configure the RPU to
> + *
> + * set RPU operation mode
> + *
> + * Return: 0 for success, negative value for failure
> + */
> +static int r5_set_mode(struct zynqmp_r5_rproc *z_rproc,
> +		       enum rpu_oper_mode rpu_mode)
> +{
> +	enum rpu_tcm_comb tcm_mode;
> +	enum rpu_oper_mode cur_rpu_mode;
> +	int ret;
> +
> +	ret = zynqmp_pm_get_rpu_mode(z_rproc->pnode_id, &cur_rpu_mode);
> +	if (ret < 0)
> +		return ret;
> +
> +	if (rpu_mode != cur_rpu_mode) {
> +		ret = zynqmp_pm_set_rpu_mode(z_rproc->pnode_id,
> +					     rpu_mode);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	tcm_mode = (rpu_mode == PM_RPU_MODE_LOCKSTEP) ?
> +		    PM_RPU_TCM_COMB : PM_RPU_TCM_SPLIT;
> +	return zynqmp_pm_set_tcm_config(z_rproc->pnode_id, tcm_mode);
> +}
> +
> +/*
> + * release TCM banks when powering down R5 core
> + */
> +static int tcm_mem_release(struct rproc *rproc, struct rproc_mem_entry *mem)
> +{
> +	u32 pnode_id = (u64)mem->priv;
> +
> +	iounmap(mem->va);
> +	return zynqmp_pm_release_node(pnode_id);
> +}
> +
> +/*
> + * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
> + * try to use xpm wake call to wake R5 core
> + */
> +static int zynqmp_r5_rproc_start(struct rproc *rproc)
> +{
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	enum rpu_boot_mem bootmem;
> +
> +	bootmem = (rproc->bootaddr & 0xF0000000) == 0xF0000000 ?
> +		  PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
> +
> +	dev_dbg(rproc->dev.parent, "RPU boot from %s.",
> +		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
> +
> +	return zynqmp_pm_request_wake(z_rproc->pnode_id, 1,
> +				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
> +}
> +
> +/*
> + * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
> + * try to use xpm power down call to power off R5 core
> + */
> +static int zynqmp_r5_rproc_stop(struct rproc *rproc)
> +{
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +
> +	return zynqmp_pm_force_pwrdwn(z_rproc->pnode_id,
> +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> +}
> +
> +/*
> + * map in physical addr for  DDR mem carveout in rproc
> + */
> +static int zynqmp_r5_rproc_mem_alloc(struct rproc *rproc,
> +				     struct rproc_mem_entry *mem)
> +{
> +	void *va;
> +
> +	va = ioremap_wc(mem->dma, mem->len);
> +	if (IS_ERR_OR_NULL(va))
> +		return -ENOMEM;
> +
> +	/* Update memory entry va */
> +	mem->va = va;
> +
> +	return 0;
> +}
> +
> +/* unmap rproc_mem_entry virtual addr */
> +static int zynqmp_r5_rproc_mem_release(struct rproc *rproc,
> +				       struct rproc_mem_entry *mem)
> +{
> +	iounmap(mem->va);
> +	return 0;
> +}
> +
> +/* construct rproc mem carveouts for DDR regions specified in device tree */
> +static int parse_mem_regions(struct rproc *rproc)
> +{
> +	int num_mems, i;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	struct device *dev = &rproc->dev;
> +	struct device_node *np = z_rproc->dev->of_node;
> +	struct rproc_mem_entry *mem;
> +
> +	num_mems = of_count_phandle_with_args(np, DDR_LIST_PROP, NULL);
> +	if (num_mems <= 0)
> +		return 0;
> +
> +	for (i = 0; i < num_mems; i++) {
> +		struct device_node *node;
> +		struct reserved_mem *rmem;
> +
> +		node = of_parse_phandle(np, DDR_LIST_PROP, i);
> +		if (!node)
> +			return -EINVAL;
> +
> +		rmem = of_reserved_mem_lookup(node);
> +		if (!rmem)
> +			return -EINVAL;
> +
> +		if (strstr(node->name, "vdev0vring")) {
> +			int vring_id;
> +			char name[16];
> +
> +			/*
> +			 * expecting form of "rpuXvdev0vringX as documented
> +			 * in xilinx remoteproc device tree binding
> +			 */
> +			if (strlen(node->name) < 15) {
> +				dev_err(dev, "%pOF is less than 14 chars",
> +					node);
> +				return -EINVAL;
> +			}
> +
> +			/*
> +			 * can be 1 of multiple vring IDs per IPC channel
> +			 * e.g. 'vdev0vring0' and 'vdev0vring1'
> +			 */
> +			vring_id = node->name[14] - '0';
> +			snprintf(name, sizeof(name), "vdev0vring%d", vring_id);
> +			/* Register vring */
> +			mem = rproc_mem_entry_init(dev, NULL,
> +						   (dma_addr_t)rmem->base,
> +						   rmem->size, rmem->base,
> +						   zynqmp_r5_rproc_mem_alloc,
> +						   zynqmp_r5_rproc_mem_release,
> +						   name);
> +		} else {
> +			/* Register DMA region */
> +			int (*alloc)(struct rproc *r,
> +				     struct rproc_mem_entry *rme);
> +			int (*release)(struct rproc *r,
> +				       struct rproc_mem_entry *rme);
> +			char name[20];
> +
> +			if (strstr(node->name, "vdev0buffer")) {
> +				alloc = NULL;
> +				release = NULL;
> +				strcpy(name, "vdev0buffer");
> +			} else {
> +				alloc = zynqmp_r5_rproc_mem_alloc;
> +				release = zynqmp_r5_rproc_mem_release;
> +				strcpy(name, node->name);
> +			}
> +
> +			mem = rproc_mem_entry_init(dev, NULL,
> +						   (dma_addr_t)rmem->base,
> +						   rmem->size, rmem->base,
> +						   alloc, release, name);
> +		}
> +		if (!mem)
> +			return -ENOMEM;
> +
> +		rproc_add_carveout(rproc, mem);
> +	}
> +
> +	return 0;
> +}
> +
> +/* call Xilinx Platform manager to request access to TCM bank */
> +static int zynqmp_r5_pm_request_tcm(struct device_node *tcm_node,
> +				    struct device *dev, u32 *pnode_id)
> +{
> +	int ret;
> +
> +	ret = of_property_read_u32(tcm_node, "pnode-id", pnode_id);
> +	if (ret)
> +		return ret;
> +
> +	return zynqmp_pm_request_node(*pnode_id, ZYNQMP_PM_CAPABILITY_ACCESS, 0,
> +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> +}
> +
> +/*
> + * Given TCM bank entry,
> + * this callback will set device address for R5 running on TCM
> + * and also setup virtual address for TCM bank remoteproc carveout
> + */
> +static int tcm_mem_alloc(struct rproc *rproc,
> +			 struct rproc_mem_entry *mem)
> +{
> +	void *va;
> +	struct device *dev = rproc->dev.parent;
> +
> +	va = ioremap_wc(mem->dma, mem->len);
> +	if (IS_ERR_OR_NULL(va))
> +		return -ENOMEM;
> +
> +	/* Update memory entry va */
> +	mem->va = va;
> +
> +	va = devm_ioremap_wc(dev, mem->da, mem->len);
> +	if (!va)
> +		return -ENOMEM;
> +	/* As R5 is 32 bit, wipe out extra high bits */
> +	mem->da &= 0x000fffff;
> +	/*
> +	 * TCM Banks 0A and 0B (0xffe00000 and 0xffe20000)
> +	 * are handled with the above line of code so do nothing
> +	 * for this 2 banks
> +	 */
> +
> +	/*
> +	 * TCM Banks 1A and 1B (0xffe90000 and 0xffeb0000) still
> +	 * need to be translated to 0x0 and 0x20000
> +	 */
> +	if (mem->da == 0x90000 || mem->da == 0xB0000)
> +		mem->da -= 0x90000;
> +
> +	/* if translated TCM bank address is not valid report error */
> +	if (mem->da != 0x0 && mem->da != 0x20000) {
> +		dev_err(dev, "invalid TCM bank address: %x\n", mem->da);
> +		return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
> +/*
> + * Given R5 node in remoteproc instance
> + * allocate remoteproc carveout for TCM memory
> + * needed for firmware to be loaded
> + */
> +static int parse_tcm_banks(struct rproc *rproc)
> +{
> +	int i, num_banks;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	struct device *dev = &rproc->dev;
> +	struct device_node *r5_node = z_rproc->dev->of_node;
> +
> +	/* go through TCM banks for r5 node */
> +	num_banks = of_count_phandle_with_args(r5_node, BANK_LIST_PROP, NULL);
> +	if (num_banks <= 0) {
> +		dev_err(dev, "need to specify TCM banks\n");
> +		return -EINVAL;
> +	}
> +	for (i = 0; i < num_banks; i++) {
> +		struct resource rsc;
> +		resource_size_t size;
> +		struct device_node *dt_node;
> +		struct rproc_mem_entry *mem;
> +		int ret;
> +		u32 pnode_id; /* zynqmp_pm* fn's expect u32 */
> +
> +		dt_node = of_parse_phandle(r5_node, BANK_LIST_PROP, i);
> +		if (!dt_node)
> +			return -EINVAL;
> +
> +		if (of_device_is_available(dt_node)) {
> +			ret = of_address_to_resource(dt_node, 0, &rsc);
> +			if (ret < 0)
> +				return ret;
> +
> +			ret = zynqmp_r5_pm_request_tcm(dt_node, dev, &pnode_id);
> +			if (ret < 0)
> +				return ret;
> +
> +			/* add carveout */
> +			size = resource_size(&rsc);
> +			mem = rproc_mem_entry_init(dev, NULL, rsc.start,
> +						   (int)size, rsc.start,
> +						   tcm_mem_alloc,
> +						   tcm_mem_release,
> +						   rsc.name);
> +			if (!mem)
> +				return -ENOMEM;
> +
> +			mem->priv = (void *)(u64)pnode_id;
> +			rproc_add_carveout(rproc, mem);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +/*
> + * when loading firmware, load in needed DDR, TCM memory regions and wire
> + * these into remoteproc core's carveouts
> + */
> +static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware *fw)
> +{
> +	int ret;
> +
> +	ret = parse_tcm_banks(rproc);
> +	if (ret)
> +		return ret;
> +
> +	ret = parse_mem_regions(rproc);
> +	if (ret)
> +		return ret;
> +
> +	ret = rproc_elf_load_rsc_table(rproc, fw);
> +	if (ret == -EINVAL) {
> +		/*
> +		 * resource table only required for IPC.
> +		 * if not present, this is not necessarily an error;
> +		 * for example, loading r5 hello world application
> +		 * so simply inform user and keep going.
> +		 */
> +		dev_info(&rproc->dev, "no resource table found.\n");
> +		ret = 0;
> +	}
> +	return ret;
> +}
> +
> +/* kick a firmware */
> +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
> +{
> +	struct sk_buff *skb;
> +	unsigned int skb_len;
> +	struct zynqmp_ipi_message *mb_msg;
> +	int ret;
> +
> +	struct device *dev = rproc->dev.parent;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +
> +	skb_len = (unsigned int)(sizeof(vqid) + sizeof(mb_msg));
> +	skb = alloc_skb(skb_len, GFP_ATOMIC);
> +	if (!skb)
> +		return;
> +
> +	mb_msg = (struct zynqmp_ipi_message *)skb_put(skb, skb_len);
> +	mb_msg->len = sizeof(vqid);
> +	memcpy(mb_msg->data, &vqid, sizeof(vqid));
> +	skb_queue_tail(&z_rproc->tx_mc_skbs, skb);
> +	ret = mbox_send_message(z_rproc->tx_chan, mb_msg);
> +	if (ret < 0) {
> +		dev_warn(dev, "Failed to kick remote.\n");
> +		skb_dequeue_tail(&z_rproc->tx_mc_skbs);
> +		kfree_skb(skb);
> +	}
> +}
> +
> +static struct rproc_ops zynqmp_r5_rproc_ops = {
> +	.start		= zynqmp_r5_rproc_start,
> +	.stop		= zynqmp_r5_rproc_stop,
> +	.load		= rproc_elf_load_segments,
> +	.parse_fw	= zynqmp_r5_parse_fw,
> +	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
> +	.sanity_check	= rproc_elf_sanity_check,
> +	.get_boot_addr	= rproc_elf_get_boot_addr,
> +	.kick		= zynqmp_r5_rproc_kick,
> +};
> +
> +/**
> + * event_notified_idr_cb() - event notified idr callback
> + * @id: idr id
> + * @ptr: pointer to idr private data
> + * @data: data passed to idr_for_each callback
> + *
> + * Pass notification to remoteproc virtio
> + *
> + * Return: 0. having return is to satisfy the idr_for_each() function
> + *          pointer input argument requirement.
> + **/
> +static int event_notified_idr_cb(int id, void *ptr, void *data)
> +{
> +	struct rproc *rproc = data;
> +
> +	(void)rproc_vq_interrupt(rproc, id);
> +	return 0;
> +}
> +
> +/**
> + * handle_event_notified() - remoteproc notification work funciton
> + * @work: pointer to the work structure
> + *
> + * It checks each registered remoteproc notify IDs.
> + */
> +static void handle_event_notified(struct work_struct *work)
> +{
> +	struct rproc *rproc;
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	z_rproc = container_of(work, struct zynqmp_r5_rproc, mbox_work);
> +
> +	(void)mbox_send_message(z_rproc->rx_chan, NULL);
> +	rproc = z_rproc->rproc;
> +	/*
> +	 * We only use IPI for interrupt. The firmware side may or may
> +	 * not write the notifyid when it trigger IPI.
> +	 * And thus, we scan through all the registered notifyids.
> +	 */
> +	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
> +}
> +
> +/**
> + * zynqmp_r5_mb_rx_cb() - Receive channel mailbox callback
> + * @cl: mailbox client
> + * @mssg: message pointer
> + *
> + * It will schedule the R5 notification work.
> + */
> +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *mssg)
> +{
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, rx_mc);
> +	if (mssg) {
> +		struct zynqmp_ipi_message *ipi_msg, *buf_msg;
> +		size_t len;
> +
> +		ipi_msg = (struct zynqmp_ipi_message *)mssg;
> +		buf_msg = (struct zynqmp_ipi_message *)z_rproc->rx_mc_buf;
> +		len = (ipi_msg->len >= IPI_BUF_LEN_MAX) ?
> +		      IPI_BUF_LEN_MAX : ipi_msg->len;
> +		buf_msg->len = len;
> +		memcpy(buf_msg->data, ipi_msg->data, len);
> +	}
> +	schedule_work(&z_rproc->mbox_work);
> +}
> +
> +/**
> + * zynqmp_r5_mb_tx_done() - Request has been sent to the remote
> + * @cl: mailbox client
> + * @mssg: pointer to the message which has been sent
> + * @r: status of last TX - OK or error
> + *
> + * It will be called by the mailbox framework when the last TX has done.
> + */
> +static void zynqmp_r5_mb_tx_done(struct mbox_client *cl, void *mssg, int r)
> +{
> +	struct zynqmp_r5_rproc *z_rproc;
> +	struct sk_buff *skb;
> +
> +	if (!mssg)
> +		return;
> +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, tx_mc);
> +	skb = skb_dequeue(&z_rproc->tx_mc_skbs);
> +	kfree_skb(skb);
> +}
> +
> +/**
> + * zynqmp_r5_setup_mbox() - Setup mailboxes
> + *			    this is used for each individual R5 core
> + *
> + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> + * @node: pointer of the device node
> + *
> + * Function to setup mailboxes to talk to RPU.
> + *
> + * Return: 0 for success, negative value for failure.
> + */
> +static int zynqmp_r5_setup_mbox(struct zynqmp_r5_rproc *z_rproc,
> +				struct device_node *node)
> +{
> +	struct mbox_client *mclient;
> +
> +	/* Setup TX mailbox channel client */
> +	mclient = &z_rproc->tx_mc;
> +	mclient->rx_callback = NULL;
> +	mclient->tx_block = false;
> +	mclient->knows_txdone = false;
> +	mclient->tx_done = zynqmp_r5_mb_tx_done;
> +	mclient->dev = z_rproc->dev;
> +
> +	/* Setup TX mailbox channel client */
> +	mclient = &z_rproc->rx_mc;
> +	mclient->dev = z_rproc->dev;
> +	mclient->rx_callback = zynqmp_r5_mb_rx_cb;
> +	mclient->tx_block = false;
> +	mclient->knows_txdone = false;
> +
> +	INIT_WORK(&z_rproc->mbox_work, handle_event_notified);
> +
> +	/* Request TX and RX channels */
> +	z_rproc->tx_chan = mbox_request_channel_byname(&z_rproc->tx_mc, "tx");
> +	if (IS_ERR(z_rproc->tx_chan)) {
> +		dev_err(z_rproc->dev, "failed to request mbox tx channel.\n");
> +		z_rproc->tx_chan = NULL;
> +		return -EINVAL;
> +	}
> +
> +	z_rproc->rx_chan = mbox_request_channel_byname(&z_rproc->rx_mc, "rx");
> +	if (IS_ERR(z_rproc->rx_chan)) {
> +		dev_err(z_rproc->dev, "failed to request mbox rx channel.\n");
> +		z_rproc->rx_chan = NULL;
> +		return -EINVAL;
> +	}
> +	skb_queue_head_init(&z_rproc->tx_mc_skbs);
> +
> +	return 0;
> +}
> +
> +/**
> + * zynqmp_r5_probe() - Probes ZynqMP R5 processor device node
> + *		       this is called for each individual R5 core to
> + *		       set up mailbox, Xilinx platform manager unique ID,
> + *		       add to rproc core
> + *
> + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> + * @pdev: domain platform device for current R5 core
> + * @node: pointer of the device node for current R5 core
> + * @rpu_mode: mode to configure RPU, split or lockstep
> + * @core: Xilinx specific remoteproc structure used later to link
> + *           in to cluster of cores

This still does not match the param list (there's no z_rproc)

> + *
> + * Function to retrieve the information of the ZynqMP R5 device node.
> + *
> + * Return: 0 for success, negative value for failure.
> + */
> +static int zynqmp_r5_probe(struct platform_device *pdev,
> +			   struct device_node *node,
> +			   enum rpu_oper_mode rpu_mode,
> +			   struct zynqmp_r5_rproc **core)
> +{
> +	int ret;
> +	struct device *dev = &pdev->dev;
> +	struct rproc *rproc_ptr;
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	/* Allocate remoteproc instance */
> +	/* dev here is parent device of the allocated rproc's dev field */
> +	rproc_ptr = rproc_alloc(dev, dev_name(dev), &zynqmp_r5_rproc_ops,
> +				NULL, sizeof(struct zynqmp_r5_rproc));
> +	if (!rproc_ptr)
> +		return -ENOMEM;
> +	z_rproc = rproc_ptr->priv;
> +	z_rproc->rproc = rproc_ptr;
> +	z_rproc->dev = dev;
> +	/* Set up DMA mask */
> +	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
> +	if (ret)
> +		goto error;
> +	/* Get R5 power domain node */
> +	ret = of_property_read_u32(node, "pnode-id", &z_rproc->pnode_id);
> +	if (ret)
> +		goto error;
> +
> +	ret = r5_set_mode(z_rproc, rpu_mode);
> +	if (ret)
> +		return ret;

goto error?

> +
> +	if (of_property_read_bool(node, "mboxes")) {
> +		ret = zynqmp_r5_setup_mbox(z_rproc, node);
> +		if (ret)
> +			goto error;
> +	}
> +	/* Add R5 remoteproc */
> +	ret = rproc_add(rproc_ptr);
> +	if (ret)
> +		goto error;
> +	*core = z_rproc;
> +
> +	return 0;
> +error:
> +	if (z_rproc->rproc)

How would this ever be NULL?

> +		rproc_free(z_rproc->rproc);
> +	z_rproc->rproc = NULL;

This looks like another use-after-free, but it also seems unnecessary.

> +	return ret;
> +}
> +
> +/*
> + * called when driver is probed, for each R5 core specified in DT,
> + * setup as needed to do remoteproc-related operations
> + */
> +static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev)
> +{
> +	int ret, i;
> +	struct device *dev = &pdev->dev;
> +	struct device_node *nc;
> +	enum rpu_oper_mode rpu_mode;
> +	struct list_head *cluster; /* list to track each core's rproc */
> +	struct zynqmp_r5_rproc *z_rproc;
> +	struct platform_device *child_pdev;
> +
> +	rpu_mode = of_property_read_bool(dev->of_node, "lockstep-mode") ?
> +		   PM_RPU_MODE_LOCKSTEP : PM_RPU_MODE_SPLIT;
> +	dev_dbg(dev, "RPU configuration: %s\n",
> +		rpu_mode == PM_RPU_MODE_LOCKSTEP ? "lockstep" : "split");
> +
> +	/*
> +	 * if 2 RPUs provided but one is lockstep, then we have an
> +	 * invalid configuration.
> +	 */
> +	i = of_get_available_child_count(dev->of_node);
> +	if ((rpu_mode == PM_RPU_MODE_LOCKSTEP && i != 1) || i > MAX_RPROCS)
> +		return -EINVAL;
> +
> +	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
> +	if (!cluster)
> +		return -ENOMEM;
> +	INIT_LIST_HEAD(cluster);
> +
> +	ret = devm_of_platform_populate(dev);
> +	if (ret) {
> +		dev_err(dev, "devm_of_platform_populate failed, ret = %d\n",
> +			ret);
> +		return ret;
> +	}
> +
> +	/* probe each individual r5 core's remoteproc-related info */
> +	i = 0;
> +	for_each_available_child_of_node(dev->of_node, nc) {
> +		child_pdev = of_find_device_by_node(nc);
> +		if (!child_pdev) {
> +			dev_err(dev, "could not get R5 core platform device\n");
> +			ret = -ENODEV;
> +			goto out;
> +		}
> +		ret = zynqmp_r5_probe(child_pdev, nc, rpu_mode, &z_rproc);
> +		dev_dbg(dev, "%s to probe rpu %pOF\n",
> +			ret ? "Failed" : "Able",
> +			nc);
> +		if (!z_rproc)
> +			ret = -EINVAL;
> +		if (ret)
> +			goto out;
> +
> +		list_add_tail(&z_rproc->elem, cluster);
> +		i++;
> +	}
> +	/* wire in so each core can be cleaned up at drive remove */
> +	platform_set_drvdata(pdev, cluster);
> +	ret = 0;

Just return 0 here instead of resetting ret.

> +out:
> +	/* undo core0 upon any failures on core1 in split-mode */
> +	if (rpu_mode == PM_RPU_MODE_SPLIT && i == 1 && ret != 0) {
> +		z_rproc = container_of(cluster, struct zynqmp_r5_rproc, elem);

This looks very wrong to me; cluster isn't a pointer to member of
zynqmp_r5_rproc. Was this path tested?

> +		if (z_rproc->rproc) {

How would z_rproc->rproc be NULL here?

> +			rproc_del(z_rproc->rproc);
> +			rproc_free(z_rproc->rproc);
> +		}
> +
> +		if (z_rproc->tx_chan)
> +			mbox_free_channel(z_rproc->tx_chan);
> +		if (z_rproc->rx_chan)
> +			mbox_free_channel(z_rproc->rx_chan);

Again, this looks like a use-after-free.

> +	}
> +	return ret;
> +}
> +
> +/*
> + * for each core, clean up the following:
> + *	single rproc entry
> + *	mailbox tx, rx
> + */
> +static int zynqmp_r5_remoteproc_remove(struct platform_device *pdev)
> +{
> +	struct list_head *pos, *cluster = (struct list_head *)
> +					  platform_get_drvdata(pdev);
> +	struct zynqmp_r5_rproc *z_rproc = NULL;
> +	struct rproc *rproc = NULL;
> +
> +	list_for_each(pos, cluster) {
> +		z_rproc = list_entry(pos, struct zynqmp_r5_rproc, elem);
> +		rproc = z_rproc->rproc;
> +		if (rproc) {
> +			rproc_del(rproc);
> +			rproc_free(rproc);
> +		}
> +
> +		if (z_rproc->tx_chan)
> +			mbox_free_channel(z_rproc->tx_chan);
> +		if (z_rproc->rx_chan)
> +			mbox_free_channel(z_rproc->rx_chan);

The use-after-free issue I pointed out on v20 is still present here (and
has now been replicated in probe() as well).

> +	}
> +	return 0;
> +}
> +
> +/* Match table for OF platform binding */
> +static const struct of_device_id zynqmp_r5_remoteproc_match[] = {
> +	{ .compatible = "xlnx,zynqmp-r5-remoteproc", },
> +	{ /* end of list */ },
> +};
> +MODULE_DEVICE_TABLE(of, zynqmp_r5_remoteproc_match);
> +
> +static struct platform_driver zynqmp_r5_remoteproc_driver = {
> +	.probe = zynqmp_r5_remoteproc_probe,
> +	.remove = zynqmp_r5_remoteproc_remove,
> +	.driver = {
> +		.name = "zynqmp_r5_remoteproc",
> +		.of_match_table = zynqmp_r5_remoteproc_match,
> +	},
> +};
> +module_platform_driver(zynqmp_r5_remoteproc_driver);
> +
> +MODULE_AUTHOR("Ben Levinsky <ben.levinsky@xilinx.com>");
> +MODULE_LICENSE("GPL v2");
> -- 
> 2.17.1
> 

RE: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc driver

[Ben Levinsky]

Hi Michael,

> -----Original Message-----
> From: Michael Auchter <michael.auchter@ni.com>
> Sent: Monday, November 2, 2020 1:14 PM
> To: Ben Levinsky <BLEVINSK@xilinx.com>
> Cc: Stefano Stabellini <stefanos@xilinx.com>; mathieu.poirier@linaro.org;
> devicetree@vger.kernel.org; linux-remoteproc@vger.kernel.org; linux-
> kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org
> Subject: Re: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc
> driver
> 
> On Mon, Nov 02, 2020 at 11:38:59AM -0800, Ben Levinsky wrote:
> > R5 is included in Xilinx Zynq UltraScale MPSoC so by adding this
> > remotproc driver, we can boot the R5 sub-system in different 2
> > configurations -
> > 	* split
> > 	* lock-step
> >
> > The Xilinx R5 Remoteproc Driver boots the R5's via calls to the Xilinx
> > Platform Management Unit that handles the R5 configuration, memory
> access
> > and R5 lifecycle management. The interface to this manager is done in this
> > driver via zynqmp_pm_* function calls.
> >
> > Signed-off-by: Wendy Liang <wendy.liang@xilinx.com>
> > Signed-off-by: Michal Simek <michal.simek@xilinx.com>
> > Signed-off-by: Ed Mooring <ed.mooring@xilinx.com>
> > Signed-off-by: Jason Wu <j.wu@xilinx.com>
> > Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
> > ---
> > Update Xilinx R5 Remoteproc Driver as follows:
> > - update documentation for zynqmp_r5_probe
> > - restructure so that cluster initialization code is all in one place
> > - add memory allocation check for cluster
> > - add error handling in case of second core fails at probe but first core
> succeeded.
> >   to clean up the first core
> > - remove unneeded lines in zynqmp_r5_remoteproc_remove
> > ---
> >
> >  drivers/remoteproc/Kconfig                |   8 +
> >  drivers/remoteproc/Makefile               |   1 +
> >  drivers/remoteproc/zynqmp_r5_remoteproc.c | 784
> ++++++++++++++++++++++
> >  3 files changed, 793 insertions(+)
> >  create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c
> >
> > diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
> > index c6659dfea7c7..c2fe54b1d94f 100644
> > --- a/drivers/remoteproc/Kconfig
> > +++ b/drivers/remoteproc/Kconfig
> > @@ -275,6 +275,14 @@ config TI_K3_DSP_REMOTEPROC
> >  	  It's safe to say N here if you're not interested in utilizing
> >  	  the DSP slave processors.
> >
> > +config ZYNQMP_R5_REMOTEPROC
> > +	tristate "ZynqMP R5 remoteproc support"
> > +	depends on PM && ARCH_ZYNQMP
> > +	select RPMSG_VIRTIO
> > +	select ZYNQMP_IPI_MBOX
> > +	help
> > +	  Say y or m here to support ZynqMP R5 remote processors via the
> remote
> > +	  processor framework.
> >  endif # REMOTEPROC
> >
> >  endmenu
> > diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
> > index 3dfa28e6c701..ef1abff654c2 100644
> > --- a/drivers/remoteproc/Makefile
> > +++ b/drivers/remoteproc/Makefile
> > @@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC)		+=
> st_remoteproc.o
> >  obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
> >  obj-$(CONFIG_STM32_RPROC)		+= stm32_rproc.o
> >  obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
> > +obj-$(CONFIG_ZYNQMP_R5_REMOTEPROC)	+= zynqmp_r5_remoteproc.o
> > diff --git a/drivers/remoteproc/zynqmp_r5_remoteproc.c
> b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> > new file mode 100644
> > index 000000000000..993bd72e5664
> > --- /dev/null
> > +++ b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> > @@ -0,0 +1,784 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Zynq R5 Remote Processor driver
> > + *
> > + * Based on origin OMAP and Zynq Remote Processor driver
> > + *
> > + */
> > +
> > +#include <linux/firmware/xlnx-zynqmp.h>
> > +#include <linux/interrupt.h>
> > +#include <linux/kernel.h>
> > +#include <linux/list.h>
> > +#include <linux/mailbox_client.h>
> > +#include <linux/mailbox/zynqmp-ipi-message.h>
> > +#include <linux/module.h>
> > +#include <linux/of_address.h>
> > +#include <linux/of_platform.h>
> > +#include <linux/of_reserved_mem.h>
> > +#include <linux/platform_device.h>
> > +#include <linux/remoteproc.h>
> > +#include <linux/skbuff.h>
> > +#include <linux/sysfs.h>
> > +
> > +#include "remoteproc_internal.h"
> > +
> > +#define MAX_RPROCS	2 /* Support up to 2 RPU */
> > +#define MAX_MEM_PNODES	4 /* Max power nodes for one RPU memory
> instance */
> > +
> > +#define BANK_LIST_PROP	"meta-memory-regions"
> > +#define DDR_LIST_PROP	"memory-regions"
> > +
> > +/* IPI buffer MAX length */
> > +#define IPI_BUF_LEN_MAX	32U
> > +/* RX mailbox client buffer max length */
> > +#define RX_MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
> > +				 sizeof(struct zynqmp_ipi_message))
> > +
> > +/**
> > + * struct zynqmp_r5_mem - zynqmp rpu memory data
> > + * @pnode_id: TCM power domain ids
> > + * @res: memory resource
> > + * @node: list node
> > + */
> > +struct zynqmp_r5_mem {
> > +	u32 pnode_id[MAX_MEM_PNODES];
> > +	struct resource res;
> > +	struct list_head node;
> > +};
> > +
> > +/**
> > + * struct zynqmp_r5_rproc - zynqmp rpu remote processor state
> > + *			    this is for each individual R5 core's state
> > + *
> > + * @rx_mc_buf: rx mailbox client buffer to save the rx message
> > + * @tx_mc: tx mailbox client
> > + * @rx_mc: rx mailbox client * @dev: device of RPU instance
> > + * @mbox_work: mbox_work for the RPU remoteproc
> > + * @tx_mc_skbs: socket buffers for tx mailbox client
> > + * @dev: device of RPU instance
> > + * @rproc: rproc handle
> > + * @tx_chan: tx mailbox channel
> > + * @rx_chan: rx mailbox channel
> > + * @pnode_id: RPU CPU power domain id
> > + * @elem: linked list item
> > + * @dt_node: device tree node that holds information for 1 R5 core.
> > + */
> > +struct zynqmp_r5_rproc {
> > +	unsigned char rx_mc_buf[RX_MBOX_CLIENT_BUF_MAX];
> > +	struct mbox_client tx_mc;
> > +	struct mbox_client rx_mc;
> > +	struct work_struct mbox_work;
> > +	struct sk_buff_head tx_mc_skbs;
> > +	struct device *dev;
> > +	struct rproc *rproc;
> > +	struct mbox_chan *tx_chan;
> > +	struct mbox_chan *rx_chan;
> > +	u32 pnode_id;
> > +	struct list_head elem;
> > +};
> > +
> > +/*
> > + * r5_set_mode - set RPU operation mode
> > + * @z_rproc: Remote processor private data
> > + * @rpu_mode: mode specified by device tree to configure the RPU to
> > + *
> > + * set RPU operation mode
> > + *
> > + * Return: 0 for success, negative value for failure
> > + */
> > +static int r5_set_mode(struct zynqmp_r5_rproc *z_rproc,
> > +		       enum rpu_oper_mode rpu_mode)
> > +{
> > +	enum rpu_tcm_comb tcm_mode;
> > +	enum rpu_oper_mode cur_rpu_mode;
> > +	int ret;
> > +
> > +	ret = zynqmp_pm_get_rpu_mode(z_rproc->pnode_id,
> &cur_rpu_mode);
> > +	if (ret < 0)
> > +		return ret;
> > +
> > +	if (rpu_mode != cur_rpu_mode) {
> > +		ret = zynqmp_pm_set_rpu_mode(z_rproc->pnode_id,
> > +					     rpu_mode);
> > +		if (ret < 0)
> > +			return ret;
> > +	}
> > +
> > +	tcm_mode = (rpu_mode == PM_RPU_MODE_LOCKSTEP) ?
> > +		    PM_RPU_TCM_COMB : PM_RPU_TCM_SPLIT;
> > +	return zynqmp_pm_set_tcm_config(z_rproc->pnode_id, tcm_mode);
> > +}
> > +
> > +/*
> > + * release TCM banks when powering down R5 core
> > + */
> > +static int tcm_mem_release(struct rproc *rproc, struct rproc_mem_entry
> *mem)
> > +{
> > +	u32 pnode_id = (u64)mem->priv;
> > +
> > +	iounmap(mem->va);
> > +	return zynqmp_pm_release_node(pnode_id);
> > +}
> > +
> > +/*
> > + * given ID corresponding to R5 core in Xilinx Platform management (xpm)
> API,
> > + * try to use xpm wake call to wake R5 core
> > + */
> > +static int zynqmp_r5_rproc_start(struct rproc *rproc)
> > +{
> > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > +	enum rpu_boot_mem bootmem;
> > +
> > +	bootmem = (rproc->bootaddr & 0xF0000000) == 0xF0000000 ?
> > +		  PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
> > +
> > +	dev_dbg(rproc->dev.parent, "RPU boot from %s.",
> > +		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
> > +
> > +	return zynqmp_pm_request_wake(z_rproc->pnode_id, 1,
> > +				     bootmem,
> ZYNQMP_PM_REQUEST_ACK_NO);
> > +}
> > +
> > +/*
> > + * given ID corresponding to R5 core in Xilinx Platform management (xpm)
> API,
> > + * try to use xpm power down call to power off R5 core
> > + */
> > +static int zynqmp_r5_rproc_stop(struct rproc *rproc)
> > +{
> > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > +
> > +	return zynqmp_pm_force_pwrdwn(z_rproc->pnode_id,
> > +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> > +}
> > +
> > +/*
> > + * map in physical addr for  DDR mem carveout in rproc
> > + */
> > +static int zynqmp_r5_rproc_mem_alloc(struct rproc *rproc,
> > +				     struct rproc_mem_entry *mem)
> > +{
> > +	void *va;
> > +
> > +	va = ioremap_wc(mem->dma, mem->len);
> > +	if (IS_ERR_OR_NULL(va))
> > +		return -ENOMEM;
> > +
> > +	/* Update memory entry va */
> > +	mem->va = va;
> > +
> > +	return 0;
> > +}
> > +
> > +/* unmap rproc_mem_entry virtual addr */
> > +static int zynqmp_r5_rproc_mem_release(struct rproc *rproc,
> > +				       struct rproc_mem_entry *mem)
> > +{
> > +	iounmap(mem->va);
> > +	return 0;
> > +}
> > +
> > +/* construct rproc mem carveouts for DDR regions specified in device tree
> */
> > +static int parse_mem_regions(struct rproc *rproc)
> > +{
> > +	int num_mems, i;
> > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > +	struct device *dev = &rproc->dev;
> > +	struct device_node *np = z_rproc->dev->of_node;
> > +	struct rproc_mem_entry *mem;
> > +
> > +	num_mems = of_count_phandle_with_args(np, DDR_LIST_PROP,
> NULL);
> > +	if (num_mems <= 0)
> > +		return 0;
> > +
> > +	for (i = 0; i < num_mems; i++) {
> > +		struct device_node *node;
> > +		struct reserved_mem *rmem;
> > +
> > +		node = of_parse_phandle(np, DDR_LIST_PROP, i);
> > +		if (!node)
> > +			return -EINVAL;
> > +
> > +		rmem = of_reserved_mem_lookup(node);
> > +		if (!rmem)
> > +			return -EINVAL;
> > +
> > +		if (strstr(node->name, "vdev0vring")) {
> > +			int vring_id;
> > +			char name[16];
> > +
> > +			/*
> > +			 * expecting form of "rpuXvdev0vringX as documented
> > +			 * in xilinx remoteproc device tree binding
> > +			 */
> > +			if (strlen(node->name) < 15) {
> > +				dev_err(dev, "%pOF is less than 14 chars",
> > +					node);
> > +				return -EINVAL;
> > +			}
> > +
> > +			/*
> > +			 * can be 1 of multiple vring IDs per IPC channel
> > +			 * e.g. 'vdev0vring0' and 'vdev0vring1'
> > +			 */
> > +			vring_id = node->name[14] - '0';
> > +			snprintf(name, sizeof(name), "vdev0vring%d",
> vring_id);
> > +			/* Register vring */
> > +			mem = rproc_mem_entry_init(dev, NULL,
> > +						   (dma_addr_t)rmem->base,
> > +						   rmem->size, rmem->base,
> > +
> zynqmp_r5_rproc_mem_alloc,
> > +
> zynqmp_r5_rproc_mem_release,
> > +						   name);
> > +		} else {
> > +			/* Register DMA region */
> > +			int (*alloc)(struct rproc *r,
> > +				     struct rproc_mem_entry *rme);
> > +			int (*release)(struct rproc *r,
> > +				       struct rproc_mem_entry *rme);
> > +			char name[20];
> > +
> > +			if (strstr(node->name, "vdev0buffer")) {
> > +				alloc = NULL;
> > +				release = NULL;
> > +				strcpy(name, "vdev0buffer");
> > +			} else {
> > +				alloc = zynqmp_r5_rproc_mem_alloc;
> > +				release = zynqmp_r5_rproc_mem_release;
> > +				strcpy(name, node->name);
> > +			}
> > +
> > +			mem = rproc_mem_entry_init(dev, NULL,
> > +						   (dma_addr_t)rmem->base,
> > +						   rmem->size, rmem->base,
> > +						   alloc, release, name);
> > +		}
> > +		if (!mem)
> > +			return -ENOMEM;
> > +
> > +		rproc_add_carveout(rproc, mem);
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/* call Xilinx Platform manager to request access to TCM bank */
> > +static int zynqmp_r5_pm_request_tcm(struct device_node *tcm_node,
> > +				    struct device *dev, u32 *pnode_id)
> > +{
> > +	int ret;
> > +
> > +	ret = of_property_read_u32(tcm_node, "pnode-id", pnode_id);
> > +	if (ret)
> > +		return ret;
> > +
> > +	return zynqmp_pm_request_node(*pnode_id,
> ZYNQMP_PM_CAPABILITY_ACCESS, 0,
> > +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> > +}
> > +
> > +/*
> > + * Given TCM bank entry,
> > + * this callback will set device address for R5 running on TCM
> > + * and also setup virtual address for TCM bank remoteproc carveout
> > + */
> > +static int tcm_mem_alloc(struct rproc *rproc,
> > +			 struct rproc_mem_entry *mem)
> > +{
> > +	void *va;
> > +	struct device *dev = rproc->dev.parent;
> > +
> > +	va = ioremap_wc(mem->dma, mem->len);
> > +	if (IS_ERR_OR_NULL(va))
> > +		return -ENOMEM;
> > +
> > +	/* Update memory entry va */
> > +	mem->va = va;
> > +
> > +	va = devm_ioremap_wc(dev, mem->da, mem->len);
> > +	if (!va)
> > +		return -ENOMEM;
> > +	/* As R5 is 32 bit, wipe out extra high bits */
> > +	mem->da &= 0x000fffff;
> > +	/*
> > +	 * TCM Banks 0A and 0B (0xffe00000 and 0xffe20000)
> > +	 * are handled with the above line of code so do nothing
> > +	 * for this 2 banks
> > +	 */
> > +
> > +	/*
> > +	 * TCM Banks 1A and 1B (0xffe90000 and 0xffeb0000) still
> > +	 * need to be translated to 0x0 and 0x20000
> > +	 */
> > +	if (mem->da == 0x90000 || mem->da == 0xB0000)
> > +		mem->da -= 0x90000;
> > +
> > +	/* if translated TCM bank address is not valid report error */
> > +	if (mem->da != 0x0 && mem->da != 0x20000) {
> > +		dev_err(dev, "invalid TCM bank address: %x\n", mem->da);
> > +		return -EINVAL;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Given R5 node in remoteproc instance
> > + * allocate remoteproc carveout for TCM memory
> > + * needed for firmware to be loaded
> > + */
> > +static int parse_tcm_banks(struct rproc *rproc)
> > +{
> > +	int i, num_banks;
> > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > +	struct device *dev = &rproc->dev;
> > +	struct device_node *r5_node = z_rproc->dev->of_node;
> > +
> > +	/* go through TCM banks for r5 node */
> > +	num_banks = of_count_phandle_with_args(r5_node,
> BANK_LIST_PROP, NULL);
> > +	if (num_banks <= 0) {
> > +		dev_err(dev, "need to specify TCM banks\n");
> > +		return -EINVAL;
> > +	}
> > +	for (i = 0; i < num_banks; i++) {
> > +		struct resource rsc;
> > +		resource_size_t size;
> > +		struct device_node *dt_node;
> > +		struct rproc_mem_entry *mem;
> > +		int ret;
> > +		u32 pnode_id; /* zynqmp_pm* fn's expect u32 */
> > +
> > +		dt_node = of_parse_phandle(r5_node, BANK_LIST_PROP, i);
> > +		if (!dt_node)
> > +			return -EINVAL;
> > +
> > +		if (of_device_is_available(dt_node)) {
> > +			ret = of_address_to_resource(dt_node, 0, &rsc);
> > +			if (ret < 0)
> > +				return ret;
> > +
> > +			ret = zynqmp_r5_pm_request_tcm(dt_node, dev,
> &pnode_id);
> > +			if (ret < 0)
> > +				return ret;
> > +
> > +			/* add carveout */
> > +			size = resource_size(&rsc);
> > +			mem = rproc_mem_entry_init(dev, NULL, rsc.start,
> > +						   (int)size, rsc.start,
> > +						   tcm_mem_alloc,
> > +						   tcm_mem_release,
> > +						   rsc.name);
> > +			if (!mem)
> > +				return -ENOMEM;
> > +
> > +			mem->priv = (void *)(u64)pnode_id;
> > +			rproc_add_carveout(rproc, mem);
> > +		}
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +/*
> > + * when loading firmware, load in needed DDR, TCM memory regions and
> wire
> > + * these into remoteproc core's carveouts
> > + */
> > +static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware
> *fw)
> > +{
> > +	int ret;
> > +
> > +	ret = parse_tcm_banks(rproc);
> > +	if (ret)
> > +		return ret;
> > +
> > +	ret = parse_mem_regions(rproc);
> > +	if (ret)
> > +		return ret;
> > +
> > +	ret = rproc_elf_load_rsc_table(rproc, fw);
> > +	if (ret == -EINVAL) {
> > +		/*
> > +		 * resource table only required for IPC.
> > +		 * if not present, this is not necessarily an error;
> > +		 * for example, loading r5 hello world application
> > +		 * so simply inform user and keep going.
> > +		 */
> > +		dev_info(&rproc->dev, "no resource table found.\n");
> > +		ret = 0;
> > +	}
> > +	return ret;
> > +}
> > +
> > +/* kick a firmware */
> > +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
> > +{
> > +	struct sk_buff *skb;
> > +	unsigned int skb_len;
> > +	struct zynqmp_ipi_message *mb_msg;
> > +	int ret;
> > +
> > +	struct device *dev = rproc->dev.parent;
> > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > +
> > +	skb_len = (unsigned int)(sizeof(vqid) + sizeof(mb_msg));
> > +	skb = alloc_skb(skb_len, GFP_ATOMIC);
> > +	if (!skb)
> > +		return;
> > +
> > +	mb_msg = (struct zynqmp_ipi_message *)skb_put(skb, skb_len);
> > +	mb_msg->len = sizeof(vqid);
> > +	memcpy(mb_msg->data, &vqid, sizeof(vqid));
> > +	skb_queue_tail(&z_rproc->tx_mc_skbs, skb);
> > +	ret = mbox_send_message(z_rproc->tx_chan, mb_msg);
> > +	if (ret < 0) {
> > +		dev_warn(dev, "Failed to kick remote.\n");
> > +		skb_dequeue_tail(&z_rproc->tx_mc_skbs);
> > +		kfree_skb(skb);
> > +	}
> > +}
> > +
> > +static struct rproc_ops zynqmp_r5_rproc_ops = {
> > +	.start		= zynqmp_r5_rproc_start,
> > +	.stop		= zynqmp_r5_rproc_stop,
> > +	.load		= rproc_elf_load_segments,
> > +	.parse_fw	= zynqmp_r5_parse_fw,
> > +	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
> > +	.sanity_check	= rproc_elf_sanity_check,
> > +	.get_boot_addr	= rproc_elf_get_boot_addr,
> > +	.kick		= zynqmp_r5_rproc_kick,
> > +};
> > +
> > +/**
> > + * event_notified_idr_cb() - event notified idr callback
> > + * @id: idr id
> > + * @ptr: pointer to idr private data
> > + * @data: data passed to idr_for_each callback
> > + *
> > + * Pass notification to remoteproc virtio
> > + *
> > + * Return: 0. having return is to satisfy the idr_for_each() function
> > + *          pointer input argument requirement.
> > + **/
> > +static int event_notified_idr_cb(int id, void *ptr, void *data)
> > +{
> > +	struct rproc *rproc = data;
> > +
> > +	(void)rproc_vq_interrupt(rproc, id);
> > +	return 0;
> > +}
> > +
> > +/**
> > + * handle_event_notified() - remoteproc notification work funciton
> > + * @work: pointer to the work structure
> > + *
> > + * It checks each registered remoteproc notify IDs.
> > + */
> > +static void handle_event_notified(struct work_struct *work)
> > +{
> > +	struct rproc *rproc;
> > +	struct zynqmp_r5_rproc *z_rproc;
> > +
> > +	z_rproc = container_of(work, struct zynqmp_r5_rproc, mbox_work);
> > +
> > +	(void)mbox_send_message(z_rproc->rx_chan, NULL);
> > +	rproc = z_rproc->rproc;
> > +	/*
> > +	 * We only use IPI for interrupt. The firmware side may or may
> > +	 * not write the notifyid when it trigger IPI.
> > +	 * And thus, we scan through all the registered notifyids.
> > +	 */
> > +	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
> > +}
> > +
> > +/**
> > + * zynqmp_r5_mb_rx_cb() - Receive channel mailbox callback
> > + * @cl: mailbox client
> > + * @mssg: message pointer
> > + *
> > + * It will schedule the R5 notification work.
> > + */
> > +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *mssg)
> > +{
> > +	struct zynqmp_r5_rproc *z_rproc;
> > +
> > +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, rx_mc);
> > +	if (mssg) {
> > +		struct zynqmp_ipi_message *ipi_msg, *buf_msg;
> > +		size_t len;
> > +
> > +		ipi_msg = (struct zynqmp_ipi_message *)mssg;
> > +		buf_msg = (struct zynqmp_ipi_message *)z_rproc->rx_mc_buf;
> > +		len = (ipi_msg->len >= IPI_BUF_LEN_MAX) ?
> > +		      IPI_BUF_LEN_MAX : ipi_msg->len;
> > +		buf_msg->len = len;
> > +		memcpy(buf_msg->data, ipi_msg->data, len);
> > +	}
> > +	schedule_work(&z_rproc->mbox_work);
> > +}
> > +
> > +/**
> > + * zynqmp_r5_mb_tx_done() - Request has been sent to the remote
> > + * @cl: mailbox client
> > + * @mssg: pointer to the message which has been sent
> > + * @r: status of last TX - OK or error
> > + *
> > + * It will be called by the mailbox framework when the last TX has done.
> > + */
> > +static void zynqmp_r5_mb_tx_done(struct mbox_client *cl, void *mssg, int
> r)
> > +{
> > +	struct zynqmp_r5_rproc *z_rproc;
> > +	struct sk_buff *skb;
> > +
> > +	if (!mssg)
> > +		return;
> > +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, tx_mc);
> > +	skb = skb_dequeue(&z_rproc->tx_mc_skbs);
> > +	kfree_skb(skb);
> > +}
> > +
> > +/**
> > + * zynqmp_r5_setup_mbox() - Setup mailboxes
> > + *			    this is used for each individual R5 core
> > + *
> > + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> > + * @node: pointer of the device node
> > + *
> > + * Function to setup mailboxes to talk to RPU.
> > + *
> > + * Return: 0 for success, negative value for failure.
> > + */
> > +static int zynqmp_r5_setup_mbox(struct zynqmp_r5_rproc *z_rproc,
> > +				struct device_node *node)
> > +{
> > +	struct mbox_client *mclient;
> > +
> > +	/* Setup TX mailbox channel client */
> > +	mclient = &z_rproc->tx_mc;
> > +	mclient->rx_callback = NULL;
> > +	mclient->tx_block = false;
> > +	mclient->knows_txdone = false;
> > +	mclient->tx_done = zynqmp_r5_mb_tx_done;
> > +	mclient->dev = z_rproc->dev;
> > +
> > +	/* Setup TX mailbox channel client */
> > +	mclient = &z_rproc->rx_mc;
> > +	mclient->dev = z_rproc->dev;
> > +	mclient->rx_callback = zynqmp_r5_mb_rx_cb;
> > +	mclient->tx_block = false;
> > +	mclient->knows_txdone = false;
> > +
> > +	INIT_WORK(&z_rproc->mbox_work, handle_event_notified);
> > +
> > +	/* Request TX and RX channels */
> > +	z_rproc->tx_chan = mbox_request_channel_byname(&z_rproc-
> >tx_mc, "tx");
> > +	if (IS_ERR(z_rproc->tx_chan)) {
> > +		dev_err(z_rproc->dev, "failed to request mbox tx channel.\n");
> > +		z_rproc->tx_chan = NULL;
> > +		return -EINVAL;
> > +	}
> > +
> > +	z_rproc->rx_chan = mbox_request_channel_byname(&z_rproc-
> >rx_mc, "rx");
> > +	if (IS_ERR(z_rproc->rx_chan)) {
> > +		dev_err(z_rproc->dev, "failed to request mbox rx channel.\n");
> > +		z_rproc->rx_chan = NULL;
> > +		return -EINVAL;
> > +	}
> > +	skb_queue_head_init(&z_rproc->tx_mc_skbs);
> > +
> > +	return 0;
> > +}
> > +
> > +/**
> > + * zynqmp_r5_probe() - Probes ZynqMP R5 processor device node
> > + *		       this is called for each individual R5 core to
> > + *		       set up mailbox, Xilinx platform manager unique ID,
> > + *		       add to rproc core
> > + *
> > + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> > + * @pdev: domain platform device for current R5 core
> > + * @node: pointer of the device node for current R5 core
> > + * @rpu_mode: mode to configure RPU, split or lockstep
> > + * @core: Xilinx specific remoteproc structure used later to link
> > + *           in to cluster of cores
> 
> This still does not match the param list (there's no z_rproc)
> 
Isn't core in this context the z_rproc? I can change the argument name if that is what you mean
> > + *
> > + * Function to retrieve the information of the ZynqMP R5 device node.
> > + *
> > + * Return: 0 for success, negative value for failure.
> > + */
> > +static int zynqmp_r5_probe(struct platform_device *pdev,
> > +			   struct device_node *node,
> > +			   enum rpu_oper_mode rpu_mode,
> > +			   struct zynqmp_r5_rproc **core)
> > +{
> > +	int ret;
> > +	struct device *dev = &pdev->dev;
> > +	struct rproc *rproc_ptr;
> > +	struct zynqmp_r5_rproc *z_rproc;
> > +
> > +	/* Allocate remoteproc instance */
> > +	/* dev here is parent device of the allocated rproc's dev field */
> > +	rproc_ptr = rproc_alloc(dev, dev_name(dev), &zynqmp_r5_rproc_ops,
> > +				NULL, sizeof(struct zynqmp_r5_rproc));
> > +	if (!rproc_ptr)
> > +		return -ENOMEM;
> > +	z_rproc = rproc_ptr->priv;
> > +	z_rproc->rproc = rproc_ptr;
> > +	z_rproc->dev = dev;
> > +	/* Set up DMA mask */
> > +	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
> > +	if (ret)
> > +		goto error;
> > +	/* Get R5 power domain node */
> > +	ret = of_property_read_u32(node, "pnode-id", &z_rproc->pnode_id);
> > +	if (ret)
> > +		goto error;
> > +
> > +	ret = r5_set_mode(z_rproc, rpu_mode);
	> > +	if (ret)
> > +		return ret;
> 
> goto error?
> 
Ok will change
> > +
> > +	if (of_property_read_bool(node, "mboxes")) {
> > +		ret = zynqmp_r5_setup_mbox(z_rproc, node);
> > +		if (ret)
> > +			goto error;
> > +	}
> > +	/* Add R5 remoteproc */
> > +	ret = rproc_add(rproc_ptr);
> > +	if (ret)
> > +		goto error;
> > +	*core = z_rproc;
> > +
> > +	return 0;
> > +error:
> > +	if (z_rproc->rproc)
> 
> How would this ever be NULL?
Ok will change
> 
> > +		rproc_free(z_rproc->rproc);
> > +	z_rproc->rproc = NULL;
> 
> This looks like another use-after-free, but it also seems unnecessary.
Ok will change
> 
> > +	return ret;
> > +}
> > +
> > +/*
> > + * called when driver is probed, for each R5 core specified in DT,
> > + * setup as needed to do remoteproc-related operations
> > + */
> > +static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev)
> > +{
> > +	int ret, i;
> > +	struct device *dev = &pdev->dev;
> > +	struct device_node *nc;
> > +	enum rpu_oper_mode rpu_mode;
> > +	struct list_head *cluster; /* list to track each core's rproc */
> > +	struct zynqmp_r5_rproc *z_rproc;
> > +	struct platform_device *child_pdev;
> > +
> > +	rpu_mode = of_property_read_bool(dev->of_node, "lockstep-mode")
> ?
> > +		   PM_RPU_MODE_LOCKSTEP : PM_RPU_MODE_SPLIT;
> > +	dev_dbg(dev, "RPU configuration: %s\n",
> > +		rpu_mode == PM_RPU_MODE_LOCKSTEP ? "lockstep" :
> "split");
> > +
> > +	/*
> > +	 * if 2 RPUs provided but one is lockstep, then we have an
> > +	 * invalid configuration.
> > +	 */
> > +	i = of_get_available_child_count(dev->of_node);
> > +	if ((rpu_mode == PM_RPU_MODE_LOCKSTEP && i != 1) || i >
> MAX_RPROCS)
> > +		return -EINVAL;
> > +
> > +	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
> > +	if (!cluster)
> > +		return -ENOMEM;
> > +	INIT_LIST_HEAD(cluster);
> > +
> > +	ret = devm_of_platform_populate(dev);
> > +	if (ret) {
> > +		dev_err(dev, "devm_of_platform_populate failed, ret =
> %d\n",
> > +			ret);
> > +		return ret;
> > +	}
> > +
> > +	/* probe each individual r5 core's remoteproc-related info */
> > +	i = 0;
> > +	for_each_available_child_of_node(dev->of_node, nc) {
> > +		child_pdev = of_find_device_by_node(nc);
> > +		if (!child_pdev) {
> > +			dev_err(dev, "could not get R5 core platform
> device\n");
> > +			ret = -ENODEV;
> > +			goto out;
> > +		}
> > +		ret = zynqmp_r5_probe(child_pdev, nc, rpu_mode, &z_rproc);
> > +		dev_dbg(dev, "%s to probe rpu %pOF\n",
> > +			ret ? "Failed" : "Able",
> > +			nc);
> > +		if (!z_rproc)
> > +			ret = -EINVAL;
> > +		if (ret)
> > +			goto out;
> > +
> > +		list_add_tail(&z_rproc->elem, cluster);
> > +		i++;
> > +	}
> > +	/* wire in so each core can be cleaned up at drive remove */
> > +	platform_set_drvdata(pdev, cluster);
> > +	ret = 0;
> 
> Just return 0 here instead of resetting ret.
Ok will change
> 
> > +out:
> > +	/* undo core0 upon any failures on core1 in split-mode */
> > +	if (rpu_mode == PM_RPU_MODE_SPLIT && i == 1 && ret != 0) {
> > +		z_rproc = container_of(cluster, struct zynqmp_r5_rproc,
> elem);
> 
> This looks very wrong to me; cluster isn't a pointer to member of
> zynqmp_r5_rproc. Was this path tested?
> 
It was tested and ran, but Ok will change
> > +		if (z_rproc->rproc) {
> 
> How would z_rproc->rproc be NULL here?

Good point, will remove this check
> 
> > +			rproc_del(z_rproc->rproc);
> > +			rproc_free(z_rproc->rproc);
> > +		}
> > +
> > +		if (z_rproc->tx_chan)
> > +			mbox_free_channel(z_rproc->tx_chan);
> > +		if (z_rproc->rx_chan)
> > +			mbox_free_channel(z_rproc->rx_chan);
> 
> Again, this looks like a use-after-free.
> 
Ok will change
> > +	}
> > +	return ret;
> > +}
> > +
> > +/*
> > + * for each core, clean up the following:
> > + *	single rproc entry
> > + *	mailbox tx, rx
> > + */
> > +static int zynqmp_r5_remoteproc_remove(struct platform_device *pdev)
> > +{
> > +	struct list_head *pos, *cluster = (struct list_head *)
> > +					  platform_get_drvdata(pdev);
> > +	struct zynqmp_r5_rproc *z_rproc = NULL;
> > +	struct rproc *rproc = NULL;
> > +
> > +	list_for_each(pos, cluster) {
> > +		z_rproc = list_entry(pos, struct zynqmp_r5_rproc, elem);
> > +		rproc = z_rproc->rproc;
> > +		if (rproc) {
> > +			rproc_del(rproc);
> > +			rproc_free(rproc);
> > +		}
> > +
> > +		if (z_rproc->tx_chan)
> > +			mbox_free_channel(z_rproc->tx_chan);
> > +		if (z_rproc->rx_chan)
> > +			mbox_free_channel(z_rproc->rx_chan);
> 
> The use-after-free issue I pointed out on v20 is still present here (and
> has now been replicated in probe() as well).
> 
So for this and the other mbox_free_channel call, is it alright to check if the mbox property is in the z_rproc pointer's dev->of_node? If so then the setup_mbox was called and ran ok if the driver got this far.
E.g.	 if (of_property_read_bool(z_rproc->dev->of_node, "mboxes")) {

[...]

Thanks
Ben

Re: RE: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc driver

[Michael Auchter]

On Mon, Nov 02, 2020 at 09:46:28PM +0000, Ben Levinsky wrote:
> Hi Michael,
> 
> > -----Original Message-----
> > From: Michael Auchter <michael.auchter@ni.com>
> > Sent: Monday, November 2, 2020 1:14 PM
> > To: Ben Levinsky <BLEVINSK@xilinx.com>
> > Cc: Stefano Stabellini <stefanos@xilinx.com>; mathieu.poirier@linaro.org;
> > devicetree@vger.kernel.org; linux-remoteproc@vger.kernel.org; linux-
> > kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org
> > Subject: Re: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc
> > driver
> > 
> > On Mon, Nov 02, 2020 at 11:38:59AM -0800, Ben Levinsky wrote:
> > > R5 is included in Xilinx Zynq UltraScale MPSoC so by adding this
> > > remotproc driver, we can boot the R5 sub-system in different 2
> > > configurations -
> > > 	* split
> > > 	* lock-step
> > >
> > > The Xilinx R5 Remoteproc Driver boots the R5's via calls to the Xilinx
> > > Platform Management Unit that handles the R5 configuration, memory
> > access
> > > and R5 lifecycle management. The interface to this manager is done in this
> > > driver via zynqmp_pm_* function calls.
> > >
> > > Signed-off-by: Wendy Liang <wendy.liang@xilinx.com>
> > > Signed-off-by: Michal Simek <michal.simek@xilinx.com>
> > > Signed-off-by: Ed Mooring <ed.mooring@xilinx.com>
> > > Signed-off-by: Jason Wu <j.wu@xilinx.com>
> > > Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
> > > ---
> > > Update Xilinx R5 Remoteproc Driver as follows:
> > > - update documentation for zynqmp_r5_probe
> > > - restructure so that cluster initialization code is all in one place
> > > - add memory allocation check for cluster
> > > - add error handling in case of second core fails at probe but first core
> > succeeded.
> > >   to clean up the first core
> > > - remove unneeded lines in zynqmp_r5_remoteproc_remove
> > > ---
> > >
> > >  drivers/remoteproc/Kconfig                |   8 +
> > >  drivers/remoteproc/Makefile               |   1 +
> > >  drivers/remoteproc/zynqmp_r5_remoteproc.c | 784
> > ++++++++++++++++++++++
> > >  3 files changed, 793 insertions(+)
> > >  create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c
> > >
> > > diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
> > > index c6659dfea7c7..c2fe54b1d94f 100644
> > > --- a/drivers/remoteproc/Kconfig
> > > +++ b/drivers/remoteproc/Kconfig
> > > @@ -275,6 +275,14 @@ config TI_K3_DSP_REMOTEPROC
> > >  	  It's safe to say N here if you're not interested in utilizing
> > >  	  the DSP slave processors.
> > >
> > > +config ZYNQMP_R5_REMOTEPROC
> > > +	tristate "ZynqMP R5 remoteproc support"
> > > +	depends on PM && ARCH_ZYNQMP
> > > +	select RPMSG_VIRTIO
> > > +	select ZYNQMP_IPI_MBOX
> > > +	help
> > > +	  Say y or m here to support ZynqMP R5 remote processors via the
> > remote
> > > +	  processor framework.
> > >  endif # REMOTEPROC
> > >
> > >  endmenu
> > > diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
> > > index 3dfa28e6c701..ef1abff654c2 100644
> > > --- a/drivers/remoteproc/Makefile
> > > +++ b/drivers/remoteproc/Makefile
> > > @@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC)		+=
> > st_remoteproc.o
> > >  obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
> > >  obj-$(CONFIG_STM32_RPROC)		+= stm32_rproc.o
> > >  obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
> > > +obj-$(CONFIG_ZYNQMP_R5_REMOTEPROC)	+= zynqmp_r5_remoteproc.o
> > > diff --git a/drivers/remoteproc/zynqmp_r5_remoteproc.c
> > b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> > > new file mode 100644
> > > index 000000000000..993bd72e5664
> > > --- /dev/null
> > > +++ b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> > > @@ -0,0 +1,784 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * Zynq R5 Remote Processor driver
> > > + *
> > > + * Based on origin OMAP and Zynq Remote Processor driver
> > > + *
> > > + */
> > > +
> > > +#include <linux/firmware/xlnx-zynqmp.h>
> > > +#include <linux/interrupt.h>
> > > +#include <linux/kernel.h>
> > > +#include <linux/list.h>
> > > +#include <linux/mailbox_client.h>
> > > +#include <linux/mailbox/zynqmp-ipi-message.h>
> > > +#include <linux/module.h>
> > > +#include <linux/of_address.h>
> > > +#include <linux/of_platform.h>
> > > +#include <linux/of_reserved_mem.h>
> > > +#include <linux/platform_device.h>
> > > +#include <linux/remoteproc.h>
> > > +#include <linux/skbuff.h>
> > > +#include <linux/sysfs.h>
> > > +
> > > +#include "remoteproc_internal.h"
> > > +
> > > +#define MAX_RPROCS	2 /* Support up to 2 RPU */
> > > +#define MAX_MEM_PNODES	4 /* Max power nodes for one RPU memory
> > instance */
> > > +
> > > +#define BANK_LIST_PROP	"meta-memory-regions"
> > > +#define DDR_LIST_PROP	"memory-regions"
> > > +
> > > +/* IPI buffer MAX length */
> > > +#define IPI_BUF_LEN_MAX	32U
> > > +/* RX mailbox client buffer max length */
> > > +#define RX_MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
> > > +				 sizeof(struct zynqmp_ipi_message))
> > > +
> > > +/**
> > > + * struct zynqmp_r5_mem - zynqmp rpu memory data
> > > + * @pnode_id: TCM power domain ids
> > > + * @res: memory resource
> > > + * @node: list node
> > > + */
> > > +struct zynqmp_r5_mem {
> > > +	u32 pnode_id[MAX_MEM_PNODES];
> > > +	struct resource res;
> > > +	struct list_head node;
> > > +};
> > > +
> > > +/**
> > > + * struct zynqmp_r5_rproc - zynqmp rpu remote processor state
> > > + *			    this is for each individual R5 core's state
> > > + *
> > > + * @rx_mc_buf: rx mailbox client buffer to save the rx message
> > > + * @tx_mc: tx mailbox client
> > > + * @rx_mc: rx mailbox client * @dev: device of RPU instance
> > > + * @mbox_work: mbox_work for the RPU remoteproc
> > > + * @tx_mc_skbs: socket buffers for tx mailbox client
> > > + * @dev: device of RPU instance
> > > + * @rproc: rproc handle
> > > + * @tx_chan: tx mailbox channel
> > > + * @rx_chan: rx mailbox channel
> > > + * @pnode_id: RPU CPU power domain id
> > > + * @elem: linked list item
> > > + * @dt_node: device tree node that holds information for 1 R5 core.
> > > + */
> > > +struct zynqmp_r5_rproc {
> > > +	unsigned char rx_mc_buf[RX_MBOX_CLIENT_BUF_MAX];
> > > +	struct mbox_client tx_mc;
> > > +	struct mbox_client rx_mc;
> > > +	struct work_struct mbox_work;
> > > +	struct sk_buff_head tx_mc_skbs;
> > > +	struct device *dev;
> > > +	struct rproc *rproc;
> > > +	struct mbox_chan *tx_chan;
> > > +	struct mbox_chan *rx_chan;
> > > +	u32 pnode_id;
> > > +	struct list_head elem;
> > > +};
> > > +
> > > +/*
> > > + * r5_set_mode - set RPU operation mode
> > > + * @z_rproc: Remote processor private data
> > > + * @rpu_mode: mode specified by device tree to configure the RPU to
> > > + *
> > > + * set RPU operation mode
> > > + *
> > > + * Return: 0 for success, negative value for failure
> > > + */
> > > +static int r5_set_mode(struct zynqmp_r5_rproc *z_rproc,
> > > +		       enum rpu_oper_mode rpu_mode)
> > > +{
> > > +	enum rpu_tcm_comb tcm_mode;
> > > +	enum rpu_oper_mode cur_rpu_mode;
> > > +	int ret;
> > > +
> > > +	ret = zynqmp_pm_get_rpu_mode(z_rproc->pnode_id,
> > &cur_rpu_mode);
> > > +	if (ret < 0)
> > > +		return ret;
> > > +
> > > +	if (rpu_mode != cur_rpu_mode) {
> > > +		ret = zynqmp_pm_set_rpu_mode(z_rproc->pnode_id,
> > > +					     rpu_mode);
> > > +		if (ret < 0)
> > > +			return ret;
> > > +	}
> > > +
> > > +	tcm_mode = (rpu_mode == PM_RPU_MODE_LOCKSTEP) ?
> > > +		    PM_RPU_TCM_COMB : PM_RPU_TCM_SPLIT;
> > > +	return zynqmp_pm_set_tcm_config(z_rproc->pnode_id, tcm_mode);
> > > +}
> > > +
> > > +/*
> > > + * release TCM banks when powering down R5 core
> > > + */
> > > +static int tcm_mem_release(struct rproc *rproc, struct rproc_mem_entry
> > *mem)
> > > +{
> > > +	u32 pnode_id = (u64)mem->priv;
> > > +
> > > +	iounmap(mem->va);
> > > +	return zynqmp_pm_release_node(pnode_id);
> > > +}
> > > +
> > > +/*
> > > + * given ID corresponding to R5 core in Xilinx Platform management (xpm)
> > API,
> > > + * try to use xpm wake call to wake R5 core
> > > + */
> > > +static int zynqmp_r5_rproc_start(struct rproc *rproc)
> > > +{
> > > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > > +	enum rpu_boot_mem bootmem;
> > > +
> > > +	bootmem = (rproc->bootaddr & 0xF0000000) == 0xF0000000 ?
> > > +		  PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
> > > +
> > > +	dev_dbg(rproc->dev.parent, "RPU boot from %s.",
> > > +		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
> > > +
> > > +	return zynqmp_pm_request_wake(z_rproc->pnode_id, 1,
> > > +				     bootmem,
> > ZYNQMP_PM_REQUEST_ACK_NO);
> > > +}
> > > +
> > > +/*
> > > + * given ID corresponding to R5 core in Xilinx Platform management (xpm)
> > API,
> > > + * try to use xpm power down call to power off R5 core
> > > + */
> > > +static int zynqmp_r5_rproc_stop(struct rproc *rproc)
> > > +{
> > > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > > +
> > > +	return zynqmp_pm_force_pwrdwn(z_rproc->pnode_id,
> > > +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> > > +}
> > > +
> > > +/*
> > > + * map in physical addr for  DDR mem carveout in rproc
> > > + */
> > > +static int zynqmp_r5_rproc_mem_alloc(struct rproc *rproc,
> > > +				     struct rproc_mem_entry *mem)
> > > +{
> > > +	void *va;
> > > +
> > > +	va = ioremap_wc(mem->dma, mem->len);
> > > +	if (IS_ERR_OR_NULL(va))
> > > +		return -ENOMEM;
> > > +
> > > +	/* Update memory entry va */
> > > +	mem->va = va;
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/* unmap rproc_mem_entry virtual addr */
> > > +static int zynqmp_r5_rproc_mem_release(struct rproc *rproc,
> > > +				       struct rproc_mem_entry *mem)
> > > +{
> > > +	iounmap(mem->va);
> > > +	return 0;
> > > +}
> > > +
> > > +/* construct rproc mem carveouts for DDR regions specified in device tree
> > */
> > > +static int parse_mem_regions(struct rproc *rproc)
> > > +{
> > > +	int num_mems, i;
> > > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > > +	struct device *dev = &rproc->dev;
> > > +	struct device_node *np = z_rproc->dev->of_node;
> > > +	struct rproc_mem_entry *mem;
> > > +
> > > +	num_mems = of_count_phandle_with_args(np, DDR_LIST_PROP,
> > NULL);
> > > +	if (num_mems <= 0)
> > > +		return 0;
> > > +
> > > +	for (i = 0; i < num_mems; i++) {
> > > +		struct device_node *node;
> > > +		struct reserved_mem *rmem;
> > > +
> > > +		node = of_parse_phandle(np, DDR_LIST_PROP, i);
> > > +		if (!node)
> > > +			return -EINVAL;
> > > +
> > > +		rmem = of_reserved_mem_lookup(node);
> > > +		if (!rmem)
> > > +			return -EINVAL;
> > > +
> > > +		if (strstr(node->name, "vdev0vring")) {
> > > +			int vring_id;
> > > +			char name[16];
> > > +
> > > +			/*
> > > +			 * expecting form of "rpuXvdev0vringX as documented
> > > +			 * in xilinx remoteproc device tree binding
> > > +			 */
> > > +			if (strlen(node->name) < 15) {
> > > +				dev_err(dev, "%pOF is less than 14 chars",
> > > +					node);
> > > +				return -EINVAL;
> > > +			}
> > > +
> > > +			/*
> > > +			 * can be 1 of multiple vring IDs per IPC channel
> > > +			 * e.g. 'vdev0vring0' and 'vdev0vring1'
> > > +			 */
> > > +			vring_id = node->name[14] - '0';
> > > +			snprintf(name, sizeof(name), "vdev0vring%d",
> > vring_id);
> > > +			/* Register vring */
> > > +			mem = rproc_mem_entry_init(dev, NULL,
> > > +						   (dma_addr_t)rmem->base,
> > > +						   rmem->size, rmem->base,
> > > +
> > zynqmp_r5_rproc_mem_alloc,
> > > +
> > zynqmp_r5_rproc_mem_release,
> > > +						   name);
> > > +		} else {
> > > +			/* Register DMA region */
> > > +			int (*alloc)(struct rproc *r,
> > > +				     struct rproc_mem_entry *rme);
> > > +			int (*release)(struct rproc *r,
> > > +				       struct rproc_mem_entry *rme);
> > > +			char name[20];
> > > +
> > > +			if (strstr(node->name, "vdev0buffer")) {
> > > +				alloc = NULL;
> > > +				release = NULL;
> > > +				strcpy(name, "vdev0buffer");
> > > +			} else {
> > > +				alloc = zynqmp_r5_rproc_mem_alloc;
> > > +				release = zynqmp_r5_rproc_mem_release;
> > > +				strcpy(name, node->name);
> > > +			}
> > > +
> > > +			mem = rproc_mem_entry_init(dev, NULL,
> > > +						   (dma_addr_t)rmem->base,
> > > +						   rmem->size, rmem->base,
> > > +						   alloc, release, name);
> > > +		}
> > > +		if (!mem)
> > > +			return -ENOMEM;
> > > +
> > > +		rproc_add_carveout(rproc, mem);
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/* call Xilinx Platform manager to request access to TCM bank */
> > > +static int zynqmp_r5_pm_request_tcm(struct device_node *tcm_node,
> > > +				    struct device *dev, u32 *pnode_id)
> > > +{
> > > +	int ret;
> > > +
> > > +	ret = of_property_read_u32(tcm_node, "pnode-id", pnode_id);
> > > +	if (ret)
> > > +		return ret;
> > > +
> > > +	return zynqmp_pm_request_node(*pnode_id,
> > ZYNQMP_PM_CAPABILITY_ACCESS, 0,
> > > +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> > > +}
> > > +
> > > +/*
> > > + * Given TCM bank entry,
> > > + * this callback will set device address for R5 running on TCM
> > > + * and also setup virtual address for TCM bank remoteproc carveout
> > > + */
> > > +static int tcm_mem_alloc(struct rproc *rproc,
> > > +			 struct rproc_mem_entry *mem)
> > > +{
> > > +	void *va;
> > > +	struct device *dev = rproc->dev.parent;
> > > +
> > > +	va = ioremap_wc(mem->dma, mem->len);
> > > +	if (IS_ERR_OR_NULL(va))
> > > +		return -ENOMEM;
> > > +
> > > +	/* Update memory entry va */
> > > +	mem->va = va;
> > > +
> > > +	va = devm_ioremap_wc(dev, mem->da, mem->len);
> > > +	if (!va)
> > > +		return -ENOMEM;
> > > +	/* As R5 is 32 bit, wipe out extra high bits */
> > > +	mem->da &= 0x000fffff;
> > > +	/*
> > > +	 * TCM Banks 0A and 0B (0xffe00000 and 0xffe20000)
> > > +	 * are handled with the above line of code so do nothing
> > > +	 * for this 2 banks
> > > +	 */
> > > +
> > > +	/*
> > > +	 * TCM Banks 1A and 1B (0xffe90000 and 0xffeb0000) still
> > > +	 * need to be translated to 0x0 and 0x20000
> > > +	 */
> > > +	if (mem->da == 0x90000 || mem->da == 0xB0000)
> > > +		mem->da -= 0x90000;
> > > +
> > > +	/* if translated TCM bank address is not valid report error */
> > > +	if (mem->da != 0x0 && mem->da != 0x20000) {
> > > +		dev_err(dev, "invalid TCM bank address: %x\n", mem->da);
> > > +		return -EINVAL;
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/*
> > > + * Given R5 node in remoteproc instance
> > > + * allocate remoteproc carveout for TCM memory
> > > + * needed for firmware to be loaded
> > > + */
> > > +static int parse_tcm_banks(struct rproc *rproc)
> > > +{
> > > +	int i, num_banks;
> > > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > > +	struct device *dev = &rproc->dev;
> > > +	struct device_node *r5_node = z_rproc->dev->of_node;
> > > +
> > > +	/* go through TCM banks for r5 node */
> > > +	num_banks = of_count_phandle_with_args(r5_node,
> > BANK_LIST_PROP, NULL);
> > > +	if (num_banks <= 0) {
> > > +		dev_err(dev, "need to specify TCM banks\n");
> > > +		return -EINVAL;
> > > +	}
> > > +	for (i = 0; i < num_banks; i++) {
> > > +		struct resource rsc;
> > > +		resource_size_t size;
> > > +		struct device_node *dt_node;
> > > +		struct rproc_mem_entry *mem;
> > > +		int ret;
> > > +		u32 pnode_id; /* zynqmp_pm* fn's expect u32 */
> > > +
> > > +		dt_node = of_parse_phandle(r5_node, BANK_LIST_PROP, i);
> > > +		if (!dt_node)
> > > +			return -EINVAL;
> > > +
> > > +		if (of_device_is_available(dt_node)) {
> > > +			ret = of_address_to_resource(dt_node, 0, &rsc);
> > > +			if (ret < 0)
> > > +				return ret;
> > > +
> > > +			ret = zynqmp_r5_pm_request_tcm(dt_node, dev,
> > &pnode_id);
> > > +			if (ret < 0)
> > > +				return ret;
> > > +
> > > +			/* add carveout */
> > > +			size = resource_size(&rsc);
> > > +			mem = rproc_mem_entry_init(dev, NULL, rsc.start,
> > > +						   (int)size, rsc.start,
> > > +						   tcm_mem_alloc,
> > > +						   tcm_mem_release,
> > > +						   rsc.name);
> > > +			if (!mem)
> > > +				return -ENOMEM;
> > > +
> > > +			mem->priv = (void *)(u64)pnode_id;
> > > +			rproc_add_carveout(rproc, mem);
> > > +		}
> > > +	}
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/*
> > > + * when loading firmware, load in needed DDR, TCM memory regions and
> > wire
> > > + * these into remoteproc core's carveouts
> > > + */
> > > +static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware
> > *fw)
> > > +{
> > > +	int ret;
> > > +
> > > +	ret = parse_tcm_banks(rproc);
> > > +	if (ret)
> > > +		return ret;
> > > +
> > > +	ret = parse_mem_regions(rproc);
> > > +	if (ret)
> > > +		return ret;
> > > +
> > > +	ret = rproc_elf_load_rsc_table(rproc, fw);
> > > +	if (ret == -EINVAL) {
> > > +		/*
> > > +		 * resource table only required for IPC.
> > > +		 * if not present, this is not necessarily an error;
> > > +		 * for example, loading r5 hello world application
> > > +		 * so simply inform user and keep going.
> > > +		 */
> > > +		dev_info(&rproc->dev, "no resource table found.\n");
> > > +		ret = 0;
> > > +	}
> > > +	return ret;
> > > +}
> > > +
> > > +/* kick a firmware */
> > > +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
> > > +{
> > > +	struct sk_buff *skb;
> > > +	unsigned int skb_len;
> > > +	struct zynqmp_ipi_message *mb_msg;
> > > +	int ret;
> > > +
> > > +	struct device *dev = rproc->dev.parent;
> > > +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> > > +
> > > +	skb_len = (unsigned int)(sizeof(vqid) + sizeof(mb_msg));
> > > +	skb = alloc_skb(skb_len, GFP_ATOMIC);
> > > +	if (!skb)
> > > +		return;
> > > +
> > > +	mb_msg = (struct zynqmp_ipi_message *)skb_put(skb, skb_len);
> > > +	mb_msg->len = sizeof(vqid);
> > > +	memcpy(mb_msg->data, &vqid, sizeof(vqid));
> > > +	skb_queue_tail(&z_rproc->tx_mc_skbs, skb);
> > > +	ret = mbox_send_message(z_rproc->tx_chan, mb_msg);
> > > +	if (ret < 0) {
> > > +		dev_warn(dev, "Failed to kick remote.\n");
> > > +		skb_dequeue_tail(&z_rproc->tx_mc_skbs);
> > > +		kfree_skb(skb);
> > > +	}
> > > +}
> > > +
> > > +static struct rproc_ops zynqmp_r5_rproc_ops = {
> > > +	.start		= zynqmp_r5_rproc_start,
> > > +	.stop		= zynqmp_r5_rproc_stop,
> > > +	.load		= rproc_elf_load_segments,
> > > +	.parse_fw	= zynqmp_r5_parse_fw,
> > > +	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
> > > +	.sanity_check	= rproc_elf_sanity_check,
> > > +	.get_boot_addr	= rproc_elf_get_boot_addr,
> > > +	.kick		= zynqmp_r5_rproc_kick,
> > > +};
> > > +
> > > +/**
> > > + * event_notified_idr_cb() - event notified idr callback
> > > + * @id: idr id
> > > + * @ptr: pointer to idr private data
> > > + * @data: data passed to idr_for_each callback
> > > + *
> > > + * Pass notification to remoteproc virtio
> > > + *
> > > + * Return: 0. having return is to satisfy the idr_for_each() function
> > > + *          pointer input argument requirement.
> > > + **/
> > > +static int event_notified_idr_cb(int id, void *ptr, void *data)
> > > +{
> > > +	struct rproc *rproc = data;
> > > +
> > > +	(void)rproc_vq_interrupt(rproc, id);
> > > +	return 0;
> > > +}
> > > +
> > > +/**
> > > + * handle_event_notified() - remoteproc notification work funciton
> > > + * @work: pointer to the work structure
> > > + *
> > > + * It checks each registered remoteproc notify IDs.
> > > + */
> > > +static void handle_event_notified(struct work_struct *work)
> > > +{
> > > +	struct rproc *rproc;
> > > +	struct zynqmp_r5_rproc *z_rproc;
> > > +
> > > +	z_rproc = container_of(work, struct zynqmp_r5_rproc, mbox_work);
> > > +
> > > +	(void)mbox_send_message(z_rproc->rx_chan, NULL);
> > > +	rproc = z_rproc->rproc;
> > > +	/*
> > > +	 * We only use IPI for interrupt. The firmware side may or may
> > > +	 * not write the notifyid when it trigger IPI.
> > > +	 * And thus, we scan through all the registered notifyids.
> > > +	 */
> > > +	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
> > > +}
> > > +
> > > +/**
> > > + * zynqmp_r5_mb_rx_cb() - Receive channel mailbox callback
> > > + * @cl: mailbox client
> > > + * @mssg: message pointer
> > > + *
> > > + * It will schedule the R5 notification work.
> > > + */
> > > +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *mssg)
> > > +{
> > > +	struct zynqmp_r5_rproc *z_rproc;
> > > +
> > > +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, rx_mc);
> > > +	if (mssg) {
> > > +		struct zynqmp_ipi_message *ipi_msg, *buf_msg;
> > > +		size_t len;
> > > +
> > > +		ipi_msg = (struct zynqmp_ipi_message *)mssg;
> > > +		buf_msg = (struct zynqmp_ipi_message *)z_rproc->rx_mc_buf;
> > > +		len = (ipi_msg->len >= IPI_BUF_LEN_MAX) ?
> > > +		      IPI_BUF_LEN_MAX : ipi_msg->len;
> > > +		buf_msg->len = len;
> > > +		memcpy(buf_msg->data, ipi_msg->data, len);
> > > +	}
> > > +	schedule_work(&z_rproc->mbox_work);
> > > +}
> > > +
> > > +/**
> > > + * zynqmp_r5_mb_tx_done() - Request has been sent to the remote
> > > + * @cl: mailbox client
> > > + * @mssg: pointer to the message which has been sent
> > > + * @r: status of last TX - OK or error
> > > + *
> > > + * It will be called by the mailbox framework when the last TX has done.
> > > + */
> > > +static void zynqmp_r5_mb_tx_done(struct mbox_client *cl, void *mssg, int
> > r)
> > > +{
> > > +	struct zynqmp_r5_rproc *z_rproc;
> > > +	struct sk_buff *skb;
> > > +
> > > +	if (!mssg)
> > > +		return;
> > > +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, tx_mc);
> > > +	skb = skb_dequeue(&z_rproc->tx_mc_skbs);
> > > +	kfree_skb(skb);
> > > +}
> > > +
> > > +/**
> > > + * zynqmp_r5_setup_mbox() - Setup mailboxes
> > > + *			    this is used for each individual R5 core
> > > + *
> > > + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> > > + * @node: pointer of the device node
> > > + *
> > > + * Function to setup mailboxes to talk to RPU.
> > > + *
> > > + * Return: 0 for success, negative value for failure.
> > > + */
> > > +static int zynqmp_r5_setup_mbox(struct zynqmp_r5_rproc *z_rproc,
> > > +				struct device_node *node)
> > > +{
> > > +	struct mbox_client *mclient;
> > > +
> > > +	/* Setup TX mailbox channel client */
> > > +	mclient = &z_rproc->tx_mc;
> > > +	mclient->rx_callback = NULL;
> > > +	mclient->tx_block = false;
> > > +	mclient->knows_txdone = false;
> > > +	mclient->tx_done = zynqmp_r5_mb_tx_done;
> > > +	mclient->dev = z_rproc->dev;
> > > +
> > > +	/* Setup TX mailbox channel client */
> > > +	mclient = &z_rproc->rx_mc;
> > > +	mclient->dev = z_rproc->dev;
> > > +	mclient->rx_callback = zynqmp_r5_mb_rx_cb;
> > > +	mclient->tx_block = false;
> > > +	mclient->knows_txdone = false;
> > > +
> > > +	INIT_WORK(&z_rproc->mbox_work, handle_event_notified);
> > > +
> > > +	/* Request TX and RX channels */
> > > +	z_rproc->tx_chan = mbox_request_channel_byname(&z_rproc-
> > >tx_mc, "tx");
> > > +	if (IS_ERR(z_rproc->tx_chan)) {
> > > +		dev_err(z_rproc->dev, "failed to request mbox tx channel.\n");
> > > +		z_rproc->tx_chan = NULL;
> > > +		return -EINVAL;
> > > +	}
> > > +
> > > +	z_rproc->rx_chan = mbox_request_channel_byname(&z_rproc-
> > >rx_mc, "rx");
> > > +	if (IS_ERR(z_rproc->rx_chan)) {
> > > +		dev_err(z_rproc->dev, "failed to request mbox rx channel.\n");
> > > +		z_rproc->rx_chan = NULL;
> > > +		return -EINVAL;
> > > +	}
> > > +	skb_queue_head_init(&z_rproc->tx_mc_skbs);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +/**
> > > + * zynqmp_r5_probe() - Probes ZynqMP R5 processor device node
> > > + *		       this is called for each individual R5 core to
> > > + *		       set up mailbox, Xilinx platform manager unique ID,
> > > + *		       add to rproc core
> > > + *
> > > + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> > > + * @pdev: domain platform device for current R5 core
> > > + * @node: pointer of the device node for current R5 core
> > > + * @rpu_mode: mode to configure RPU, split or lockstep
> > > + * @core: Xilinx specific remoteproc structure used later to link
> > > + *           in to cluster of cores
> > 
> > This still does not match the param list (there's no z_rproc)
> > 
> Isn't core in this context the z_rproc? I can change the argument name if that is what you mean

The doc lists 5 arguments (z_rproc, pdev, node, rpu_mode, core), but the
function takes 4 arguments (pdev, node, rpu_mode, core).

> > > + *
> > > + * Function to retrieve the information of the ZynqMP R5 device node.
> > > + *
> > > + * Return: 0 for success, negative value for failure.
> > > + */
> > > +static int zynqmp_r5_probe(struct platform_device *pdev,
> > > +			   struct device_node *node,
> > > +			   enum rpu_oper_mode rpu_mode,
> > > +			   struct zynqmp_r5_rproc **core)
> > > +{
> > > +	int ret;
> > > +	struct device *dev = &pdev->dev;
> > > +	struct rproc *rproc_ptr;
> > > +	struct zynqmp_r5_rproc *z_rproc;
> > > +
> > > +	/* Allocate remoteproc instance */
> > > +	/* dev here is parent device of the allocated rproc's dev field */
> > > +	rproc_ptr = rproc_alloc(dev, dev_name(dev), &zynqmp_r5_rproc_ops,
> > > +				NULL, sizeof(struct zynqmp_r5_rproc));
> > > +	if (!rproc_ptr)
> > > +		return -ENOMEM;
> > > +	z_rproc = rproc_ptr->priv;
> > > +	z_rproc->rproc = rproc_ptr;
> > > +	z_rproc->dev = dev;
> > > +	/* Set up DMA mask */
> > > +	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
> > > +	if (ret)
> > > +		goto error;
> > > +	/* Get R5 power domain node */
> > > +	ret = of_property_read_u32(node, "pnode-id", &z_rproc->pnode_id);
> > > +	if (ret)
> > > +		goto error;
> > > +
> > > +	ret = r5_set_mode(z_rproc, rpu_mode);
> 	> > +	if (ret)
> > > +		return ret;
> > 
> > goto error?
> > 
> Ok will change
> > > +
> > > +	if (of_property_read_bool(node, "mboxes")) {
> > > +		ret = zynqmp_r5_setup_mbox(z_rproc, node);
> > > +		if (ret)
> > > +			goto error;
> > > +	}
> > > +	/* Add R5 remoteproc */
> > > +	ret = rproc_add(rproc_ptr);
> > > +	if (ret)
> > > +		goto error;
> > > +	*core = z_rproc;
> > > +
> > > +	return 0;
> > > +error:
> > > +	if (z_rproc->rproc)
> > 
> > How would this ever be NULL?
> Ok will change
> > 
> > > +		rproc_free(z_rproc->rproc);
> > > +	z_rproc->rproc = NULL;
> > 
> > This looks like another use-after-free, but it also seems unnecessary.
> Ok will change
> > 
> > > +	return ret;
> > > +}
> > > +
> > > +/*
> > > + * called when driver is probed, for each R5 core specified in DT,
> > > + * setup as needed to do remoteproc-related operations
> > > + */
> > > +static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev)
> > > +{
> > > +	int ret, i;
> > > +	struct device *dev = &pdev->dev;
> > > +	struct device_node *nc;
> > > +	enum rpu_oper_mode rpu_mode;
> > > +	struct list_head *cluster; /* list to track each core's rproc */
> > > +	struct zynqmp_r5_rproc *z_rproc;
> > > +	struct platform_device *child_pdev;
> > > +
> > > +	rpu_mode = of_property_read_bool(dev->of_node, "lockstep-mode")
> > ?
> > > +		   PM_RPU_MODE_LOCKSTEP : PM_RPU_MODE_SPLIT;
> > > +	dev_dbg(dev, "RPU configuration: %s\n",
> > > +		rpu_mode == PM_RPU_MODE_LOCKSTEP ? "lockstep" :
> > "split");
> > > +
> > > +	/*
> > > +	 * if 2 RPUs provided but one is lockstep, then we have an
> > > +	 * invalid configuration.
> > > +	 */
> > > +	i = of_get_available_child_count(dev->of_node);
> > > +	if ((rpu_mode == PM_RPU_MODE_LOCKSTEP && i != 1) || i >
> > MAX_RPROCS)
> > > +		return -EINVAL;
> > > +
> > > +	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
> > > +	if (!cluster)
> > > +		return -ENOMEM;
> > > +	INIT_LIST_HEAD(cluster);
> > > +
> > > +	ret = devm_of_platform_populate(dev);
> > > +	if (ret) {
> > > +		dev_err(dev, "devm_of_platform_populate failed, ret =
> > %d\n",
> > > +			ret);
> > > +		return ret;
> > > +	}
> > > +
> > > +	/* probe each individual r5 core's remoteproc-related info */
> > > +	i = 0;
> > > +	for_each_available_child_of_node(dev->of_node, nc) {
> > > +		child_pdev = of_find_device_by_node(nc);
> > > +		if (!child_pdev) {
> > > +			dev_err(dev, "could not get R5 core platform
> > device\n");
> > > +			ret = -ENODEV;
> > > +			goto out;
> > > +		}
> > > +		ret = zynqmp_r5_probe(child_pdev, nc, rpu_mode, &z_rproc);
> > > +		dev_dbg(dev, "%s to probe rpu %pOF\n",
> > > +			ret ? "Failed" : "Able",
> > > +			nc);
> > > +		if (!z_rproc)
> > > +			ret = -EINVAL;
> > > +		if (ret)
> > > +			goto out;
> > > +
> > > +		list_add_tail(&z_rproc->elem, cluster);
> > > +		i++;
> > > +	}
> > > +	/* wire in so each core can be cleaned up at drive remove */
> > > +	platform_set_drvdata(pdev, cluster);
> > > +	ret = 0;
> > 
> > Just return 0 here instead of resetting ret.
> Ok will change
> > 
> > > +out:
> > > +	/* undo core0 upon any failures on core1 in split-mode */
> > > +	if (rpu_mode == PM_RPU_MODE_SPLIT && i == 1 && ret != 0) {
> > > +		z_rproc = container_of(cluster, struct zynqmp_r5_rproc,
> > elem);
> > 
> > This looks very wrong to me; cluster isn't a pointer to member of
> > zynqmp_r5_rproc. Was this path tested?
> > 
> It was tested and ran, but Ok will change
> > > +		if (z_rproc->rproc) {
> > 
> > How would z_rproc->rproc be NULL here?
> 
> Good point, will remove this check
> > 
> > > +			rproc_del(z_rproc->rproc);
> > > +			rproc_free(z_rproc->rproc);
> > > +		}
> > > +
> > > +		if (z_rproc->tx_chan)
> > > +			mbox_free_channel(z_rproc->tx_chan);
> > > +		if (z_rproc->rx_chan)
> > > +			mbox_free_channel(z_rproc->rx_chan);
> > 
> > Again, this looks like a use-after-free.
> > 
> Ok will change
> > > +	}
> > > +	return ret;
> > > +}
> > > +
> > > +/*
> > > + * for each core, clean up the following:
> > > + *	single rproc entry
> > > + *	mailbox tx, rx
> > > + */
> > > +static int zynqmp_r5_remoteproc_remove(struct platform_device *pdev)
> > > +{
> > > +	struct list_head *pos, *cluster = (struct list_head *)
> > > +					  platform_get_drvdata(pdev);
> > > +	struct zynqmp_r5_rproc *z_rproc = NULL;
> > > +	struct rproc *rproc = NULL;
> > > +
> > > +	list_for_each(pos, cluster) {
> > > +		z_rproc = list_entry(pos, struct zynqmp_r5_rproc, elem);
> > > +		rproc = z_rproc->rproc;
> > > +		if (rproc) {
> > > +			rproc_del(rproc);
> > > +			rproc_free(rproc);
> > > +		}
> > > +
> > > +		if (z_rproc->tx_chan)
> > > +			mbox_free_channel(z_rproc->tx_chan);
> > > +		if (z_rproc->rx_chan)
> > > +			mbox_free_channel(z_rproc->rx_chan);
> > 
> > The use-after-free issue I pointed out on v20 is still present here (and
> > has now been replicated in probe() as well).
> > 
> So for this and the other mbox_free_channel call, is it alright to
> check if the mbox property is in the z_rproc pointer's dev->of_node?
> If so then the setup_mbox was called and ran ok if the driver got this
> far.
> E.g.	 if (of_property_read_bool(z_rproc->dev->of_node, "mboxes")) {

I'm not sure if I understand what you're proposing here... but to me,
the most obvious way to fix a use-after-free is to not free the memory
until you're actually done using it. Moving the rproc_free until the
end of this block would accomplish that.

> 
> [...]
> 
> Thanks
> Ben

Re: [PATCH v21 5/5] remoteproc: Add initial zynqmp R5 remoteproc driver

[Michael Auchter]

On Mon, Nov 02, 2020 at 11:38:59AM -0800, Ben Levinsky wrote:
> R5 is included in Xilinx Zynq UltraScale MPSoC so by adding this
> remotproc driver, we can boot the R5 sub-system in different 2
> configurations -
> 	* split
> 	* lock-step
> 
> The Xilinx R5 Remoteproc Driver boots the R5's via calls to the Xilinx
> Platform Management Unit that handles the R5 configuration, memory access
> and R5 lifecycle management. The interface to this manager is done in this
> driver via zynqmp_pm_* function calls.
> 
> Signed-off-by: Wendy Liang <wendy.liang@xilinx.com>
> Signed-off-by: Michal Simek <michal.simek@xilinx.com>
> Signed-off-by: Ed Mooring <ed.mooring@xilinx.com>
> Signed-off-by: Jason Wu <j.wu@xilinx.com>
> Signed-off-by: Ben Levinsky <ben.levinsky@xilinx.com>
> ---
> Update Xilinx R5 Remoteproc Driver as follows:
> - update documentation for zynqmp_r5_probe
> - restructure so that cluster initialization code is all in one place
> - add memory allocation check for cluster
> - add error handling in case of second core fails at probe but first core succeeded.
>   to clean up the first core
> - remove unneeded lines in zynqmp_r5_remoteproc_remove
> ---
> 
>  drivers/remoteproc/Kconfig                |   8 +
>  drivers/remoteproc/Makefile               |   1 +
>  drivers/remoteproc/zynqmp_r5_remoteproc.c | 784 ++++++++++++++++++++++
>  3 files changed, 793 insertions(+)
>  create mode 100644 drivers/remoteproc/zynqmp_r5_remoteproc.c
> 
> diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
> index c6659dfea7c7..c2fe54b1d94f 100644
> --- a/drivers/remoteproc/Kconfig
> +++ b/drivers/remoteproc/Kconfig
> @@ -275,6 +275,14 @@ config TI_K3_DSP_REMOTEPROC
>  	  It's safe to say N here if you're not interested in utilizing
>  	  the DSP slave processors.
>  
> +config ZYNQMP_R5_REMOTEPROC
> +	tristate "ZynqMP R5 remoteproc support"
> +	depends on PM && ARCH_ZYNQMP
> +	select RPMSG_VIRTIO
> +	select ZYNQMP_IPI_MBOX
> +	help
> +	  Say y or m here to support ZynqMP R5 remote processors via the remote
> +	  processor framework.
>  endif # REMOTEPROC
>  
>  endmenu
> diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
> index 3dfa28e6c701..ef1abff654c2 100644
> --- a/drivers/remoteproc/Makefile
> +++ b/drivers/remoteproc/Makefile
> @@ -33,3 +33,4 @@ obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
>  obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
>  obj-$(CONFIG_STM32_RPROC)		+= stm32_rproc.o
>  obj-$(CONFIG_TI_K3_DSP_REMOTEPROC)	+= ti_k3_dsp_remoteproc.o
> +obj-$(CONFIG_ZYNQMP_R5_REMOTEPROC)	+= zynqmp_r5_remoteproc.o
> diff --git a/drivers/remoteproc/zynqmp_r5_remoteproc.c b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> new file mode 100644
> index 000000000000..993bd72e5664
> --- /dev/null
> +++ b/drivers/remoteproc/zynqmp_r5_remoteproc.c
> @@ -0,0 +1,784 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Zynq R5 Remote Processor driver
> + *
> + * Based on origin OMAP and Zynq Remote Processor driver
> + *
> + */
> +
> +#include <linux/firmware/xlnx-zynqmp.h>
> +#include <linux/interrupt.h>
> +#include <linux/kernel.h>
> +#include <linux/list.h>
> +#include <linux/mailbox_client.h>
> +#include <linux/mailbox/zynqmp-ipi-message.h>
> +#include <linux/module.h>
> +#include <linux/of_address.h>
> +#include <linux/of_platform.h>
> +#include <linux/of_reserved_mem.h>
> +#include <linux/platform_device.h>
> +#include <linux/remoteproc.h>
> +#include <linux/skbuff.h>
> +#include <linux/sysfs.h>
> +
> +#include "remoteproc_internal.h"
> +
> +#define MAX_RPROCS	2 /* Support up to 2 RPU */
> +#define MAX_MEM_PNODES	4 /* Max power nodes for one RPU memory instance */
> +
> +#define BANK_LIST_PROP	"meta-memory-regions"
> +#define DDR_LIST_PROP	"memory-regions"
> +
> +/* IPI buffer MAX length */
> +#define IPI_BUF_LEN_MAX	32U
> +/* RX mailbox client buffer max length */
> +#define RX_MBOX_CLIENT_BUF_MAX	(IPI_BUF_LEN_MAX + \
> +				 sizeof(struct zynqmp_ipi_message))
> +
> +/**
> + * struct zynqmp_r5_mem - zynqmp rpu memory data
> + * @pnode_id: TCM power domain ids
> + * @res: memory resource
> + * @node: list node
> + */
> +struct zynqmp_r5_mem {
> +	u32 pnode_id[MAX_MEM_PNODES];
> +	struct resource res;
> +	struct list_head node;
> +};
> +
> +/**
> + * struct zynqmp_r5_rproc - zynqmp rpu remote processor state
> + *			    this is for each individual R5 core's state
> + *
> + * @rx_mc_buf: rx mailbox client buffer to save the rx message
> + * @tx_mc: tx mailbox client
> + * @rx_mc: rx mailbox client * @dev: device of RPU instance
> + * @mbox_work: mbox_work for the RPU remoteproc
> + * @tx_mc_skbs: socket buffers for tx mailbox client
> + * @dev: device of RPU instance
> + * @rproc: rproc handle
> + * @tx_chan: tx mailbox channel
> + * @rx_chan: rx mailbox channel
> + * @pnode_id: RPU CPU power domain id
> + * @elem: linked list item
> + * @dt_node: device tree node that holds information for 1 R5 core.
> + */
> +struct zynqmp_r5_rproc {
> +	unsigned char rx_mc_buf[RX_MBOX_CLIENT_BUF_MAX];
> +	struct mbox_client tx_mc;
> +	struct mbox_client rx_mc;
> +	struct work_struct mbox_work;
> +	struct sk_buff_head tx_mc_skbs;
> +	struct device *dev;
> +	struct rproc *rproc;
> +	struct mbox_chan *tx_chan;
> +	struct mbox_chan *rx_chan;
> +	u32 pnode_id;
> +	struct list_head elem;
> +};
> +
> +/*
> + * r5_set_mode - set RPU operation mode
> + * @z_rproc: Remote processor private data
> + * @rpu_mode: mode specified by device tree to configure the RPU to
> + *
> + * set RPU operation mode
> + *
> + * Return: 0 for success, negative value for failure
> + */
> +static int r5_set_mode(struct zynqmp_r5_rproc *z_rproc,
> +		       enum rpu_oper_mode rpu_mode)
> +{
> +	enum rpu_tcm_comb tcm_mode;
> +	enum rpu_oper_mode cur_rpu_mode;
> +	int ret;
> +
> +	ret = zynqmp_pm_get_rpu_mode(z_rproc->pnode_id, &cur_rpu_mode);
> +	if (ret < 0)
> +		return ret;
> +
> +	if (rpu_mode != cur_rpu_mode) {
> +		ret = zynqmp_pm_set_rpu_mode(z_rproc->pnode_id,
> +					     rpu_mode);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	tcm_mode = (rpu_mode == PM_RPU_MODE_LOCKSTEP) ?
> +		    PM_RPU_TCM_COMB : PM_RPU_TCM_SPLIT;
> +	return zynqmp_pm_set_tcm_config(z_rproc->pnode_id, tcm_mode);
> +}
> +
> +/*
> + * release TCM banks when powering down R5 core
> + */
> +static int tcm_mem_release(struct rproc *rproc, struct rproc_mem_entry *mem)
> +{
> +	u32 pnode_id = (u64)mem->priv;
> +
> +	iounmap(mem->va);
> +	return zynqmp_pm_release_node(pnode_id);
> +}
> +
> +/*
> + * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
> + * try to use xpm wake call to wake R5 core
> + */
> +static int zynqmp_r5_rproc_start(struct rproc *rproc)
> +{
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	enum rpu_boot_mem bootmem;
> +
> +	bootmem = (rproc->bootaddr & 0xF0000000) == 0xF0000000 ?
> +		  PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
> +
> +	dev_dbg(rproc->dev.parent, "RPU boot from %s.",
> +		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
> +
> +	return zynqmp_pm_request_wake(z_rproc->pnode_id, 1,
> +				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
> +}
> +
> +/*
> + * given ID corresponding to R5 core in Xilinx Platform management (xpm) API,
> + * try to use xpm power down call to power off R5 core
> + */
> +static int zynqmp_r5_rproc_stop(struct rproc *rproc)
> +{
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +
> +	return zynqmp_pm_force_pwrdwn(z_rproc->pnode_id,
> +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> +}
> +
> +/*
> + * map in physical addr for  DDR mem carveout in rproc
> + */
> +static int zynqmp_r5_rproc_mem_alloc(struct rproc *rproc,
> +				     struct rproc_mem_entry *mem)
> +{
> +	void *va;
> +
> +	va = ioremap_wc(mem->dma, mem->len);
> +	if (IS_ERR_OR_NULL(va))
> +		return -ENOMEM;
> +
> +	/* Update memory entry va */
> +	mem->va = va;
> +
> +	return 0;
> +}
> +
> +/* unmap rproc_mem_entry virtual addr */
> +static int zynqmp_r5_rproc_mem_release(struct rproc *rproc,
> +				       struct rproc_mem_entry *mem)
> +{
> +	iounmap(mem->va);
> +	return 0;
> +}
> +
> +/* construct rproc mem carveouts for DDR regions specified in device tree */
> +static int parse_mem_regions(struct rproc *rproc)
> +{
> +	int num_mems, i;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	struct device *dev = &rproc->dev;
> +	struct device_node *np = z_rproc->dev->of_node;
> +	struct rproc_mem_entry *mem;
> +
> +	num_mems = of_count_phandle_with_args(np, DDR_LIST_PROP, NULL);
> +	if (num_mems <= 0)
> +		return 0;
> +
> +	for (i = 0; i < num_mems; i++) {
> +		struct device_node *node;
> +		struct reserved_mem *rmem;
> +
> +		node = of_parse_phandle(np, DDR_LIST_PROP, i);
> +		if (!node)
> +			return -EINVAL;
> +
> +		rmem = of_reserved_mem_lookup(node);
> +		if (!rmem)
> +			return -EINVAL;
> +
> +		if (strstr(node->name, "vdev0vring")) {
> +			int vring_id;
> +			char name[16];
> +
> +			/*
> +			 * expecting form of "rpuXvdev0vringX as documented
> +			 * in xilinx remoteproc device tree binding
> +			 */
> +			if (strlen(node->name) < 15) {
> +				dev_err(dev, "%pOF is less than 14 chars",
> +					node);
> +				return -EINVAL;
> +			}
> +
> +			/*
> +			 * can be 1 of multiple vring IDs per IPC channel
> +			 * e.g. 'vdev0vring0' and 'vdev0vring1'
> +			 */
> +			vring_id = node->name[14] - '0';
> +			snprintf(name, sizeof(name), "vdev0vring%d", vring_id);
> +			/* Register vring */
> +			mem = rproc_mem_entry_init(dev, NULL,
> +						   (dma_addr_t)rmem->base,
> +						   rmem->size, rmem->base,
> +						   zynqmp_r5_rproc_mem_alloc,
> +						   zynqmp_r5_rproc_mem_release,
> +						   name);
> +		} else {
> +			/* Register DMA region */
> +			int (*alloc)(struct rproc *r,
> +				     struct rproc_mem_entry *rme);
> +			int (*release)(struct rproc *r,
> +				       struct rproc_mem_entry *rme);
> +			char name[20];
> +
> +			if (strstr(node->name, "vdev0buffer")) {
> +				alloc = NULL;
> +				release = NULL;
> +				strcpy(name, "vdev0buffer");
> +			} else {
> +				alloc = zynqmp_r5_rproc_mem_alloc;
> +				release = zynqmp_r5_rproc_mem_release;
> +				strcpy(name, node->name);
> +			}
> +
> +			mem = rproc_mem_entry_init(dev, NULL,
> +						   (dma_addr_t)rmem->base,
> +						   rmem->size, rmem->base,
> +						   alloc, release, name);
> +		}
> +		if (!mem)
> +			return -ENOMEM;
> +
> +		rproc_add_carveout(rproc, mem);
> +	}
> +
> +	return 0;
> +}
> +
> +/* call Xilinx Platform manager to request access to TCM bank */
> +static int zynqmp_r5_pm_request_tcm(struct device_node *tcm_node,
> +				    struct device *dev, u32 *pnode_id)
> +{
> +	int ret;
> +
> +	ret = of_property_read_u32(tcm_node, "pnode-id", pnode_id);
> +	if (ret)
> +		return ret;
> +
> +	return zynqmp_pm_request_node(*pnode_id, ZYNQMP_PM_CAPABILITY_ACCESS, 0,
> +				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
> +}
> +
> +/*
> + * Given TCM bank entry,
> + * this callback will set device address for R5 running on TCM
> + * and also setup virtual address for TCM bank remoteproc carveout
> + */
> +static int tcm_mem_alloc(struct rproc *rproc,
> +			 struct rproc_mem_entry *mem)
> +{
> +	void *va;
> +	struct device *dev = rproc->dev.parent;
> +
> +	va = ioremap_wc(mem->dma, mem->len);
> +	if (IS_ERR_OR_NULL(va))
> +		return -ENOMEM;
> +
> +	/* Update memory entry va */
> +	mem->va = va;
> +
> +	va = devm_ioremap_wc(dev, mem->da, mem->len);
> +	if (!va)
> +		return -ENOMEM;
> +	/* As R5 is 32 bit, wipe out extra high bits */
> +	mem->da &= 0x000fffff;
> +	/*
> +	 * TCM Banks 0A and 0B (0xffe00000 and 0xffe20000)
> +	 * are handled with the above line of code so do nothing
> +	 * for this 2 banks
> +	 */
> +
> +	/*
> +	 * TCM Banks 1A and 1B (0xffe90000 and 0xffeb0000) still
> +	 * need to be translated to 0x0 and 0x20000
> +	 */
> +	if (mem->da == 0x90000 || mem->da == 0xB0000)
> +		mem->da -= 0x90000;
> +
> +	/* if translated TCM bank address is not valid report error */
> +	if (mem->da != 0x0 && mem->da != 0x20000) {
> +		dev_err(dev, "invalid TCM bank address: %x\n", mem->da);
> +		return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
> +/*
> + * Given R5 node in remoteproc instance
> + * allocate remoteproc carveout for TCM memory
> + * needed for firmware to be loaded
> + */
> +static int parse_tcm_banks(struct rproc *rproc)
> +{
> +	int i, num_banks;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +	struct device *dev = &rproc->dev;
> +	struct device_node *r5_node = z_rproc->dev->of_node;
> +
> +	/* go through TCM banks for r5 node */
> +	num_banks = of_count_phandle_with_args(r5_node, BANK_LIST_PROP, NULL);
> +	if (num_banks <= 0) {
> +		dev_err(dev, "need to specify TCM banks\n");
> +		return -EINVAL;
> +	}
> +	for (i = 0; i < num_banks; i++) {
> +		struct resource rsc;
> +		resource_size_t size;
> +		struct device_node *dt_node;
> +		struct rproc_mem_entry *mem;
> +		int ret;
> +		u32 pnode_id; /* zynqmp_pm* fn's expect u32 */
> +
> +		dt_node = of_parse_phandle(r5_node, BANK_LIST_PROP, i);
> +		if (!dt_node)
> +			return -EINVAL;
> +
> +		if (of_device_is_available(dt_node)) {
> +			ret = of_address_to_resource(dt_node, 0, &rsc);
> +			if (ret < 0)
> +				return ret;
> +
> +			ret = zynqmp_r5_pm_request_tcm(dt_node, dev, &pnode_id);
> +			if (ret < 0)
> +				return ret;
> +
> +			/* add carveout */
> +			size = resource_size(&rsc);
> +			mem = rproc_mem_entry_init(dev, NULL, rsc.start,
> +						   (int)size, rsc.start,
> +						   tcm_mem_alloc,
> +						   tcm_mem_release,
> +						   rsc.name);
> +			if (!mem)
> +				return -ENOMEM;
> +
> +			mem->priv = (void *)(u64)pnode_id;
> +			rproc_add_carveout(rproc, mem);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +/*
> + * when loading firmware, load in needed DDR, TCM memory regions and wire
> + * these into remoteproc core's carveouts
> + */
> +static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware *fw)
> +{
> +	int ret;
> +
> +	ret = parse_tcm_banks(rproc);
> +	if (ret)
> +		return ret;
> +
> +	ret = parse_mem_regions(rproc);
> +	if (ret)
> +		return ret;
> +
> +	ret = rproc_elf_load_rsc_table(rproc, fw);
> +	if (ret == -EINVAL) {
> +		/*
> +		 * resource table only required for IPC.
> +		 * if not present, this is not necessarily an error;
> +		 * for example, loading r5 hello world application
> +		 * so simply inform user and keep going.
> +		 */
> +		dev_info(&rproc->dev, "no resource table found.\n");
> +		ret = 0;
> +	}
> +	return ret;
> +}
> +
> +/* kick a firmware */
> +static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
> +{
> +	struct sk_buff *skb;
> +	unsigned int skb_len;
> +	struct zynqmp_ipi_message *mb_msg;
> +	int ret;
> +
> +	struct device *dev = rproc->dev.parent;
> +	struct zynqmp_r5_rproc *z_rproc = rproc->priv;
> +
> +	skb_len = (unsigned int)(sizeof(vqid) + sizeof(mb_msg));
> +	skb = alloc_skb(skb_len, GFP_ATOMIC);
> +	if (!skb)
> +		return;
> +
> +	mb_msg = (struct zynqmp_ipi_message *)skb_put(skb, skb_len);
> +	mb_msg->len = sizeof(vqid);
> +	memcpy(mb_msg->data, &vqid, sizeof(vqid));
> +	skb_queue_tail(&z_rproc->tx_mc_skbs, skb);
> +	ret = mbox_send_message(z_rproc->tx_chan, mb_msg);
> +	if (ret < 0) {
> +		dev_warn(dev, "Failed to kick remote.\n");
> +		skb_dequeue_tail(&z_rproc->tx_mc_skbs);
> +		kfree_skb(skb);
> +	}
> +}
> +
> +static struct rproc_ops zynqmp_r5_rproc_ops = {
> +	.start		= zynqmp_r5_rproc_start,
> +	.stop		= zynqmp_r5_rproc_stop,
> +	.load		= rproc_elf_load_segments,
> +	.parse_fw	= zynqmp_r5_parse_fw,
> +	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
> +	.sanity_check	= rproc_elf_sanity_check,
> +	.get_boot_addr	= rproc_elf_get_boot_addr,
> +	.kick		= zynqmp_r5_rproc_kick,
> +};
> +
> +/**
> + * event_notified_idr_cb() - event notified idr callback
> + * @id: idr id
> + * @ptr: pointer to idr private data
> + * @data: data passed to idr_for_each callback
> + *
> + * Pass notification to remoteproc virtio
> + *
> + * Return: 0. having return is to satisfy the idr_for_each() function
> + *          pointer input argument requirement.
> + **/
> +static int event_notified_idr_cb(int id, void *ptr, void *data)
> +{
> +	struct rproc *rproc = data;
> +
> +	(void)rproc_vq_interrupt(rproc, id);
> +	return 0;
> +}
> +
> +/**
> + * handle_event_notified() - remoteproc notification work funciton
> + * @work: pointer to the work structure
> + *
> + * It checks each registered remoteproc notify IDs.
> + */
> +static void handle_event_notified(struct work_struct *work)
> +{
> +	struct rproc *rproc;
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	z_rproc = container_of(work, struct zynqmp_r5_rproc, mbox_work);
> +
> +	(void)mbox_send_message(z_rproc->rx_chan, NULL);
> +	rproc = z_rproc->rproc;
> +	/*
> +	 * We only use IPI for interrupt. The firmware side may or may
> +	 * not write the notifyid when it trigger IPI.
> +	 * And thus, we scan through all the registered notifyids.
> +	 */
> +	idr_for_each(&rproc->notifyids, event_notified_idr_cb, rproc);
> +}
> +
> +/**
> + * zynqmp_r5_mb_rx_cb() - Receive channel mailbox callback
> + * @cl: mailbox client
> + * @mssg: message pointer
> + *
> + * It will schedule the R5 notification work.
> + */
> +static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *mssg)
> +{
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, rx_mc);
> +	if (mssg) {
> +		struct zynqmp_ipi_message *ipi_msg, *buf_msg;
> +		size_t len;
> +
> +		ipi_msg = (struct zynqmp_ipi_message *)mssg;
> +		buf_msg = (struct zynqmp_ipi_message *)z_rproc->rx_mc_buf;
> +		len = (ipi_msg->len >= IPI_BUF_LEN_MAX) ?
> +		      IPI_BUF_LEN_MAX : ipi_msg->len;
> +		buf_msg->len = len;
> +		memcpy(buf_msg->data, ipi_msg->data, len);
> +	}
> +	schedule_work(&z_rproc->mbox_work);
> +}
> +
> +/**
> + * zynqmp_r5_mb_tx_done() - Request has been sent to the remote
> + * @cl: mailbox client
> + * @mssg: pointer to the message which has been sent
> + * @r: status of last TX - OK or error
> + *
> + * It will be called by the mailbox framework when the last TX has done.
> + */
> +static void zynqmp_r5_mb_tx_done(struct mbox_client *cl, void *mssg, int r)
> +{
> +	struct zynqmp_r5_rproc *z_rproc;
> +	struct sk_buff *skb;
> +
> +	if (!mssg)
> +		return;
> +	z_rproc = container_of(cl, struct zynqmp_r5_rproc, tx_mc);
> +	skb = skb_dequeue(&z_rproc->tx_mc_skbs);
> +	kfree_skb(skb);
> +}
> +
> +/**
> + * zynqmp_r5_setup_mbox() - Setup mailboxes
> + *			    this is used for each individual R5 core
> + *
> + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> + * @node: pointer of the device node
> + *
> + * Function to setup mailboxes to talk to RPU.
> + *
> + * Return: 0 for success, negative value for failure.
> + */
> +static int zynqmp_r5_setup_mbox(struct zynqmp_r5_rproc *z_rproc,
> +				struct device_node *node)
> +{
> +	struct mbox_client *mclient;
> +
> +	/* Setup TX mailbox channel client */
> +	mclient = &z_rproc->tx_mc;
> +	mclient->rx_callback = NULL;
> +	mclient->tx_block = false;
> +	mclient->knows_txdone = false;
> +	mclient->tx_done = zynqmp_r5_mb_tx_done;
> +	mclient->dev = z_rproc->dev;
> +
> +	/* Setup TX mailbox channel client */
> +	mclient = &z_rproc->rx_mc;
> +	mclient->dev = z_rproc->dev;
> +	mclient->rx_callback = zynqmp_r5_mb_rx_cb;
> +	mclient->tx_block = false;
> +	mclient->knows_txdone = false;
> +
> +	INIT_WORK(&z_rproc->mbox_work, handle_event_notified);
> +
> +	/* Request TX and RX channels */
> +	z_rproc->tx_chan = mbox_request_channel_byname(&z_rproc->tx_mc, "tx");
> +	if (IS_ERR(z_rproc->tx_chan)) {
> +		dev_err(z_rproc->dev, "failed to request mbox tx channel.\n");
> +		z_rproc->tx_chan = NULL;
> +		return -EINVAL;
> +	}
> +
> +	z_rproc->rx_chan = mbox_request_channel_byname(&z_rproc->rx_mc, "rx");
> +	if (IS_ERR(z_rproc->rx_chan)) {
> +		dev_err(z_rproc->dev, "failed to request mbox rx channel.\n");
> +		z_rproc->rx_chan = NULL;
> +		return -EINVAL;
> +	}
> +	skb_queue_head_init(&z_rproc->tx_mc_skbs);
> +
> +	return 0;
> +}
> +
> +/**
> + * zynqmp_r5_probe() - Probes ZynqMP R5 processor device node
> + *		       this is called for each individual R5 core to
> + *		       set up mailbox, Xilinx platform manager unique ID,
> + *		       add to rproc core
> + *
> + * @z_rproc: pointer to the ZynqMP R5 processor platform data
> + * @pdev: domain platform device for current R5 core
> + * @node: pointer of the device node for current R5 core
> + * @rpu_mode: mode to configure RPU, split or lockstep
> + * @core: Xilinx specific remoteproc structure used later to link
> + *           in to cluster of cores
> + *
> + * Function to retrieve the information of the ZynqMP R5 device node.
> + *
> + * Return: 0 for success, negative value for failure.
> + */
> +static int zynqmp_r5_probe(struct platform_device *pdev,
> +			   struct device_node *node,
> +			   enum rpu_oper_mode rpu_mode,
> +			   struct zynqmp_r5_rproc **core)
> +{
> +	int ret;
> +	struct device *dev = &pdev->dev;
> +	struct rproc *rproc_ptr;
> +	struct zynqmp_r5_rproc *z_rproc;
> +
> +	/* Allocate remoteproc instance */
> +	/* dev here is parent device of the allocated rproc's dev field */
> +	rproc_ptr = rproc_alloc(dev, dev_name(dev), &zynqmp_r5_rproc_ops,
> +				NULL, sizeof(struct zynqmp_r5_rproc));
> +	if (!rproc_ptr)
> +		return -ENOMEM;

Should rproc_ptr->auto_boot be set to false here?

I ask because I noticed the following error traces in my kernel log when
this driver is probed:

remoteproc remoteproc0: zynqmp-r5-remoteproc@0 is available
remoteproc remoteproc0: Direct firmware load for rproc-zynqmp-r5-remoteproc@0-fw failed with error -2
remoteproc remoteproc0: powering up zynqmp-r5-remoteproc@0
remoteproc remoteproc0: Direct firmware load for rproc-zynqmp-r5-remoteproc@0-fw failed with error -2
remoteproc remoteproc0: request_firmware failed: -2
remoteproc remoteproc1: zynqmp-r5-remoteproc@0 is available
remoteproc remoteproc1: Direct firmware load for rproc-zynqmp-r5-remoteproc@0-fw failed with error -2
remoteproc remoteproc1: powering up zynqmp-r5-remoteproc@0
remoteproc remoteproc1: Direct firmware load for rproc-zynqmp-r5-remoteproc@0-fw failed with error -2
remoteproc remoteproc1: request_firmware failed: -2

These are gone if I set auto_boot to false.

> +	z_rproc = rproc_ptr->priv;
> +	z_rproc->rproc = rproc_ptr;
> +	z_rproc->dev = dev;
> +	/* Set up DMA mask */
> +	ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
> +	if (ret)
> +		goto error;
> +	/* Get R5 power domain node */
> +	ret = of_property_read_u32(node, "pnode-id", &z_rproc->pnode_id);
> +	if (ret)
> +		goto error;
> +
> +	ret = r5_set_mode(z_rproc, rpu_mode);
> +	if (ret)
> +		return ret;
> +
> +	if (of_property_read_bool(node, "mboxes")) {
> +		ret = zynqmp_r5_setup_mbox(z_rproc, node);
> +		if (ret)
> +			goto error;
> +	}
> +	/* Add R5 remoteproc */
> +	ret = rproc_add(rproc_ptr);
> +	if (ret)
> +		goto error;
> +	*core = z_rproc;
> +
> +	return 0;
> +error:
> +	if (z_rproc->rproc)
> +		rproc_free(z_rproc->rproc);
> +	z_rproc->rproc = NULL;
> +	return ret;
> +}
> +
> +/*
> + * called when driver is probed, for each R5 core specified in DT,
> + * setup as needed to do remoteproc-related operations
> + */
> +static int zynqmp_r5_remoteproc_probe(struct platform_device *pdev)
> +{
> +	int ret, i;
> +	struct device *dev = &pdev->dev;
> +	struct device_node *nc;
> +	enum rpu_oper_mode rpu_mode;
> +	struct list_head *cluster; /* list to track each core's rproc */
> +	struct zynqmp_r5_rproc *z_rproc;
> +	struct platform_device *child_pdev;
> +
> +	rpu_mode = of_property_read_bool(dev->of_node, "lockstep-mode") ?
> +		   PM_RPU_MODE_LOCKSTEP : PM_RPU_MODE_SPLIT;
> +	dev_dbg(dev, "RPU configuration: %s\n",
> +		rpu_mode == PM_RPU_MODE_LOCKSTEP ? "lockstep" : "split");
> +
> +	/*
> +	 * if 2 RPUs provided but one is lockstep, then we have an
> +	 * invalid configuration.
> +	 */
> +	i = of_get_available_child_count(dev->of_node);
> +	if ((rpu_mode == PM_RPU_MODE_LOCKSTEP && i != 1) || i > MAX_RPROCS)
> +		return -EINVAL;
> +
> +	cluster = devm_kzalloc(dev, sizeof(*cluster), GFP_KERNEL);
> +	if (!cluster)
> +		return -ENOMEM;
> +	INIT_LIST_HEAD(cluster);
> +
> +	ret = devm_of_platform_populate(dev);
> +	if (ret) {
> +		dev_err(dev, "devm_of_platform_populate failed, ret = %d\n",
> +			ret);
> +		return ret;
> +	}
> +
> +	/* probe each individual r5 core's remoteproc-related info */
> +	i = 0;
> +	for_each_available_child_of_node(dev->of_node, nc) {
> +		child_pdev = of_find_device_by_node(nc);
> +		if (!child_pdev) {
> +			dev_err(dev, "could not get R5 core platform device\n");
> +			ret = -ENODEV;
> +			goto out;
> +		}
> +		ret = zynqmp_r5_probe(child_pdev, nc, rpu_mode, &z_rproc);
> +		dev_dbg(dev, "%s to probe rpu %pOF\n",
> +			ret ? "Failed" : "Able",
> +			nc);
> +		if (!z_rproc)
> +			ret = -EINVAL;
> +		if (ret)
> +			goto out;
> +
> +		list_add_tail(&z_rproc->elem, cluster);
> +		i++;
> +	}
> +	/* wire in so each core can be cleaned up at drive remove */
> +	platform_set_drvdata(pdev, cluster);
> +	ret = 0;
> +out:
> +	/* undo core0 upon any failures on core1 in split-mode */
> +	if (rpu_mode == PM_RPU_MODE_SPLIT && i == 1 && ret != 0) {
> +		z_rproc = container_of(cluster, struct zynqmp_r5_rproc, elem);
> +		if (z_rproc->rproc) {
> +			rproc_del(z_rproc->rproc);
> +			rproc_free(z_rproc->rproc);
> +		}
> +
> +		if (z_rproc->tx_chan)
> +			mbox_free_channel(z_rproc->tx_chan);
> +		if (z_rproc->rx_chan)
> +			mbox_free_channel(z_rproc->rx_chan);
> +	}
> +	return ret;
> +}
> +
> +/*
> + * for each core, clean up the following:
> + *	single rproc entry
> + *	mailbox tx, rx
> + */
> +static int zynqmp_r5_remoteproc_remove(struct platform_device *pdev)
> +{
> +	struct list_head *pos, *cluster = (struct list_head *)
> +					  platform_get_drvdata(pdev);
> +	struct zynqmp_r5_rproc *z_rproc = NULL;
> +	struct rproc *rproc = NULL;
> +
> +	list_for_each(pos, cluster) {
> +		z_rproc = list_entry(pos, struct zynqmp_r5_rproc, elem);
> +		rproc = z_rproc->rproc;
> +		if (rproc) {
> +			rproc_del(rproc);
> +			rproc_free(rproc);
> +		}
> +
> +		if (z_rproc->tx_chan)
> +			mbox_free_channel(z_rproc->tx_chan);
> +		if (z_rproc->rx_chan)
> +			mbox_free_channel(z_rproc->rx_chan);
> +	}
> +	return 0;
> +}
> +
> +/* Match table for OF platform binding */
> +static const struct of_device_id zynqmp_r5_remoteproc_match[] = {
> +	{ .compatible = "xlnx,zynqmp-r5-remoteproc", },
> +	{ /* end of list */ },
> +};
> +MODULE_DEVICE_TABLE(of, zynqmp_r5_remoteproc_match);
> +
> +static struct platform_driver zynqmp_r5_remoteproc_driver = {
> +	.probe = zynqmp_r5_remoteproc_probe,
> +	.remove = zynqmp_r5_remoteproc_remove,
> +	.driver = {
> +		.name = "zynqmp_r5_remoteproc",
> +		.of_match_table = zynqmp_r5_remoteproc_match,
> +	},
> +};
> +module_platform_driver(zynqmp_r5_remoteproc_driver);
> +
> +MODULE_AUTHOR("Ben Levinsky <ben.levinsky@xilinx.com>");
> +MODULE_LICENSE("GPL v2");
> -- 
> 2.17.1
>